Compare commits

..
Author SHA1 Message Date
Janik Besendorf 3222bcd98f Merge main and resolve upstream integration conflicts 2026-10-11 14:27:54 +02:00
besendorf 5169a46b59 Merge branch 'main' into fix_tombstone 2026-03-18 09:11:04 +01:00
Janik Besendorf 5a129430ae Add test artifact files for tombstone protobuf tests 2026-01-20 14:45:12 +01:00
Janik Besendorf 190a5f119f Remove unused TombstoneCrashArtifact init in test 2026-01-20 14:34:40 +01:00
Janik Besendorf 02c52f43c8 Tombstone: fall back to file timestamp on empty pb timestamp
- Allow empty protobuf timestamps to fall back to the file timestamp
- Update _parse_timestamp_string to accept a fallback
- Pass file_timestamp to its callers
- Add tests for empty and whitespace tombstone timestamps
- Adjust imports in bugreport module to include DumpsysADBState
  Tombstone: use file timestamp on empty pb
- skip 0 bytes tombstones and show debug warning
2026-01-20 13:38:07 +01:00
41 changed files with 307 additions and 969 deletions

No files matched your search

-5
View File
@@ -86,11 +86,6 @@ writing and installing them, and the
[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/)
for the entry points a package registers commands in.
Extraction commands run independent modules concurrently with four workers by
default. Use `--jobs 1` for sequential execution or another positive value to
set the worker limit. Dependencies retain their declared ordering, and parallel
console logs are printed as grouped module blocks.
## License
+1 -29
View File
@@ -46,33 +46,6 @@ on it, and runs the rest of the analysis. Selecting such a module with
A cycle in the dependency graph is a programming error rather than a
configuration problem: the command logs a warning and runs no modules at all.
## Parallel module execution
Extraction modules run concurrently, using four worker threads by default. Use
`--jobs INTEGER` to change the worker limit, or `--jobs 1` for sequential
execution and live per-line logging:
```bash
mvt-ios check-backup --jobs 8 --output ./out ./backup
```
The scheduler starts a module only after all of its declared dependencies have
completed. Results, alerts, and timeline entries are still aggregated in stable
topological order. During parallel execution, console records from each module
are buffered and printed together as a labeled block when that module finishes;
`command.log` is written immediately and retains the complete log stream.
Parallel-safe modules must log through `self.log`, must not write directly to
stdout or stderr, and must not mutate shared global state. A module that uses a
non-thread-safe resource or direct terminal output can opt out:
```python
class TerminalModule(MVTModule):
parallel_safe = False
```
Such modules run synchronously and exclusively after active workers finish.
## Custom modules
MVT's module-running `check-*` commands can run forensic modules which are not
@@ -409,8 +382,7 @@ module came from:
Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be
take to avoid inefficient code paths as we add new modules.
MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable. Profiling forces sequential execution even
when `--jobs` is greater than one.
MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable.
```bash
MVT_PROFILE=1 dev/mvt-ios check-backup test_backup
+13 -3
View File
@@ -145,7 +145,7 @@ class TombstoneCrashArtifact(AndroidArtifact):
# Add some extra metadata
tombstone_dict["timestamp"] = self._parse_timestamp_string(
tombstone_pb.timestamp
tombstone_pb.timestamp, file_timestamp
)
tombstone_dict["file_name"] = file_name
tombstone_dict["file_timestamp"] = convert_datetime_to_iso(file_timestamp)
@@ -288,11 +288,21 @@ class TombstoneCrashArtifact(AndroidArtifact):
def _load_timestamp_line(self, line: str, tombstone: dict) -> bool:
timestamp = line.split(":", 1)[1].strip()
tombstone["timestamp"] = self._parse_timestamp_string(timestamp)
tombstone["timestamp"] = self._parse_timestamp_string(timestamp, None)
return True
@staticmethod
def _parse_timestamp_string(timestamp: str) -> str:
def _parse_timestamp_string(
timestamp: str, fallback_timestamp: Optional[datetime.datetime]
) -> str:
"""Parse timestamp string, using fallback if timestamp is empty."""
# Handle empty or whitespace-only timestamps
if not timestamp or not timestamp.strip():
if fallback_timestamp:
return convert_datetime_to_iso(fallback_timestamp)
else:
raise ValueError("Empty timestamp with no fallback provided")
timestamp_parsed = parser.parse(timestamp)
return convert_datetime_to_iso(timestamp_parsed)
-21
View File
@@ -28,7 +28,6 @@ from mvt.common.help import (
HELP_MSG_DISABLE_UPDATE_CHECK,
HELP_MSG_HASHES,
HELP_MSG_IOC,
HELP_MSG_JOBS,
HELP_MSG_LIST_MODULES,
HELP_MSG_LOAD_MODULE,
HELP_MSG_MODULE,
@@ -162,9 +161,6 @@ def check_adb(ctx):
),
)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
@click.pass_context
def check_bugreport(
@@ -176,7 +172,6 @@ def check_bugreport(
load_module,
timezone,
verbose,
jobs,
bugreport_path,
):
from .cmd_check_bugreport import CmdAndroidCheckBugreport
@@ -199,7 +194,6 @@ def check_bugreport(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
@@ -242,9 +236,6 @@ def check_bugreport(
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
@click.pass_context
def check_backup(
@@ -256,7 +247,6 @@ def check_backup(
non_interactive,
backup_password,
verbose,
jobs,
backup_path,
):
from .cmd_check_backup import CmdAndroidCheckBackup
@@ -278,7 +268,6 @@ def check_backup(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
@@ -322,9 +311,6 @@ def check_backup(
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("ANDROIDQF_PATH", type=click.Path(exists=True))
@click.pass_context
def check_androidqf(
@@ -340,7 +326,6 @@ def check_androidqf(
non_interactive,
backup_password,
verbose,
jobs,
androidqf_path,
):
from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
@@ -364,7 +349,6 @@ def check_androidqf(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
@@ -414,9 +398,6 @@ def check_androidqf(
),
)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("LOGS_PATH", type=click.Path(exists=True))
@click.pass_context
def check_intrusion_logs(
@@ -428,7 +409,6 @@ def check_intrusion_logs(
load_module,
timezone,
verbose,
jobs,
logs_path,
):
from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
@@ -449,7 +429,6 @@ def check_intrusion_logs(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
-5
View File
@@ -53,7 +53,6 @@ class CmdAndroidCheckAndroidQF(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -69,7 +68,6 @@ class CmdAndroidCheckAndroidQF(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "android"
@@ -261,7 +259,6 @@ class CmdAndroidCheckAndroidQF(Command):
hashes=self.hashes,
sub_command=True,
custom_modules=self.custom_modules,
jobs=self.jobs,
)
cmd.from_zip(bugreport)
cmd.run()
@@ -293,7 +290,6 @@ class CmdAndroidCheckAndroidQF(Command):
hashes=self.hashes,
sub_command=True,
custom_modules=self.custom_modules,
jobs=self.jobs,
)
try:
cmd.from_ab(backup)
@@ -380,7 +376,6 @@ class CmdAndroidCheckAndroidQF(Command):
hashes=self.hashes,
sub_command=True,
custom_modules=self.custom_modules,
jobs=self.jobs,
)
cmd.run()
-2
View File
@@ -47,7 +47,6 @@ class CmdAndroidCheckBackup(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -63,7 +62,6 @@ class CmdAndroidCheckBackup(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "android"
-2
View File
@@ -40,7 +40,6 @@ class CmdAndroidCheckBugreport(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -56,7 +55,6 @@ class CmdAndroidCheckBugreport(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "android"
@@ -37,7 +37,6 @@ class CmdAndroidCheckIntrusionLogs(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -53,7 +52,6 @@ class CmdAndroidCheckIntrusionLogs(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "android"
@@ -8,6 +8,8 @@ import logging
import time
from typing import Optional
from rich.progress import track
from mvt.android.utils import (
BROWSER_INSTALLERS,
PLAY_STORE_INSTALLERS,
@@ -151,9 +153,10 @@ class AQFPackages(AndroidQFModule):
if total_hashes == 0:
return
self.log.info("Looking up %d package files on VirusTotal...", total_hashes)
for index, file_hash in enumerate(files_by_hash):
self.log.debug("VirusTotal lookup %d/%d", index + 1, total_hashes)
progress_desc = f"Looking up {total_hashes} package files on VirusTotal..."
for index, file_hash in enumerate(
track(files_by_hash, description=progress_desc)
):
try:
results = virustotal_lookup(file_hash)
except VTNoKey as exc:
+7 -8
View File
@@ -81,13 +81,12 @@ class AndroidQFModule(MVTModule):
return None
def _get_file_content(self, file_path):
with self.resource_lock:
if self.archive:
handle = self.archive.open(file_path)
else:
handle = open(os.path.join(self.parent_path, file_path), "rb")
if self.archive:
handle = self.archive.open(file_path)
else:
handle = open(os.path.join(self.parent_path, file_path), "rb")
data = handle.read()
handle.close()
data = handle.read()
handle.close()
return data
return data
+16 -17
View File
@@ -55,22 +55,21 @@ class BackupModule(MVTModule):
return fnmatch.filter(self.files, pattern)
def _get_file_content(self, file_path: str) -> bytes:
with self.resource_lock:
handle = None
if self.tar:
try:
member = self.tar.getmember(file_path)
handle = self.tar.extractfile(member)
if not handle:
raise ValueError(f"Could not extract file: {file_path}")
except KeyError:
raise FileNotFoundError(f"File not found in tar: {file_path}")
elif self.backup_path:
handle = open(os.path.join(self.backup_path, file_path), "rb")
else:
raise ValueError("No backup path or tar file provided")
handle = None
if self.tar:
try:
member = self.tar.getmember(file_path)
handle = self.tar.extractfile(member)
if not handle:
raise ValueError(f"Could not extract file: {file_path}")
except KeyError:
raise FileNotFoundError(f"File not found in tar: {file_path}")
elif self.backup_path:
handle = open(os.path.join(self.backup_path, file_path), "rb")
else:
raise ValueError("No backup path or tar file provided")
data = handle.read()
handle.close()
data = handle.read()
handle.close()
return data
return data
@@ -7,6 +7,7 @@ from mvt.common.module import MVTModule
from .dumpsys_accessibility import DumpsysAccessibility
from .dumpsys_activities import DumpsysActivities
from .dumpsys_adb_state import DumpsysADBState
from .dumpsys_appops import DumpsysAppops
from .dumpsys_battery_daily import DumpsysBatteryDaily
from .dumpsys_battery_history import DumpsysBatteryHistory
@@ -15,7 +16,6 @@ from .dumpsys_getprop import DumpsysGetProp
from .dumpsys_packages import DumpsysPackages
from .dumpsys_platform_compat import DumpsysPlatformCompat
from .dumpsys_receivers import DumpsysReceivers
from .dumpsys_adb_state import DumpsysADBState
from .fs_timestamps import BugReportTimestamps
from .tombstones import Tombstones
from .mounts import Mounts
+32 -164
View File
@@ -4,7 +4,6 @@
# https://github.com/mvt-project/mvt/blob/main/LICENSE
import datetime
import fnmatch
import io
import logging
import os
import re
@@ -13,7 +12,6 @@ from typing import List, Optional
from zipfile import ZipFile
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from mvt.android.artifacts.artifact import AndroidArtifact
from mvt.common.module import ModuleResults, MVTModule
# `------ 0.101s was the duration of 'SOME SECTION' ------`, printed when that
@@ -26,24 +24,6 @@ SECTION_DURATION = re.compile(r"^-{3,}\s*[0-9.]+s was the duration of", re.IGNOR
DUMPSTATE_ENTRY_POINTS = ("main_entry.txt", "dumpState_*.log", "*/dumpsys.txt")
_DUMPSTATE_BYTES_CACHE_KEY = ("android_bugreport", "dumpstate", "bytes")
_DUMPSTATE_SECTIONS_CACHE_KEY = ("android_bugreport", "dumpstate", "sections")
_SYSTEM_PROPERTIES_SECTION = b"------ SYSTEM PROPERTIES"
_DUMPSYS_SEPARATORS = {
b"DUMP OF SERVICE accessibility:",
b"DUMP OF SERVICE adb:",
b"DUMP OF SERVICE appops:",
b"DUMP OF SERVICE batterystats:",
b"DUMP OF SERVICE dbinfo:",
b"DUMP OF SERVICE package:",
b"DUMP OF SERVICE platform_compat:",
}
_DUMPSTATE_DELIMITER = (
b"------------------------------------------------------------------------------"
)
_GETPROP_LINE_RE = re.compile(rb"\[(.+?)\]: \[(.+?)\]")
class BugReportModule(MVTModule):
"""This class provides a base for all Android Bug Report modules."""
@@ -97,156 +77,45 @@ class BugReportModule(MVTModule):
return []
def _get_file_content(self, file_path: str) -> bytes:
with self.resource_lock:
if self.zip_archive:
handle = self.zip_archive.open(file_path)
else:
if not self.extract_path:
raise ValueError("extract_path is not set")
joined = os.path.join(self.extract_path, file_path)
if (
not Path(joined)
.resolve()
.is_relative_to(Path(self.extract_path).resolve())
):
raise ValueError("unsafe file_path")
handle = open(joined, "rb")
if self.zip_archive:
handle = self.zip_archive.open(file_path)
else:
if not self.extract_path:
raise ValueError("extract_path is not set")
joined = os.path.join(self.extract_path, file_path)
if (
not Path(joined)
.resolve()
.is_relative_to(Path(self.extract_path).resolve())
):
raise ValueError("unsafe file_path")
handle = open(joined, "rb")
data = handle.read()
handle.close()
data = handle.read()
handle.close()
return data
return data
def _get_dumpstate_file(self) -> Optional[bytes]:
with self.resource_lock:
if _DUMPSTATE_BYTES_CACHE_KEY in self.resource_cache:
return self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY]
main_entry, dumpstate_log, dumpsys_txt = DUMPSTATE_ENTRY_POINTS
content = None
main_entry, dumpstate_log, dumpsys_txt = DUMPSTATE_ENTRY_POINTS
main = self._get_files_by_pattern(main_entry)
if main:
main_content = self._get_file_content(main[0])
try:
content = self._get_file_content(main_content.decode().strip())
except KeyError:
pass
else:
dumpstate_logs = self._get_files_by_pattern(dumpstate_log)
if dumpstate_logs:
content = self._get_file_content(dumpstate_logs[0])
else:
dumpsys_files = self._get_files_by_pattern(dumpsys_txt)
if dumpsys_files:
content = self._get_file_content(dumpsys_files[0])
main = self._get_files_by_pattern(main_entry)
if main:
main_content = self._get_file_content(main[0])
try:
return self._get_file_content(main_content.decode().strip())
except KeyError:
return None
self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY] = content
return content
dumpstate_logs = self._get_files_by_pattern(dumpstate_log)
if dumpstate_logs:
return self._get_file_content(dumpstate_logs[0])
def _get_dumpstate_sections(self) -> dict[bytes, bytes]:
with self.resource_lock:
if _DUMPSTATE_SECTIONS_CACHE_KEY in self.resource_cache:
return self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY]
dumpsys_files = self._get_files_by_pattern(dumpsys_txt)
if dumpsys_files:
return self._get_file_content(dumpsys_files[0])
section_lines: dict[bytes, list[bytes]] = {
separator: [] for separator in _DUMPSYS_SEPARATORS
}
section_lines[_SYSTEM_PROPERTIES_SECTION] = []
current_section = None
completed_sections = set()
in_system_properties = False
content = self._get_dumpstate_file()
if content:
for raw_line in io.BytesIO(content):
line = raw_line.rstrip(b"\r\n")
stripped = line.strip()
if stripped.startswith(_SYSTEM_PROPERTIES_SECTION):
in_system_properties = True
elif in_system_properties:
if stripped == b"------":
in_system_properties = False
elif _GETPROP_LINE_RE.search(line):
section_lines[_SYSTEM_PROPERTIES_SECTION].append(line)
if stripped in _DUMPSYS_SEPARATORS:
if current_section is not None:
completed_sections.add(current_section)
current_section = (
None if stripped in completed_sections else stripped
)
continue
if stripped.startswith(_DUMPSTATE_DELIMITER):
if current_section is not None:
completed_sections.add(current_section)
current_section = None
continue
if current_section is not None:
section_lines[current_section].append(line)
sections = {
separator: b"\n".join(lines)
for separator, lines in section_lines.items()
}
self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY] = sections
return sections
def _get_dumpsys_section_bytes(self, separator: bytes) -> Optional[bytes]:
sections = self._get_dumpstate_sections()
if separator in sections:
return sections[separator]
content = self._get_dumpstate_file()
if not content:
return None
return AndroidArtifact.extract_dumpsys_section(content, separator, binary=True)
def _get_dumpsys_section(
self, separator: str, errors: str = "replace"
) -> Optional[str]:
cache_key = (
"android_bugreport",
"dumpstate",
"section",
separator,
errors,
)
with self.resource_lock:
if cache_key in self.resource_cache:
return self.resource_cache[cache_key]
section_bytes = self._get_dumpsys_section_bytes(separator.encode("utf-8"))
section = (
section_bytes.decode("utf-8", errors=errors)
if section_bytes is not None
else None
)
self.resource_cache[cache_key] = section
return section
def _get_system_properties_text(self, errors: str = "ignore") -> Optional[str]:
cache_key = (
"android_bugreport",
"dumpstate",
"system_properties",
errors,
)
with self.resource_lock:
if cache_key in self.resource_cache:
return self.resource_cache[cache_key]
sections = self._get_dumpstate_sections()
section_bytes = sections.get(_SYSTEM_PROPERTIES_SECTION)
section = (
section_bytes.decode("utf-8", errors=errors)
if section_bytes is not None
else None
)
self.resource_cache[cache_key] = section
return section
return None
@staticmethod
def extract_command_section(content: str, heading: str) -> str:
@@ -295,8 +164,7 @@ class BugReportModule(MVTModule):
instant returned in UTC.
"""
if self.zip_archive:
with self.resource_lock:
file_timetuple = self.zip_archive.getinfo(file_path).date_time
file_timetuple = self.zip_archive.getinfo(file_path).date_time
return datetime.datetime(*file_timetuple, tzinfo=self._device_timezone())
if not self.extract_path:
raise ValueError("extract_path is not set")
@@ -34,16 +34,18 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
)
def run(self) -> None:
content = self._get_dumpsys_section(
"DUMP OF SERVICE accessibility:", errors="ignore"
)
if content is None:
full_dumpsys = self._get_dumpstate_file()
if not full_dumpsys:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
content = self.extract_dumpsys_section(
full_dumpsys.decode("utf-8", errors="ignore"),
"DUMP OF SERVICE accessibility:",
)
self.parse(content)
listed = unnamed = 0
@@ -38,14 +38,19 @@ class DumpsysActivities(DumpsysPackageActivitiesArtifact, BugReportModule):
self.results = results if results else []
def run(self) -> None:
section = self._get_dumpsys_section("DUMP OF SERVICE package:", errors="ignore")
if section is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
# Extract package section
section = self.extract_dumpsys_section(
content.decode("utf-8", errors="ignore"), "DUMP OF SERVICE package:"
)
# Parse
self.parse(section)
@@ -38,14 +38,19 @@ class DumpsysADBState(DumpsysADBArtifact, BugReportModule):
)
def run(self) -> None:
content = self._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:")
if content is None:
full_dumpsys = self._get_dumpstate_file()
if not full_dumpsys:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
content = self.extract_dumpsys_section(
full_dumpsys,
b"DUMP OF SERVICE adb:",
binary=True,
)
self.parse(content)
if self.results:
self.log.info(
@@ -34,14 +34,17 @@ class DumpsysAppops(DumpsysAppopsArtifact, BugReportModule):
)
def run(self) -> None:
section = self._get_dumpsys_section("DUMP OF SERVICE appops:")
if section is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
section = self.extract_dumpsys_section(
content.decode("utf-8", errors="replace"), "DUMP OF SERVICE appops:"
)
self.parse(section)
self.log.info(
@@ -34,14 +34,17 @@ class DumpsysBatteryDaily(DumpsysBatteryDailyArtifact, BugReportModule):
)
def run(self) -> None:
dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:")
if dumpsys_section is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
dumpsys_section = self.extract_dumpsys_section(
content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:"
)
self.parse(dumpsys_section)
self.log.info("Extracted a total of %d battery daily stats", len(self.results))
@@ -34,14 +34,17 @@ class DumpsysBatteryHistory(DumpsysBatteryHistoryArtifact, BugReportModule):
)
def run(self) -> None:
dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:")
if dumpsys_section is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
dumpsys_section = self.extract_dumpsys_section(
content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:"
)
self.parse(dumpsys_section)
self.log.info(
@@ -36,14 +36,18 @@ class DumpsysDBInfo(DumpsysDBInfoArtifact, BugReportModule):
)
def run(self) -> None:
section = self._get_dumpsys_section("DUMP OF SERVICE dbinfo:", errors="ignore")
if section is None:
data = self._get_dumpstate_file()
if not data:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
section = self.extract_dumpsys_section(
data.decode("utf-8", errors="ignore"), "DUMP OF SERVICE dbinfo:"
)
self.parse(section)
self.log.info(
"Extracted a total of %d database connection pool records",
@@ -36,13 +36,16 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule):
self.results = [] if not results else results
def run(self) -> None:
content = self._get_system_properties_text()
if content is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
self.parse(content)
section = self.extract_command_section(
content.decode(errors="ignore"), "------ SYSTEM PROPERTIES"
)
self.parse(section)
self.log.info("Extracted %d Android system properties", len(self.results))
@@ -35,14 +35,17 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule):
)
def run(self) -> None:
content = self._get_dumpsys_section("DUMP OF SERVICE package:")
if content is None:
data = self._get_dumpstate_file()
if not data:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
content = self.extract_dumpsys_section(
data.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:"
)
self.parse(content)
for result in self.results:
@@ -34,14 +34,18 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
)
def run(self) -> None:
content = self._get_dumpsys_section("DUMP OF SERVICE platform_compat:")
if content is None:
data = self._get_dumpstate_file()
if not data:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
decoded_data = data.decode("utf-8", errors="replace")
content = self.extract_dumpsys_section(
decoded_data, "DUMP OF SERVICE platform_compat:"
)
self.parse(content)
self.log.info("Found %d package compatibility overrides", len(self.results))
@@ -36,13 +36,17 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule):
self.results = results if results else []
def run(self) -> None:
dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE package:")
if dumpsys_section is None:
content = self._get_dumpstate_file()
if not content:
self.log.error(
"Unable to find dumpstate file. "
"Did you provide a valid bug report archive?"
)
return
dumpsys_section = self.extract_dumpsys_section(
content.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:"
)
self.parse(dumpsys_section)
self.log.info("Extracted %d package receivers", len(self.results))
@@ -68,6 +68,11 @@ class Tombstones(TombstoneCrashArtifact, BugReportModule):
}
try:
data = self._get_file_content(file_path)
if not data:
self.log.debug("Skipping empty tombstone file %s (0 bytes)", file_path)
source_info["error"] = "empty tombstone"
parsed_sources[source] = source_info
continue
if source == "protobuf":
record = self.parse_protobuf_record(
file_name, file_timestamp, data
+41 -226
View File
@@ -6,10 +6,7 @@
import json
import logging
import os
import queue
import sys
import threading
from concurrent.futures import Future, ThreadPoolExecutor
from datetime import datetime
from functools import cached_property
from heapq import heappop, heappush
@@ -22,7 +19,6 @@ from rich.text import Text
from .alerts import AlertLevel, AlertStore
from .config import settings
from .indicators import Indicators
from .log import MVTLogHandler, finish_module_log_buffer, start_module_log_buffer
from .module import EncryptedBackupError, MVTModule, run_module, save_timeline
from .module_loader import (
ModuleOrigin,
@@ -56,7 +52,6 @@ class Command:
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
self.name = ""
self.platform = ""
@@ -72,11 +67,6 @@ class Command:
self.sub_command = sub_command
self.disable_version_check = disable_version_check
self.disable_indicator_check = disable_indicator_check
if jobs < 1:
raise ValueError("jobs must be at least 1")
self.jobs = jobs
self._resource_lock = threading.RLock()
self._resource_cache: dict[object, Any] = {}
# This dictionary can contain options that will be passed down from
# the Command to all modules. This can for example be used to pass
@@ -720,22 +710,41 @@ class Command:
return ordered
def _run_module(
self,
module: type[MVTModule],
executed_by_type: dict[type[MVTModule], MVTModule],
capture_logs: bool,
) -> tuple[MVTModule, list[logging.LogRecord], bool]:
"""Initialize and run one module, optionally buffering console logs."""
token = start_module_log_buffer() if capture_logs else None
records: list[logging.LogRecord] = []
encrypted = False
def run(self) -> None:
# The output folder and its command.log exist for a run, so that
# listing modules or rejecting a target leaves nothing behind.
# Resolving the module list can warn, so the log comes first.
self._create_storage()
self._setup_logging()
ordered_modules = self._ordered_modules()
if ordered_modules is None:
return
# Read the indicators once the run is certain to happen, before
# init() does any work on the target, so that a missing indicators
# file is reported first and every module gets the same object.
iocs = self.iocs
# Commands announce their target from init(), so it goes before the
# module list.
try:
self.init()
except NotImplementedError:
pass
self._log_loaded_modules(ordered_modules)
executed_by_type: dict[type[MVTModule], MVTModule] = {}
for module in ordered_modules:
module_logger = get_module_logger(module)
m = module(
target_path=self.target_path,
results_path=self.results_path,
module_options=self.module_options,
log=get_module_logger(module),
log=module_logger,
)
# Dependencies are keyed by the module class they declare, even
# when it was replaced, so that a module asking for the results
@@ -744,12 +753,10 @@ class Command:
dependency: executed_by_type[resolved]
for dependency, resolved in self._module_dependencies(module)
}
m.resource_lock = self._resource_lock
m.resource_cache = self._resource_cache
if self.iocs.total_ioc_count:
# IOC collections are shared read-only during module execution.
m.indicators = self.iocs
if iocs.total_ioc_count:
m.indicators = iocs
m.indicators.log = m.log
if self.serial:
m.serial = self.serial
@@ -762,215 +769,23 @@ class Command:
try:
run_module(m)
except EncryptedBackupError:
encrypted = True
finally:
if token is not None:
records = finish_module_log_buffer(token)
return m, records, encrypted
@staticmethod
def _console_handler() -> Optional[MVTLogHandler]:
for handler in reversed(logging.getLogger("mvt").handlers):
if isinstance(handler, MVTLogHandler):
return handler
return None
def _publish_completed_modules(
self,
ordered_modules: list[type[MVTModule]],
completed: dict[type[MVTModule], MVTModule],
) -> None:
"""Expose a stable snapshot of completed modules to initialization hooks."""
self.executed = [
completed[module] for module in ordered_modules if module in completed
]
def _aggregate_modules(
self,
ordered_modules: list[type[MVTModule]],
completed: dict[type[MVTModule], MVTModule],
) -> None:
"""Aggregate results in stable topological order."""
for module in ordered_modules:
if module not in completed:
continue
instance = completed[module]
self.timeline.extend(instance.timeline)
self.url_results.extend(instance.url_results)
self.alertstore.extend(instance.alertstore.alerts)
def _run_sequential(
self, ordered_modules: list[type[MVTModule]]
) -> tuple[dict[type[MVTModule], MVTModule], bool]:
completed: dict[type[MVTModule], MVTModule] = {}
for module in ordered_modules:
instance, _, encrypted = self._run_module(module, completed, False)
if encrypted:
return completed, True
completed[module] = instance
self._publish_completed_modules(ordered_modules, completed)
return completed, False
def _run_parallel(
self, ordered_modules: list[type[MVTModule]], jobs: int
) -> tuple[dict[type[MVTModule], MVTModule], bool]:
"""Run a stable module DAG with bounded worker threads."""
completed: dict[type[MVTModule], MVTModule] = {}
scheduled: set[type[MVTModule]] = set()
running: dict[Future, type[MVTModule]] = {}
completion_queue: queue.Queue[Future] = queue.Queue()
handler = self._console_handler()
status = None
fatal = False
def ready_modules() -> list[type[MVTModule]]:
return [
module
for module in ordered_modules
if module not in scheduled
and all(
dependency in completed
for _, dependency in self._module_dependencies(module)
self.log.critical(
"The backup appears to be encrypted. "
"Please decrypt it first using `mvt-ios decrypt-backup`."
)
]
def update_status() -> None:
if status is None:
return
names = ", ".join(module.__name__ for module in running.values())
status.update(
f"Modules: {len(completed)}/{len(ordered_modules)} complete"
+ (f"; running {names}" if names else "")
)
if handler and handler.console.is_terminal:
status = handler.console.status("")
status.start()
try:
with ThreadPoolExecutor(max_workers=jobs) as executor:
while len(completed) < len(ordered_modules):
ready = ready_modules()
# An unsafe module is a scheduler barrier: it starts only
# after active workers drain and blocks later ready work.
if ready and not ready[0].parallel_safe:
if running:
future = completion_queue.get()
module = running.pop(future)
instance, records, encrypted = future.result()
if handler:
handler.emit_module_records(module.__name__, records)
if encrypted:
fatal = True
break
completed[module] = instance
self._publish_completed_modules(ordered_modules, completed)
update_status()
continue
module = ready[0]
scheduled.add(module)
if status is not None:
status.update(
f"Modules: {len(completed)}/{len(ordered_modules)} "
f"complete; running {module.__name__}"
)
instance, records, encrypted = self._run_module(
module, completed, True
)
if handler:
handler.emit_module_records(module.__name__, records)
if encrypted:
fatal = True
break
completed[module] = instance
self._publish_completed_modules(ordered_modules, completed)
update_status()
continue
for module in ready:
if len(running) >= jobs or not module.parallel_safe:
break
scheduled.add(module)
future = executor.submit(
self._run_module, module, dict(completed), True
)
running[future] = module
future.add_done_callback(completion_queue.put)
update_status()
if not running:
break
future = completion_queue.get()
module = running.pop(future)
instance, records, encrypted = future.result()
if handler:
handler.emit_module_records(module.__name__, records)
if encrypted:
fatal = True
break
completed[module] = instance
self._publish_completed_modules(ordered_modules, completed)
update_status()
if fatal:
for future in running:
future.cancel()
# The executor context waits for already-active work. Its
# results are intentionally discarded after the fatal error.
finally:
if status is not None:
status.stop()
return completed, fatal
def run(self) -> None:
self._create_storage()
self._setup_logging()
ordered_modules = self._ordered_modules()
if ordered_modules is None:
return
# Load shared indicators before touching the acquisition.
self.iocs
try:
self.init()
except NotImplementedError:
pass
self._log_loaded_modules(ordered_modules)
jobs = self.jobs
if settings.PROFILE and jobs > 1:
self.log.warning(
"MVT_PROFILE is enabled; forcing sequential module execution."
)
jobs = 1
if jobs == 1:
completed, fatal = self._run_sequential(ordered_modules)
else:
completed, fatal = self._run_parallel(ordered_modules, jobs)
self._aggregate_modules(ordered_modules, completed)
self.executed.append(m)
executed_by_type[module] = m
self.timeline.extend(m.timeline)
self.url_results.extend(m.url_results)
self.alertstore.extend(m.alertstore.alerts)
try:
self.finish()
except NotImplementedError:
pass
if fatal:
self.log.critical(
"The backup appears to be encrypted. "
"Please decrypt it first using `mvt-ios decrypt-backup`."
)
return
# We only store the timeline from the parent/main command
if self.sub_command:
return
-1
View File
@@ -17,7 +17,6 @@ HELP_MSG_LOAD_MODULE = (
HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing"
HELP_MSG_HASHES = "Generate hashes of all the files analyzed"
HELP_MSG_VERBOSE = "Verbose mode"
HELP_MSG_JOBS = "Number of extraction modules to run concurrently"
HELP_MSG_VERBOSE_COMMAND = (
"Verbose mode (kept for compatibility, pass --verbose before the command "
"name instead)"
+1 -41
View File
@@ -4,8 +4,6 @@
# https://license.mvt.re/1.1/
import logging
import threading
from contextvars import ContextVar, Token
from rich.console import Console
from rich.logging import RichHandler
from typing import Optional
@@ -29,33 +27,16 @@ logging.addLevelName(HIGH_ALERT, "HIGH")
logging.addLevelName(CRITICAL_ALERT, "CRITICAL")
_module_log_buffer: ContextVar[Optional[list[logging.LogRecord]]] = ContextVar(
"mvt_module_log_buffer", default=None
)
def start_module_log_buffer() -> Token:
"""Capture console records in the current module execution context."""
return _module_log_buffer.set([])
def finish_module_log_buffer(token: Token) -> list[logging.LogRecord]:
records = _module_log_buffer.get() or []
_module_log_buffer.reset(token)
return records
class MVTLogHandler(RichHandler):
def __init__(self, console: Optional[Console] = None, level: int = logging.DEBUG):
super().__init__(console=console, level=level)
self._console_lock = threading.RLock()
def __add_prefix_space(self, level: str) -> str:
max_length = len("CRITICAL ALERT")
space = max_length - len(level)
return f"{level}{' ' * space}"
def _emit_record(self, record: logging.LogRecord) -> None:
def emit(self, record: logging.LogRecord):
try:
msg = rf"[grey50]\[{record.name}][/] {self.format(record)}"
@@ -82,24 +63,3 @@ class MVTLogHandler(RichHandler):
except Exception:
self.handleError(record)
def emit(self, record: logging.LogRecord):
buffer = _module_log_buffer.get()
if buffer is not None:
# More than one CLI can be imported in-process (notably in tests),
# which can install multiple console handlers for the same record.
if not buffer or buffer[-1] is not record:
buffer.append(record)
return
with self._console_lock:
self._emit_record(record)
def emit_module_records(
self, module_name: str, records: list[logging.LogRecord]
) -> None:
"""Print one module's captured records without interleaving."""
with self._console_lock:
self.console.print(f"[bold cyan]--- {module_name} ---[/bold cyan]")
for record in records:
self._emit_record(record)
+8 -19
View File
@@ -8,7 +8,6 @@ import json
import logging
import os
import re
import threading
from dataclasses import asdict, is_dataclass
from typing import Any, Dict, Optional, Sequence
@@ -47,7 +46,6 @@ class MVTModule:
slug: Optional[str] = None
dependencies: Sequence[type["MVTModule"]] = ()
supported_commands: Sequence[tuple[str, str]] = ()
parallel_safe: bool = True
# A custom module can name a module class it supersedes, usually a
# built-in one. When both are available to a command, the named module is
# dropped from the run and this module takes its place, including in the
@@ -92,12 +90,6 @@ class MVTModule:
self.timeline: ModuleTimeline = []
self.url_results: list[URLResult] = []
self.dependency_modules: Dict[type["MVTModule"], "MVTModule"] = {}
# Commands replace this with a command-scoped lock for modules sharing
# archive handles or other non-thread-safe read resources.
self.resource_lock = threading.RLock()
# Commands also replace this with a command-scoped cache. Cached values
# must be immutable, because parallel modules can read them concurrently.
self.resource_cache: Dict[object, Any] = {}
def get_dependency_results(
self, module_class: type["MVTModule"]
@@ -193,20 +185,17 @@ class MVTModule:
"""
timeline_set = set()
deduplicated = []
for record in timeline:
serialized = json.dumps(
asdict(record)
if is_dataclass(record) and not isinstance(record, type)
else record,
sort_keys=True,
timeline_set.add(
json.dumps(
asdict(record)
if is_dataclass(record) and not isinstance(record, type)
else record,
sort_keys=True,
)
)
if serialized in timeline_set:
continue
timeline_set.add(serialized)
deduplicated.append(json.loads(serialized))
return deduplicated
return [json.loads(record) for record in timeline_set]
def to_timeline(self) -> None:
"""Convert results into a timeline."""
-11
View File
@@ -36,7 +36,6 @@ from mvt.common.help import (
HELP_MSG_LOAD_MODULE,
HELP_MSG_MODULE,
HELP_MSG_VERBOSE,
HELP_MSG_JOBS,
HELP_MSG_VERBOSE_COMMAND,
HELP_MSG_CHECK_FS,
HELP_MSG_CHECK_IOCS,
@@ -280,9 +279,6 @@ def extract_key(password, key_file, backup_path):
)
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
@click.pass_context
def check_backup(
@@ -295,7 +291,6 @@ def check_backup(
load_module,
hashes,
verbose,
jobs,
backup_path,
):
from .cmd_check_backup import CmdIOSCheckBackup
@@ -314,7 +309,6 @@ def check_backup(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
@@ -354,9 +348,6 @@ def check_backup(
)
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
@click.option(
"--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
)
@click.argument("DUMP_PATH", type=click.Path(exists=True))
@click.pass_context
def check_fs(
@@ -369,7 +360,6 @@ def check_fs(
load_module,
hashes,
verbose,
jobs,
dump_path,
):
from .cmd_check_fs import CmdIOSCheckFS
@@ -388,7 +378,6 @@ def check_fs(
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
jobs=jobs,
)
if list_modules:
-2
View File
@@ -38,7 +38,6 @@ class CmdIOSCheckBackup(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -54,7 +53,6 @@ class CmdIOSCheckBackup(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "ios"
-2
View File
@@ -31,7 +31,6 @@ class CmdIOSCheckFS(Command):
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
jobs: int = 4,
) -> None:
super().__init__(
target_path=target_path,
@@ -46,7 +45,6 @@ class CmdIOSCheckFS(Command):
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
jobs=jobs,
)
self.platform = "ios"
+81 -1
View File
@@ -2,8 +2,8 @@
# Copyright (c) 2021-2023 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import datetime
import os
import pytest
@@ -130,3 +130,83 @@ class TestTombstoneCrashArtifact:
# Tombstones include an explicit offset, so normalize them to UTC.
assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.518290"
def test_tombstone_pb_empty_timestamp(self):
"""Test parsing a protobuf tombstone with an empty timestamp."""
tombstone_artifact = TombstoneCrashArtifact()
artifact_path = "android_data/bugreport_tombstones/tombstone_empty_timestamp.pb"
file = get_artifact(artifact_path)
with open(file, "rb") as f:
data = f.read()
file_name = os.path.basename(artifact_path)
file_timestamp = datetime.datetime(2024, 1, 15, 10, 30, 45, 123456)
tombstone_artifact.parse_protobuf(file_name, file_timestamp, data)
assert len(tombstone_artifact.results) == 1
result = tombstone_artifact.results[0]
# When tombstone has empty timestamp, should use file modification time
assert result.get("timestamp") == "2024-01-15 10:30:45.123456"
assert result.get("pid") == 12345
assert result.get("uid") == 1000
assert result.get("signal_info", {}).get("name") == "SIGSEGV"
def test_tombstone_pb_empty_timestamp_with_threads(self):
"""Test parsing a protobuf tombstone with empty timestamp and thread info."""
tombstone_artifact = TombstoneCrashArtifact()
artifact_path = "android_data/bugreport_tombstones/tombstone_empty_timestamp_with_threads.pb"
file = get_artifact(artifact_path)
with open(file, "rb") as f:
data = f.read()
file_name = os.path.basename(artifact_path)
file_timestamp = datetime.datetime(2024, 2, 20, 14, 15, 30, 0)
tombstone_artifact.parse_protobuf(file_name, file_timestamp, data)
assert len(tombstone_artifact.results) == 1
result = tombstone_artifact.results[0]
# Verify timestamp fallback
assert result.get("timestamp") == "2024-02-20 14:15:30.000000"
assert result.get("pid") == 9876
assert result.get("uid") == 10001
assert result.get("signal_info", {}).get("name") == "SIGABRT"
assert result.get("process_name") == "ExampleThread"
def test_tombstone_pb_whitespace_timestamp(self):
"""Test parsing a protobuf tombstone with whitespace-only timestamp."""
tombstone_artifact = TombstoneCrashArtifact()
artifact_path = (
"android_data/bugreport_tombstones/tombstone_whitespace_timestamp.pb"
)
file = get_artifact(artifact_path)
with open(file, "rb") as f:
data = f.read()
file_name = os.path.basename(artifact_path)
file_timestamp = datetime.datetime(2024, 3, 10, 8, 0, 0, 0)
tombstone_artifact.parse_protobuf(file_name, file_timestamp, data)
assert len(tombstone_artifact.results) == 1
result = tombstone_artifact.results[0]
# Verify whitespace timestamp is treated as empty
assert result.get("timestamp") == "2024-03-10 08:00:00.000000"
assert result.get("pid") == 11111
assert result.get("uid") == 2000
assert result.get("signal_info", {}).get("name") == "SIGILL"
def test_tombstone_pb_empty_file(self):
"""Test that empty (0 bytes) tombstone files are handled gracefully."""
artifact_path = "android_data/bugreport_tombstones/tombstone_empty_file.pb"
file = get_artifact(artifact_path)
with open(file, "rb") as f:
data = f.read()
# Verify the file is actually empty
assert len(data) == 0, "Test file should be empty (0 bytes)"
# Empty files should be skipped in the module (not parsed)
# The actual skipping happens in the Tombstones module's run() method
# This test verifies that empty data is detectable
-55
View File
@@ -4,13 +4,8 @@
# https://license.mvt.re/1.1/
import os
import threading
from concurrent.futures import ThreadPoolExecutor
from io import BytesIO
from pathlib import Path
from unittest.mock import MagicMock
from mvt.android.modules.bugreport.base import BugReportModule
from mvt.android.modules.bugreport.dumpsys_appops import DumpsysAppops
from mvt.android.modules.bugreport.dumpsys_getprop import DumpsysGetProp
from mvt.android.modules.bugreport.dumpsys_packages import DumpsysPackages
@@ -123,53 +118,3 @@ class TestBugreportAnalysis:
assert len(m.results) == 2
assert m.results[1]["pid"] == 3559
assert m.results[0]["sources"]["text"]["parsed"] is True
def test_dumpstate_section_is_cached_across_parallel_modules(self):
dumpstate = (
b"header\n"
b"------ SYSTEM PROPERTIES (getprop) ------\n"
b"[persist.sys.timezone]: [Europe/Berlin]\n"
b"------------------------------------------------------------------------------\n"
b"------\n"
b"DUMP OF SERVICE package:\n"
b"package data\n"
b"------------------------------------------------------------------------------\n"
b"DUMP OF SERVICE package:\n"
b"duplicate package data\n"
b"------------------------------------------------------------------------------\n"
b"DUMP OF SERVICE adb:\n"
b"adb data\n"
b"------------------------------------------------------------------------------\n"
)
archive = MagicMock()
archive.open.side_effect = lambda _: BytesIO(dumpstate)
shared_lock = threading.RLock()
shared_cache = {}
modules = []
for _ in range(4):
module = BugReportModule()
module.from_zip(archive, ["dumpState_test.log"])
module.resource_lock = shared_lock
module.resource_cache = shared_cache
modules.append(module)
with ThreadPoolExecutor(max_workers=4) as executor:
sections = list(
executor.map(
lambda module: module._get_dumpsys_section(
"DUMP OF SERVICE package:"
),
modules,
)
)
assert sections == ["package data"] * 4
assert all(section is sections[0] for section in sections)
assert modules[0]._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:") == (
b"adb data"
)
assert modules[0]._get_system_properties_text() == (
"[persist.sys.timezone]: [Europe/Berlin]"
)
archive.open.assert_called_once_with("dumpState_test.log")
Whitespace-only changes.
@@ -0,0 +1 @@
5test/build/fingerprint:12/TEST/V1.0:user/release-keys test-rev-001(¹`0º`8èBu:r:system_app:s0J/system/bin/test_process <R SIGSEGV" SEGV_MAPERR
@@ -0,0 +1 @@
5test/build/fingerprint:13/TEST/V2.0:user/release-keys test-rev-002(”M0•M8‘NBu:r:untrusted_app:s0J-/data/app/com.example.app/lib/arm64/libapp.so xRSIGABRT"SI_TKILL‚•M•M ExampleThread
@@ -0,0 +1,2 @@
5test/build/fingerprint:11/TEST/V3.0:user/release-keys test-rev-003" (çV0èV8ÐB u:r:shell:s0J/system/bin/app_process64 RSIGILL"
ILL_ILLOPN
+19 -304
View File
@@ -5,22 +5,14 @@
import json
import logging
import threading
import time
from io import StringIO
import pytest
from rich.console import Console
from mvt.common.command import Command
from mvt.common.config import settings
from mvt.common.log import MVTLogHandler
from mvt.common.module import EncryptedBackupError, MVTModule
from unittest.mock import patch
import pytest
from click.testing import CliRunner
from mvt.common.command import Command
from mvt.common.indicators import Indicators
from mvt.common.module import MVTModule
class RecordingModule(MVTModule):
@@ -306,15 +298,6 @@ class TestCommand:
alerts = json.loads((tmp_path / "alerts.json").read_text())
assert alerts[0]["event"]["payload"] == "\\xa8\\xa9"
def test_timeline_deduplication_preserves_first_seen_order(self):
timeline = [
{"timestamp": "same", "event": "first"},
{"timestamp": "same", "event": "second"},
{"timestamp": "same", "event": "first"},
]
assert MVTModule._deduplicate_timeline(timeline) == timeline[:2]
def test_stores_collected_urls(self, tmp_path):
cmd = RecordingCommand(results_path=str(tmp_path))
cmd.modules = [URLRecordingModule]
@@ -331,7 +314,7 @@ class TestCommand:
]
def test_modules_run_in_stable_topological_order(self):
cmd = RecordingCommand(jobs=1)
cmd = RecordingCommand()
cmd.modules = [ThirdModule, IndependentModule, SecondModule, FirstModule]
cmd.run()
@@ -346,7 +329,7 @@ class TestCommand:
assert second.results == ["first", "second"]
def test_selected_module_runs_transitive_dependencies(self):
cmd = RecordingCommand(module_name="ThirdModule", jobs=1)
cmd = RecordingCommand(module_name="ThirdModule")
cmd.modules = [ThirdModule, SecondModule, FirstModule, IndependentModule]
cmd.run()
@@ -357,38 +340,6 @@ class TestCommand:
"ThirdModule",
]
@pytest.mark.parametrize("jobs", [1, 2])
@pytest.mark.parametrize("parallel_safe", [False, True])
def test_initialization_hook_sees_completed_dependencies(self, jobs, parallel_safe):
class Prerequisite(FirstModule):
pass
Prerequisite.parallel_safe = parallel_safe
class Dependent(SecondModule):
dependencies = (Prerequisite,)
def run(self):
self.results = self.get_dependency_results(Prerequisite) + ["second"]
class HookCommand(RecordingCommand):
def module_init(self, module):
if isinstance(module, Dependent):
self.seen = [
executed.results
for executed in self.executed
if isinstance(executed, Prerequisite)
]
cmd = HookCommand(jobs=jobs)
cmd.modules = [Dependent, Prerequisite]
cmd.run()
assert cmd.seen == [["first"]]
assert [type(module) for module in cmd.executed] == [Prerequisite, Dependent]
assert cmd.executed[1].results == ["first", "second"]
def test_circular_dependency_warns_and_stops(self, caplog):
class CircularOne(RecordingModule):
pass
@@ -421,10 +372,7 @@ class TestCommand:
with caplog.at_level(logging.WARNING):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
"IndependentModule",
"FirstModule",
]
assert RecordingModule.run_order == ["IndependentModule", "FirstModule"]
assert cmd.initialized
assert "Module DependentModule will be SKIPPED" in caplog.text
assert "depends on module UnavailableModule" in caplog.text
@@ -528,7 +476,7 @@ class TestCommand:
]
def test_selected_custom_module_runs(self):
cmd = RecordingCommand(module_name="CustomIOSBackupModule", jobs=1)
cmd = RecordingCommand(module_name="CustomIOSBackupModule")
cmd.platform = "ios"
cmd.name = "check-backup"
cmd.custom_modules = [CustomIOSBackupModule]
@@ -548,7 +496,7 @@ class TestCommand:
assert RecordingModule.run_order == []
def test_custom_module_dependencies_use_topological_order(self):
cmd = RecordingCommand(module_name="CustomDependsOnBuiltin", jobs=1)
cmd = RecordingCommand(module_name="CustomDependsOnBuiltin")
cmd.platform = "ios"
cmd.name = "check-backup"
cmd.modules = [SecondModule, FirstModule]
@@ -558,228 +506,6 @@ class TestCommand:
assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"]
def test_parallel_modules_overlap_and_respect_worker_limit(self):
lock = threading.Lock()
barrier = threading.Barrier(2)
active = 0
maximum = 0
overlapped = []
class ParallelModule(RecordingModule):
def run(self):
nonlocal active, maximum
with lock:
active += 1
maximum = max(maximum, active)
try:
barrier.wait(timeout=2)
overlapped.append(self.__class__.__name__)
time.sleep(0.02)
finally:
with lock:
active -= 1
class ParallelOne(ParallelModule):
pass
class ParallelTwo(ParallelModule):
pass
class ParallelThree(ParallelModule):
pass
cmd = RecordingCommand(jobs=2)
cmd.modules = [ParallelOne, ParallelTwo, ParallelThree]
cmd.run()
assert maximum == 2
assert {"ParallelOne", "ParallelTwo"}.issubset(overlapped)
def test_dependencies_wait_and_transitive_dependencies_run_once(self):
prerequisite_finished = threading.Event()
runs = []
class Prerequisite(RecordingModule):
def run(self):
runs.append("prerequisite")
self.results = ["ready"]
prerequisite_finished.set()
class Dependent(RecordingModule):
dependencies = (Prerequisite,)
def run(self):
assert prerequisite_finished.is_set()
assert self.get_dependency_results(Prerequisite) == ["ready"]
runs.append(self.__class__.__name__)
class DependentOne(Dependent):
pass
class DependentTwo(Dependent):
pass
cmd = RecordingCommand(jobs=3)
cmd.modules = [DependentOne, DependentTwo, Prerequisite]
cmd.run()
assert runs.count("prerequisite") == 1
assert set(runs[1:]) == {"DependentOne", "DependentTwo"}
def test_parallel_unsafe_module_runs_exclusively(self):
lock = threading.Lock()
active = 0
unsafe_ran_exclusively = []
class SafeOne(RecordingModule):
def run(self):
nonlocal active
with lock:
active += 1
time.sleep(0.03)
with lock:
active -= 1
class Unsafe(RecordingModule):
parallel_safe = False
def run(self):
nonlocal active
with lock:
unsafe_ran_exclusively.append(active == 0)
active += 1
time.sleep(0.01)
with lock:
active -= 1
class SafeTwo(SafeOne):
pass
cmd = RecordingCommand(jobs=3)
cmd.modules = [SafeOne, Unsafe, SafeTwo]
cmd.run()
assert unsafe_ran_exclusively == [True]
def test_results_are_aggregated_in_topological_order(self):
release_first = threading.Event()
class SlowFirst(RecordingModule):
def run(self):
release_first.wait(timeout=2)
self.results = ["first"]
self.timeline = [{"module": "first"}]
self.alertstore.info("first", "", {"order": 1})
class FastSecond(RecordingModule):
def run(self):
self.results = ["second"]
self.timeline = [{"module": "second"}]
self.alertstore.info("second", "", {"order": 2})
release_first.set()
cmd = RecordingCommand(jobs=2)
cmd.modules = [SlowFirst, FastSecond]
cmd.run()
assert [type(module) for module in cmd.executed] == [SlowFirst, FastSecond]
assert [entry["module"] for entry in cmd.timeline] == ["first", "second"]
assert [alert.message for alert in cmd.alertstore.alerts] == ["first", "second"]
def test_parallel_console_logs_are_grouped_and_file_log_is_complete(
self, tmp_path
):
output = StringIO()
handler = MVTLogHandler(
console=Console(file=output, force_terminal=False, color_system=None)
)
handler.setFormatter(logging.Formatter("%(message)s"))
logger = logging.getLogger("mvt")
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
release_slow = threading.Event()
class SlowLog(RecordingModule):
def run(self):
self.log.info("slow-a")
release_slow.wait(timeout=2)
self.log.info("slow-b")
class FastLog(RecordingModule):
def run(self):
self.log.info("fast-a")
self.log.info("fast-b")
release_slow.set()
SlowLog.__module__ = "mvt.tests.parallel"
FastLog.__module__ = "mvt.tests.parallel"
try:
cmd = RecordingCommand(results_path=str(tmp_path), jobs=2)
cmd.modules = [SlowLog, FastLog]
cmd.run()
finally:
logger.removeHandler(handler)
console_output = output.getvalue()
fast_block = (
console_output.index("--- FastLog ---"),
console_output.index("fast-a"),
console_output.index("fast-b"),
)
slow_block = (
console_output.index("--- SlowLog ---"),
console_output.index("slow-a"),
console_output.index("slow-b"),
)
assert fast_block == tuple(sorted(fast_block))
assert slow_block == tuple(sorted(slow_block))
assert fast_block[-1] < slow_block[0] or slow_block[-1] < fast_block[0]
file_output = (tmp_path / "command.log").read_text()
for message in ("slow-a", "slow-b", "fast-a", "fast-b"):
assert message in file_output
def test_encrypted_backup_stops_scheduling_and_cleans_up(self, caplog):
finish_called = []
class Encrypted(RecordingModule):
def run(self):
raise EncryptedBackupError
class NeverScheduled(RecordingModule):
dependencies = (Encrypted,)
class CleanupCommand(RecordingCommand):
def finish(self):
finish_called.append(True)
cmd = CleanupCommand(jobs=2)
cmd.modules = [Encrypted, NeverScheduled]
with caplog.at_level(logging.CRITICAL):
cmd.run()
assert finish_called == [True]
assert not any(isinstance(module, NeverScheduled) for module in cmd.executed)
assert "backup appears to be encrypted" in caplog.text
def test_profiling_forces_sequential_execution(self, monkeypatch, caplog):
run_order = []
class ProfileOne(RecordingModule):
def run(self):
run_order.append("one")
class ProfileTwo(RecordingModule):
def run(self):
run_order.append("two")
monkeypatch.setattr(settings, "PROFILE", True)
cmd = RecordingCommand(jobs=4)
cmd.modules = [ProfileOne, ProfileTwo]
with caplog.at_level(logging.WARNING):
cmd.run()
assert run_order == ["one", "two"]
assert "forcing sequential module execution" in caplog.text
def test_custom_module_replaces_builtin(self, caplog):
cmd = RecordingCommand()
cmd.platform = "ios"
@@ -790,10 +516,7 @@ class TestCommand:
with caplog.at_level(logging.INFO):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
"IndependentModule",
"ReplacementModule",
]
assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"]
assert cmd.module_replacements == {FirstModule: ReplacementModule}
assert (
"Module ReplacementModule from" in caplog.text
@@ -809,10 +532,7 @@ class TestCommand:
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
"FirstModule",
"CustomIOSBackupModule",
]
assert RecordingModule.run_order == ["FirstModule", "CustomIOSBackupModule"]
assert cmd.module_replacements == {}
def test_unavailable_replaced_module_is_ignored(self, caplog):
@@ -825,10 +545,7 @@ class TestCommand:
with caplog.at_level(logging.INFO):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
"IndependentModule",
"ReplacementModule",
]
assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"]
assert cmd.module_replacements == {}
assert "replaces module FirstModule" not in caplog.text
@@ -946,7 +663,7 @@ class TestCommand:
with caplog.at_level(logging.WARNING):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
assert RecordingModule.run_order == [
"ReplacementModule",
"OtherReplacementModule",
]
@@ -1005,9 +722,8 @@ class TestCommand:
assert cmd.module_replacements == {}
assert logged_substitutions(caplog) == []
@pytest.mark.parametrize("jobs", [1, 4])
def test_modules_replacing_each_other_all_run(self, caplog, jobs):
cmd = RecordingCommand(jobs=jobs)
def test_modules_replacing_each_other_all_run(self, caplog):
cmd = RecordingCommand()
cmd.platform = "ios"
cmd.name = "check-backup"
cmd.custom_modules = [MutualReplacementOne, MutualReplacementTwo]
@@ -1015,7 +731,7 @@ class TestCommand:
with caplog.at_level(logging.INFO):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
assert RecordingModule.run_order == [
"MutualReplacementOne",
"MutualReplacementTwo",
]
@@ -1026,9 +742,8 @@ class TestCommand:
)
assert logged_substitutions(caplog) == []
@pytest.mark.parametrize("jobs", [1, 4])
def test_replacement_cycle_of_three_modules_all_run(self, caplog, jobs):
cmd = RecordingCommand(jobs=jobs)
def test_replacement_cycle_of_three_modules_all_run(self, caplog):
cmd = RecordingCommand()
cmd.platform = "ios"
cmd.name = "check-backup"
cmd.custom_modules = [CycleOneModule, CycleTwoModule, CycleThreeModule]
@@ -1036,7 +751,7 @@ class TestCommand:
with caplog.at_level(logging.INFO):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
assert RecordingModule.run_order == [
"CycleOneModule",
"CycleTwoModule",
"CycleThreeModule",
@@ -1119,7 +834,7 @@ class TestCommand:
with caplog.at_level(logging.WARNING):
cmd.run()
assert [type(module).__name__ for module in cmd.executed] == [
assert RecordingModule.run_order == [
"SharedSlugModule",
"OtherSharedSlugModule",
]
-8
View File
@@ -41,11 +41,3 @@ class TestCheckBackupCommand:
result = runner.invoke(check_backup, [str(tmp_path)])
assert result.exit_code == 1
assert "does not appear to be an iTunes backup folder" in caplog.text
def test_check_rejects_non_positive_jobs(self):
result = CliRunner().invoke(
check_backup, ["--jobs", "0", get_ios_backup_folder()]
)
assert result.exit_code == 2
assert "Invalid value for '--jobs'" in result.output
+8 -12
View File
@@ -288,19 +288,15 @@ def test_androidqf_propagates_custom_modules_to_nested_commands(tmp_path, monkey
cmd = CmdAndroidCheckAndroidQF(
target_path=str(tmp_path),
custom_modules=custom_modules,
jobs=7,
)
def record_available(name):
def _record(command):
records[name] = {
"jobs": command.jobs,
"modules": [
module.__name__
for module in command._available_modules()
if module.__name__.startswith("Nested")
],
}
records[name] = [
module.__name__
for module in command._available_modules()
if module.__name__.startswith("Nested")
]
return _record
@@ -343,7 +339,7 @@ def test_androidqf_propagates_custom_modules_to_nested_commands(tmp_path, monkey
assert cmd.run_backup_cmd()
assert cmd.run_intrusion_logs_cmd()
assert records == {
"bugreport": {"jobs": 7, "modules": ["NestedBugreportModule"]},
"backup": {"jobs": 7, "modules": ["NestedBackupModule"]},
"intrusion_logs": {"jobs": 7, "modules": ["NestedIntrusionLogsModule"]},
"bugreport": ["NestedBugreportModule"],
"backup": ["NestedBackupModule"],
"intrusion_logs": ["NestedIntrusionLogsModule"],
}