Add a SysdiagnoseInfo module to check-sysdiagnose (#917)

* Add a SysdiagnoseInfo module to check-sysdiagnose

check-sysdiagnose had no module of its own: it prepared the archive for
plugin modules and refused to run without one. SysdiagnoseInfo is the
first built-in module. It writes sysdiagnose_info.json with details
about the device and the archive: product type and model, iOS version
and build, serial number, IMEI, MEID and UDID from remotectl_dumpstate.txt
and the mobile activation request, the Apple account name and email from
the App Store daemon database, and the archive's original file name and
creation time from sysdiagnose.log. The build is checked against the
known iOS versions the way BackupInfo does.

The App Store database is copied out of the archive together with its
-wal and -shm sidecars before it is opened, so rows still in the
write-ahead log are read.

With a built-in module the command's list is never empty, so the "no
custom modules" error and its test go. The module joins
IOS_CHECK_IOCS_MODULES like every other module that writes a results
file.

* Note that newer sysdiagnoses lack the App Store daemon database

* Keep refusing check-sysdiagnose runs without a custom module

* Warn instead of refusing when no forensic sysdiagnose module is loaded
This commit is contained in:
Donncha Ó Cearbhaill
2026-09-05 23:45:14 +02:00
committed by GitHub
parent b9055e365c
commit 2eb40b85cf
12 changed files with 451 additions and 22 deletions
+8
View File
@@ -435,3 +435,11 @@ This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extr
This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup.
---
## Records extracted by `check-sysdiagnose`
### `sysdiagnose_info.json`
This JSON file is created by mvt-ios' `SysdiagnoseInfo` module. The module extracts details about the device and the sysdiagnose itself: the product type and model, iOS version and build, serial number, IMEI, MEID and UDID from the remotectl dump state and the mobile activation request, the Apple account name and email from the App Store daemon database (no longer part of a sysdiagnose on newer iOS versions, still read from older archives), and the original file name and creation time of the archive from *sysdiagnose.log*.
+7 -4
View File
@@ -1,10 +1,13 @@
# Check an iOS Sysdiagnose
`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
custom MVT modules. MVT does not include built-in sysdiagnose modules. The
command runs the modules of the installed
`mvt-ios check-sysdiagnose` analyzes an iOS sysdiagnose archive. MVT's own
`SysdiagnoseInfo` module extracts details about the device and the archive
(see [`sysdiagnose_info.json`](records.md#sysdiagnose_infojson)); the checks
come from the modules of the installed
[plugin packages](../development/index.md#installed-module-packages) which
declare support for it. Install at least one such package first.
declare support for the command. Without any such module the command still
records the device details, and warns that no forensic sysdiagnose modules
have been loaded so that the run cannot pass for a clean analysis.
The command accepts either an extracted sysdiagnose directory or the original
gzip-compressed tar archive.
+8 -5
View File
@@ -449,11 +449,14 @@ def check_sysdiagnose(
custom_modules=custom_modules,
)
if not cmd._available_modules():
raise click.ClickException(
"No custom modules support mvt-ios check-sysdiagnose. "
"Load a module that declares supported_commands = "
"((\"ios\", \"check-sysdiagnose\"),)."
# MVT's own module only records the device details; the checks come from
# custom modules, so a run without any must not look like a clean analysis.
if all(module in cmd.modules for module in cmd._available_modules()):
log.warning(
"No forensic sysdiagnose modules have been loaded: MVT's own "
"SysdiagnoseInfo module only records the device details. Install a "
"module package or load a module that declares supported_commands = "
'(("ios", "check-sysdiagnose"),) to check the sysdiagnose.'
)
if list_modules:
+3
View File
@@ -16,6 +16,8 @@ from mvt.common.command import Command
from mvt.common.indicators import Indicators
from mvt.common.module import MVTModule
from .modules.sysdiagnose import SYSDIAGNOSE_MODULES
log = logging.getLogger(__name__)
@@ -52,6 +54,7 @@ class CmdIOSCheckSysdiagnose(Command):
)
self.platform = "ios"
self.name = "check-sysdiagnose"
self.modules = SYSDIAGNOSE_MODULES
self.sysdiagnose_format: Optional[str] = None
self.sysdiagnose_archive: Optional[tarfile.TarFile] = None
self.sysdiagnose_files: list[str] = []
+2 -1
View File
@@ -16,7 +16,8 @@ from mvt.common.module import MVTModule
from .modules.backup import BACKUP_MODULES
from .modules.fs import FS_MODULES
from .modules.mixed import MIXED_MODULES
from .modules.sysdiagnose import SYSDIAGNOSE_MODULES
IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = (
BACKUP_MODULES + FS_MODULES + MIXED_MODULES
BACKUP_MODULES + FS_MODULES + MIXED_MODULES + SYSDIAGNOSE_MODULES
)
@@ -3,4 +3,9 @@
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from mvt.common.module import MVTModule
from .base import SysdiagnoseExtraction
from .sysdiagnose_info import SysdiagnoseInfo
SYSDIAGNOSE_MODULES: list[type[MVTModule]] = [SysdiagnoseInfo]
@@ -0,0 +1,228 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import json
import logging
import os
import plistlib
import re
import sqlite3
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional
from mvt.common.module_types import ModuleResults
from mvt.common.utils import convert_datetime_to_iso
from mvt.ios.versions import (
find_version_by_build,
get_device_desc_from_id,
is_ios_version_outdated,
)
from .base import SysdiagnoseExtraction
# The fields dumpsys prints in the remotectl dump state and the mobile
# activation request which are worth a log line of their own.
LOGGED_FIELDS = (
"ProductName",
"ProductType",
"SerialNumber",
"OSVersion",
"RegionCode",
"IMEI",
"BuildVersion",
)
class SysdiagnoseInfo(SysdiagnoseExtraction):
"""Extract details about the device and the sysdiagnose itself.
The fields come from four files of the archive: the remotectl dump state
(product type, OS version, serial number, region and the rest of its
Properties block), the mobile activation request (UDID, IMEI, MEID and the
OS build), the App Store daemon database (the Apple account name and email)
and sysdiagnose.log (the archive's original file name and creation time).
Newer iOS versions no longer include the App Store daemon database in a
sysdiagnose; it is still read for the analysis of older archives.
"""
def __init__(
self,
file_path: Optional[str] = None,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
module_options: Optional[dict] = None,
log: logging.Logger = logging.getLogger(__name__),
results: Optional[ModuleResults] = None,
) -> None:
super().__init__(
file_path=file_path,
target_path=target_path,
results_path=results_path,
module_options=module_options,
log=log,
results=results,
)
self.results: dict = results if results is not None else {}
def _copy_sqlite_db(self, file_path: str, directory: str) -> str:
"""Copy a database and its WAL sidecars out of the archive.
A database dumped mid-transaction keeps its latest rows in the -wal
file next to it, which SQLite only reads when both sit in the same
directory under the same name.
"""
available_files = self.tar_files if self.tar else self.files
for suffix in ("", "-wal", "-shm"):
if suffix and f"{file_path}{suffix}" not in available_files:
continue
copy_path = os.path.join(directory, f"{Path(file_path).name}{suffix}")
with open(copy_path, "wb") as handle:
handle.write(self._get_file_content(f"{file_path}{suffix}"))
return os.path.join(directory, Path(file_path).name)
def _process_appstored(self, file_path: str) -> None:
self.log.info("Found App Store daemon database at: %s", file_path)
with tempfile.TemporaryDirectory(prefix="mvt_sqlite_") as directory:
db_path = Path(self._copy_sqlite_db(file_path, directory)).resolve()
conn = sqlite3.connect(f"{db_path.as_uri()}?mode=ro", uri=True)
try:
self._read_appstored(conn)
finally:
conn.close()
def _read_appstored(self, conn: sqlite3.Connection) -> None:
cur = conn.cursor()
# The account name sits in an opaque structure of every asset row.
try:
rows = cur.execute("SELECT sinfs_data FROM asset;").fetchall()
except sqlite3.DatabaseError as exc:
self.log.debug("Unable to read the asset table: %s", exc)
rows = []
for (sinfs_data,) in rows:
try:
sinf = plistlib.loads(sinfs_data)[0]["sinf"]
except (plistlib.InvalidFileException, IndexError, KeyError, TypeError):
continue
match = re.search(rb"name(.*?)\x00", sinf)
if match:
self.results["Account Name"] = match.group(1).decode(
"utf-8", errors="replace"
)
break
try:
row = cur.execute(
"SELECT store_account_name FROM job_software "
"WHERE store_account_name IS NOT NULL LIMIT 1;"
).fetchone()
except sqlite3.DatabaseError as exc:
self.log.debug("Unable to read the job_software table: %s", exc)
return
if row:
self.results["Email Address"] = row[0]
def _process_activation_log(self, file_path: str) -> None:
self.log.info("Found mobile activation request at: %s", file_path)
content = self._get_file_content(file_path)
match = re.search(rb"BODY:\s+({.+?})\s", content, re.MULTILINE)
if not match:
return
try:
body = json.loads(match.group(1))
except json.JSONDecodeError as exc:
self.log.warning("Unable to parse the activation request body: %s", exc)
return
self.results.update(
{
"SerialNumber": body.get("serial-number"),
"ProductType": body.get("productType"),
"ProductName": body.get("productName"),
"IMEI": body.get("imei"),
"ProductVersion": body.get("os-version"),
"UniqueIdentifier": body.get("udid"),
"MEID": body.get("meid"),
"BuildVersion": body.get("os-build"),
}
)
def _process_dumpstate(self, file_path: str) -> None:
self.log.info("Found remotectl dump state at: %s", file_path)
content = self._get_file_content(file_path).decode("utf-8", errors="replace")
in_properties = False
for line in content.splitlines():
if not in_properties:
in_properties = line == "\tProperties: {"
continue
if line == "\t}":
break
key, separator, value = line.partition("=>")
if separator:
self.results[key.strip()] = value.strip()
def _process_sysdiagnose_log(self, file_path: str) -> None:
self.log.info("Found sysdiagnose.log at: %s", file_path)
content = self._get_file_content(file_path).decode("utf-8", errors="replace")
match = re.search(r"sysdiagnose_\S+?\.tar\.gz", content)
if not match:
self.log.info("Could not find the original output path in sysdiagnose.log")
return
file_name = os.path.basename(match.group(0))
try:
created = datetime.strptime(
"_".join(file_name.split("_")[1:3]), "%Y.%m.%d_%H-%M-%S%z"
)
except ValueError:
self.log.warning("Unexpected sysdiagnose file name: %s", file_name)
return
self.results["OriginalFilename"] = file_name
self.results["CreatedTimestamp"] = convert_datetime_to_iso(created)
def run(self) -> None:
for file_path in self._get_files_by_pattern(
"*/logs/appinstallation/appstored.sqlitedb"
):
self._process_appstored(file_path)
for file_path in self._get_files_by_pattern(
"*/logs/MobileActivation/collection_oob_request.txt"
):
self._process_activation_log(file_path)
for file_path in self._get_files_by_pattern("*/remotectl_dumpstate.txt"):
self._process_dumpstate(file_path)
for file_path in self._get_files_by_pattern("*/sysdiagnose.log"):
self._process_sysdiagnose_log(file_path)
# The activation request names the product "iPhone OS"; the model
# description is what an analyst wants to read.
product_name = get_device_desc_from_id(self.results.get("ProductType", ""))
if product_name:
self.results["ProductName"] = product_name
for field in LOGGED_FIELDS:
if field not in self.results:
continue
value = self.results[field]
if field == "BuildVersion" and value:
self.log.info("%s: %s - %s", field, value, find_version_by_build(value))
else:
self.log.info("%s: %s", field, value)
if self.results.get("BuildVersion"):
is_ios_version_outdated(self.results["BuildVersion"], self.log)
+4 -1
View File
@@ -12,13 +12,16 @@ from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES
from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES
from mvt.ios.modules.fs import FS_MODULES
from mvt.ios.modules.mixed import MIXED_MODULES
from mvt.ios.modules.sysdiagnose import SYSDIAGNOSE_MODULES
def test_the_check_iocs_lists_are_the_families_of_their_platform():
# The CLI reads these same lists, so nothing composing one elsewhere can
# drift from what the command runs. This pins what the lists are composed
# of.
assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES
assert IOS_CHECK_IOCS_MODULES == (
IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES + SYSDIAGNOSE_MODULES
)
assert ANDROID_CHECK_IOCS_MODULES == (
ANDROID_BACKUP_MODULES
+ BUGREPORT_MODULES
+4
View File
@@ -0,0 +1,4 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
@@ -0,0 +1,161 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import json
import plistlib
import sqlite3
import tarfile
from mvt.common.module import run_module
from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
from mvt.ios.modules.sysdiagnose.sysdiagnose_info import SysdiagnoseInfo
from mvt.ios.versions import get_device_desc_from_id
# The name sysdiagnose gives its archive: the time it ran, then the OS and build.
ARCHIVE_NAME = "sysdiagnose_2024.01.02_03-04-05+0200_iPhone-OS_iPhone_21C62"
DUMPSTATE = (
"Found device: ...\n"
"\tProperties: {\n"
"\t\tProductType => iPhone12,1\n"
"\t\tOSVersion => 17.2\n"
"\t\tSerialNumber => C0FFEE000000\n"
"\t\tRegionCode => LL\n"
"\t}\n"
"\tServices: {\n"
"\t\tcom.apple.example => ignored\n"
"\t}\n"
)
ACTIVATION_BODY = {
"serial-number": "C0FFEE000000",
"productType": "iPhone12,1",
"productName": "iPhone OS",
"imei": "000000000000000",
"os-version": "17.2",
"os-build": "21C62",
"udid": "00000000-0000000000000000",
"meid": "00000000000000",
}
def make_sysdiagnose(tmp_path, activation_body=None):
folder = tmp_path / ARCHIVE_NAME
folder.mkdir()
(folder / "sysdiagnose.log").write_text(
f"Output available at '/private/var/tmp/{ARCHIVE_NAME}.tar.gz'\n",
encoding="utf-8",
)
(folder / "remotectl_dumpstate.txt").write_text(DUMPSTATE, encoding="utf-8")
activation = folder / "logs" / "MobileActivation"
activation.mkdir(parents=True)
body = json.dumps(
activation_body if activation_body is not None else ACTIVATION_BODY
)
(activation / "collection_oob_request.txt").write_text(
f"HEADERS: {{}}\nBODY: {body}\nEND\n", encoding="utf-8"
)
appinstallation = folder / "logs" / "appinstallation"
appinstallation.mkdir(parents=True)
conn = sqlite3.connect(appinstallation / "appstored.sqlitedb")
conn.execute("CREATE TABLE asset (sinfs_data BLOB)")
conn.execute(
"INSERT INTO asset VALUES (?)",
(plistlib.dumps([{"sinf": b"\x00\x10nameExample Person\x00\x00rest"}]),),
)
conn.execute("CREATE TABLE job_software (store_account_name TEXT)")
conn.execute("INSERT INTO job_software VALUES (NULL)")
conn.execute("INSERT INTO job_software VALUES ('person@example.com')")
conn.commit()
conn.close()
return folder
def run_command(target, results_path=None):
command = CmdIOSCheckSysdiagnose(target_path=str(target), results_path=results_path)
command.run()
(module,) = [m for m in command.executed if isinstance(m, SysdiagnoseInfo)]
return module
def test_device_details_from_a_sysdiagnose_folder(tmp_path):
results_path = tmp_path / "results"
results_path.mkdir()
module = run_command(make_sysdiagnose(tmp_path), str(results_path))
assert module.results["SerialNumber"] == "C0FFEE000000"
assert module.results["ProductType"] == "iPhone12,1"
assert module.results["ProductName"] == get_device_desc_from_id("iPhone12,1")
assert module.results["ProductName"] != "iPhone OS"
assert module.results["OSVersion"] == "17.2"
assert module.results["BuildVersion"] == "21C62"
assert module.results["UniqueIdentifier"] == "00000000-0000000000000000"
assert module.results["RegionCode"] == "LL"
assert "com.apple.example" not in module.results
assert module.results["Account Name"] == "Example Person"
assert module.results["Email Address"] == "person@example.com"
assert module.results["OriginalFilename"] == f"{ARCHIVE_NAME}.tar.gz"
assert module.results["CreatedTimestamp"] == "2024-01-02 01:04:05.000000"
assert (results_path / "sysdiagnose_info.json").exists()
def test_device_details_from_a_sysdiagnose_archive(tmp_path):
folder = make_sysdiagnose(tmp_path)
archive_path = tmp_path / f"{ARCHIVE_NAME}.tar.gz"
with tarfile.open(archive_path, "w:gz") as archive:
archive.add(folder, arcname=ARCHIVE_NAME)
module = run_command(archive_path)
assert module.results["SerialNumber"] == "C0FFEE000000"
assert module.results["Account Name"] == "Example Person"
assert module.results["OriginalFilename"] == f"{ARCHIVE_NAME}.tar.gz"
def test_wal_sidecars_are_copied_beside_the_database(tmp_path):
folder = tmp_path / ARCHIVE_NAME
(folder / "logs").mkdir(parents=True)
(folder / "logs" / "db.sqlite").write_bytes(b"main")
(folder / "logs" / "db.sqlite-wal").write_bytes(b"wal")
module = SysdiagnoseInfo()
module.from_sysdiagnose_folder(
str(folder),
[f"{ARCHIVE_NAME}/logs/db.sqlite", f"{ARCHIVE_NAME}/logs/db.sqlite-wal"],
)
copies = tmp_path / "copies"
copies.mkdir()
db_path = module._copy_sqlite_db(f"{ARCHIVE_NAME}/logs/db.sqlite", str(copies))
assert db_path == str(copies / "db.sqlite")
assert (copies / "db.sqlite").read_bytes() == b"main"
assert (copies / "db.sqlite-wal").read_bytes() == b"wal"
assert not (copies / "db.sqlite-shm").exists()
def test_a_sysdiagnose_without_the_files_yields_nothing(tmp_path):
folder = tmp_path / ARCHIVE_NAME
folder.mkdir()
(folder / "other.txt").write_text("nothing here", encoding="utf-8")
module = SysdiagnoseInfo()
module.from_sysdiagnose_folder(str(folder), [f"{ARCHIVE_NAME}/other.txt"])
run_module(module)
assert module.results == {}
def test_a_malformed_activation_request_is_skipped(tmp_path):
folder = make_sysdiagnose(tmp_path)
(folder / "logs" / "MobileActivation" / "collection_oob_request.txt").write_text(
"BODY: {not json}\n", encoding="utf-8"
)
module = run_command(folder)
assert "IMEI" not in module.results
assert module.results["SerialNumber"] == "C0FFEE000000"
+11 -4
View File
@@ -1,3 +1,5 @@
import logging
from click.testing import CliRunner
from mvt.ios.cli import check_sysdiagnose
@@ -50,8 +52,13 @@ def test_check_sysdiagnose_runs_explicitly_scoped_custom_module(tmp_path):
assert (output_path / "custom_sysdiagnose_module.json").exists()
def test_check_sysdiagnose_requires_an_explicitly_scoped_module(tmp_path):
result = CliRunner().invoke(check_sysdiagnose, [str(_create_sysdiagnose_folder(tmp_path))])
def test_check_sysdiagnose_warns_without_a_custom_module(tmp_path, caplog):
# The built-in SysdiagnoseInfo alone performs no check, so the run goes
# ahead but says so.
with caplog.at_level(logging.WARNING, logger="mvt"):
result = CliRunner().invoke(
check_sysdiagnose, [str(_create_sysdiagnose_folder(tmp_path))]
)
assert result.exit_code != 0
assert "No custom modules support mvt-ios check-sysdiagnose" in result.output
assert result.exit_code == 0
assert "No forensic sysdiagnose modules have been loaded" in caplog.text
+10 -7
View File
@@ -45,6 +45,11 @@ def _create_sysdiagnose_archive(tmp_path, folder):
return archive_path
def _test_module(command):
(module,) = [m for m in command.executed if isinstance(m, SysdiagnoseTestModule)]
return module
def _run_command(path):
command = CmdIOSCheckSysdiagnose(
target_path=str(path), custom_modules=[SysdiagnoseTestModule]
@@ -56,10 +61,10 @@ def _run_command(path):
def test_check_sysdiagnose_from_folder(tmp_path):
command = _run_command(_create_sysdiagnose_folder(tmp_path))
assert command.executed[0].results == [
assert _test_module(command).results == [
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
]
assert command.executed[0].ips_files == [
assert _test_module(command).ips_files == [
{"file_path": str(tmp_path / "sysdiagnose" / "report.ips"), "bug_type": 210}
]
@@ -68,14 +73,12 @@ def test_check_sysdiagnose_from_archive_closes_archive(tmp_path):
folder = _create_sysdiagnose_folder(tmp_path)
command = _run_command(_create_sysdiagnose_archive(tmp_path, folder))
assert command.executed[0].results == [
assert _test_module(command).results == [
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
]
assert command.executed[0].ips_files == [
assert _test_module(command).ips_files == [
{
"file_path": str(
Path(command.extracted_sysdiagnose_path) / "report.ips"
),
"file_path": str(Path(command.extracted_sysdiagnose_path) / "report.ips"),
"bug_type": 210,
}
]