Add a SysdiagnoseInfo module to check-sysdiagnose (#917)

* Add a SysdiagnoseInfo module to check-sysdiagnose

check-sysdiagnose had no module of its own: it prepared the archive for
plugin modules and refused to run without one. SysdiagnoseInfo is the
first built-in module. It writes sysdiagnose_info.json with details
about the device and the archive: product type and model, iOS version
and build, serial number, IMEI, MEID and UDID from remotectl_dumpstate.txt
and the mobile activation request, the Apple account name and email from
the App Store daemon database, and the archive's original file name and
creation time from sysdiagnose.log. The build is checked against the
known iOS versions the way BackupInfo does.

The App Store database is copied out of the archive together with its
-wal and -shm sidecars before it is opened, so rows still in the
write-ahead log are read.

With a built-in module the command's list is never empty, so the "no
custom modules" error and its test go. The module joins
IOS_CHECK_IOCS_MODULES like every other module that writes a results
file.

* Note that newer sysdiagnoses lack the App Store daemon database

* Keep refusing check-sysdiagnose runs without a custom module

* Warn instead of refusing when no forensic sysdiagnose module is loaded

docs/ios/records.md

@@ -435,3 +435,11 @@ This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extr
 
 This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup.
 
+
+---
+
+## Records extracted by `check-sysdiagnose`
+
+### `sysdiagnose_info.json`
+
+This JSON file is created by mvt-ios' `SysdiagnoseInfo` module. The module extracts details about the device and the sysdiagnose itself: the product type and model, iOS version and build, serial number, IMEI, MEID and UDID from the remotectl dump state and the mobile activation request, the Apple account name and email from the App Store daemon database (no longer part of a sysdiagnose on newer iOS versions, still read from older archives), and the original file name and creation time of the archive from *sysdiagnose.log*.

docs/ios/sysdiagnose.md

@@ -1,10 +1,13 @@
 # Check an iOS Sysdiagnose
 
-`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
-custom MVT modules. MVT does not include built-in sysdiagnose modules. The
-command runs the modules of the installed
+`mvt-ios check-sysdiagnose` analyzes an iOS sysdiagnose archive. MVT's own
+`SysdiagnoseInfo` module extracts details about the device and the archive
+(see [`sysdiagnose_info.json`](records.md#sysdiagnose_infojson)); the checks
+come from the modules of the installed
 [plugin packages](../development/index.md#installed-module-packages) which
-declare support for it. Install at least one such package first.
+declare support for the command. Without any such module the command still
+records the device details, and warns that no forensic sysdiagnose modules
+have been loaded so that the run cannot pass for a clean analysis.
 
 The command accepts either an extracted sysdiagnose directory or the original
 gzip-compressed tar archive.

src/mvt/common/utils.py

@@ -3,7 +3,6 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import base64
 import cProfile
 import datetime
 import hashlib
@@ -185,29 +184,6 @@ def keys_bytes_to_string(obj: Any) -> Any:
     return new_obj
 
 
-def sanitize_json_data(obj: Any) -> Any:
-    """Recursively convert values to JSON-compatible representations.
-
-    SQLite and plist records can contain binary and date values nested inside
-    dictionaries. Preserve binary data as base64 rather than discarding it or
-    relying on a lossy text decoding during JSON serialization.
-    """
-    if isinstance(obj, datetime.datetime):
-        return convert_datetime_to_iso(obj)
-    if isinstance(obj, datetime.date):
-        return obj.isoformat()
-    if isinstance(obj, bytes):
-        return base64.b64encode(obj).decode("ascii")
-    if isinstance(obj, dict):
-        return {
-            sanitize_json_data(key): sanitize_json_data(value)
-            for key, value in obj.items()
-        }
-    if isinstance(obj, (tuple, list, set)):
-        return [sanitize_json_data(value) for value in obj]
-    return obj
-
-
 def get_sha256_from_file_path(file_path: str) -> str:
     """Calculate the SHA256 hash of a file from a file path.
 

src/mvt/ios/cli.py

@@ -449,11 +449,14 @@ def check_sysdiagnose(
         custom_modules=custom_modules,
     )
 
-    if not cmd._available_modules():
-        raise click.ClickException(
-            "No custom modules support mvt-ios check-sysdiagnose. "
-            "Load a module that declares supported_commands = "
-            "((\"ios\", \"check-sysdiagnose\"),)."
+    # MVT's own module only records the device details; the checks come from
+    # custom modules, so a run without any must not look like a clean analysis.
+    if all(module in cmd.modules for module in cmd._available_modules()):
+        log.warning(
+            "No forensic sysdiagnose modules have been loaded: MVT's own "
+            "SysdiagnoseInfo module only records the device details. Install a "
+            "module package or load a module that declares supported_commands = "
+            '(("ios", "check-sysdiagnose"),) to check the sysdiagnose.'
         )
 
     if list_modules:

src/mvt/ios/cmd_check_sysdiagnose.py

@@ -16,6 +16,8 @@ from mvt.common.command import Command
 from mvt.common.indicators import Indicators
 from mvt.common.module import MVTModule
 
+from .modules.sysdiagnose import SYSDIAGNOSE_MODULES
+
 log = logging.getLogger(__name__)
 
 
@@ -52,6 +54,7 @@ class CmdIOSCheckSysdiagnose(Command):
         )
         self.platform = "ios"
         self.name = "check-sysdiagnose"
+        self.modules = SYSDIAGNOSE_MODULES
         self.sysdiagnose_format: Optional[str] = None
         self.sysdiagnose_archive: Optional[tarfile.TarFile] = None
         self.sysdiagnose_files: list[str] = []

src/mvt/ios/command_modules.py

@@ -16,7 +16,8 @@ from mvt.common.module import MVTModule
 from .modules.backup import BACKUP_MODULES
 from .modules.fs import FS_MODULES
 from .modules.mixed import MIXED_MODULES
+from .modules.sysdiagnose import SYSDIAGNOSE_MODULES
 
 IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = (
-    BACKUP_MODULES + FS_MODULES + MIXED_MODULES
+    BACKUP_MODULES + FS_MODULES + MIXED_MODULES + SYSDIAGNOSE_MODULES
 )

src/mvt/ios/modules/mixed/calendar.py

@@ -11,7 +11,7 @@ from mvt.common.module_types import (
     ModuleResults,
     ModuleSerializedResult,
 )
-from mvt.common.utils import convert_mactime_to_iso, sanitize_json_data
+from mvt.common.utils import convert_mactime_to_iso
 
 from ..base import IOSExtraction
 
@@ -90,63 +90,54 @@ class Calendar(IOSExtraction):
         """
         conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
-        try:
-            cur.execute("PRAGMA table_info(CalendarItem);")
-            calendar_columns = [row[1] for row in cur]
-            cur.execute("PRAGMA table_info(Participant);")
-            participant_columns = [row[1] for row in cur]
-            selected = [
-                f'c."{column}" AS "calendar_{column}"' for column in calendar_columns
-            ] + [
-                f'p."{column}" AS "participant_{column}"'
-                for column in participant_columns
-            ]
-            cur.execute(
-                f"SELECT {', '.join(selected)} FROM CalendarItem c "
-                "LEFT JOIN Participant p ON p.ROWID = c.organizer_id;"
-            )
-            names = [description[0] for description in cur.description]
-            for row in cur:
-                raw = sanitize_json_data(dict(zip(names, row)))
-                entry = {
-                    "id": raw.get("calendar_ROWID"),
-                    "summary": raw.get("calendar_summary"),
-                    "description": raw.get("calendar_description"),
-                    "all_day": raw.get("calendar_all_day"),
-                    "calendar_id": raw.get("calendar_calendar_id"),
-                    "organizer_id": raw.get("calendar_organizer_id"),
-                    "url": raw.get("calendar_url"),
-                    "external_id": raw.get("calendar_external_id"),
-                    "external_mod_tag": raw.get("calendar_external_mod_tag"),
-                    "unique_identifier": raw.get("calendar_unique_identifier"),
-                    "hidden": raw.get("calendar_hidden"),
-                    "uuid": raw.get("calendar_UUID"),
-                    "action": raw.get("calendar_action"),
-                    "created_by_id": raw.get("calendar_created_by_id"),
-                    "participant_uuid": raw.get("participant_UUID"),
-                    "participant_email": raw.get("participant_email"),
-                    "participant_phone": raw.get("participant_phone_number"),
-                    "participant_comment": raw.get("participant_comment"),
-                    "record": raw,
-                }
-                timestamp_sources = {
-                    "start_date": "calendar_start_date",
-                    "end_date": "calendar_end_date",
-                    "last_modified": "calendar_last_modified",
-                    "creation_date": "calendar_creation_date",
-                    "participant_last_modified": "participant_last_modified",
-                }
-                for output, source in timestamp_sources.items():
-                    value = raw.get(source)
-                    entry[output] = (
-                        convert_mactime_to_iso(value)
-                        if value is not None and not isinstance(value, str)
-                        else value
-                    )
-                self.results.append(entry)
-        finally:
-            cur.close()
-            conn.close()
+
+        cur.execute(
+            """
+        SELECT
+            CalendarItem.ROWID as "id",
+            CalendarItem.summary as "summary",
+            CalendarItem.description as "description",
+            CalendarItem.start_date as "start_date",
+            CalendarItem.end_date as "end_date",
+            CalendarItem.all_day as "all_day",
+            CalendarItem.calendar_id as "calendar_id",
+            CalendarItem.organizer_id as "organizer_id",
+            CalendarItem.url as "url",
+            CalendarItem.last_modified as "last_modified",
+            CalendarItem.external_id as "external_id",
+            CalendarItem.external_mod_tag as "external_mod_tag",
+            CalendarItem.unique_identifier as "unique_identifier",
+            CalendarItem.hidden as "hidden",
+            CalendarItem.UUID as "uuid",
+            CalendarItem.creation_date as "creation_date",
+            CalendarItem.action as "action",
+            CalendarItem.created_by_id as "created_by_id",
+            Participant.UUID as "participant_uuid",
+            Participant.email as "participant_email",
+            Participant.phone_number as "participant_phone",
+            Participant.comment as "participant_comment",
+            Participant.last_modified as "participant_last_modified"
+        FROM CalendarItem
+            LEFT JOIN Participant ON Participant.ROWID = CalendarItem.organizer_id;
+        """
+        )
+
+        names = [description[0] for description in cur.description]
+        for item in cur:
+            entry = {}
+            for index, value in enumerate(item):
+                if names[index] in self.timestamps:
+                    if value is None or isinstance(value, str):
+                        entry[names[index]] = value
+                    else:
+                        entry[names[index]] = convert_mactime_to_iso(value)
+                else:
+                    entry[names[index]] = value
+
+            self.results.append(entry)
+
+        cur.close()
+        conn.close()
 
     def run(self) -> None:
         self._find_ios_database(

src/mvt/ios/modules/mixed/calls.py

@@ -7,7 +7,7 @@ import logging
 from typing import Optional
 
 from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
-from mvt.common.utils import convert_mactime_to_iso, sanitize_json_data
+from mvt.common.utils import convert_mactime_to_iso
 
 from ..base import IOSExtraction
 
@@ -57,30 +57,31 @@ class Calls(IOSExtraction):
             return
         conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
-        try:
-            cur.execute("SELECT * FROM ZCALLRECORD;")
-            names = [description[0] for description in cur.description]
-            for row in cur:
-                original = dict(zip(names, row))
-                address = original.get("ZADDRESS")
-                if isinstance(address, bytes):
-                    address = address.decode("utf-8", errors="replace")
-                raw = sanitize_json_data(original)
-                date = raw.get("ZDATE")
-                self.results.append(
-                    {
-                        "isodate": (
-                            convert_mactime_to_iso(date) if date is not None else ""
-                        ),
-                        "duration": raw.get("ZDURATION"),
-                        "location": raw.get("ZLOCATION"),
-                        "number": address,
-                        "provider": raw.get("ZSERVICE_PROVIDER"),
-                        "call": raw,
-                    }
-                )
-        finally:
-            cur.close()
-            conn.close()
+        cur.execute(
+            """
+            SELECT
+                ZDATE, ZDURATION, ZLOCATION, ZADDRESS, ZSERVICE_PROVIDER
+            FROM ZCALLRECORD;
+        """
+        )
+        # names = [description[0] for description in cur.description]
+
+        for row in cur:
+            self.results.append(
+                {
+                    "isodate": convert_mactime_to_iso(row[0]),
+                    "duration": row[1],
+                    "location": row[2],
+                    "number": (
+                        row[3].decode("utf-8")
+                        if isinstance(row[3], bytes)
+                        else row[3]
+                    ),
+                    "provider": row[4],
+                }
+            )
+
+        cur.close()
+        conn.close()
 
         self.log.info("Extracted a total of %d calls", len(self.results))

src/mvt/ios/modules/mixed/contacts.py

@@ -8,7 +8,6 @@ import sqlite3
 from typing import Optional
 
 from mvt.common.module_types import ModuleResults
-from mvt.common.utils import sanitize_json_data
 
 from ..base import IOSExtraction
 
@@ -52,45 +51,28 @@ class Contacts(IOSExtraction):
         conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
         try:
-            cur.execute("PRAGMA table_info(ABPerson);")
-            person_columns = [row[1] for row in cur]
-            cur.execute("PRAGMA table_info(ABMultiValue);")
-            multivalue_columns = [row[1] for row in cur]
-            if not person_columns or not multivalue_columns:
-                return
-
-            select_columns = [
-                f'p."{column}" AS "person_{column}"' for column in person_columns
-            ] + [
-                f'm."{column}" AS "multivalue_{column}"'
-                for column in multivalue_columns
-            ]
-            cur.execute(
-                f"SELECT {', '.join(select_columns)} "
-                "FROM ABPerson p LEFT JOIN ABMultiValue m "
-                "ON p.ROWID = m.record_id ORDER BY p.ROWID, m.ROWID;"
-            )
-            names = [description[0] for description in cur.description]
-            for row in cur:
-                raw = sanitize_json_data(dict(zip(names, row)))
-                self.results.append(
-                    {
-                        "value": raw.get("multivalue_value"),
-                        "first": raw.get("person_First"),
-                        "middle": raw.get("person_Middle"),
-                        "last": raw.get("person_Last"),
-                        "organization": raw.get("person_Organization"),
-                        # Keep the capitalization returned by the previous
-                        # query for consumers of existing JSON output.
-                        "First": raw.get("person_First"),
-                        "Middle": raw.get("person_Middle"),
-                        "Last": raw.get("person_Last"),
-                        "Organization": raw.get("person_Organization"),
-                        "contact": raw,
-                    }
+            try:
+                cur.execute(
+                    """
+                    SELECT
+                        multi.value, person.first, person.middle, person.last,
+                        person.organization
+                    FROM ABPerson person, ABMultiValue multi
+                    WHERE person.rowid = multi.record_id and multi.value not null
+                    ORDER by person.rowid ASC;
+                """
                 )
-        except sqlite3.OperationalError as exc:
-            self.log.info("Error while reading the contact table: %s", exc)
+            except sqlite3.OperationalError as e:
+                self.log.info("Error while reading the contact table: %s", e)
+                return None
+            names = [description[0] for description in cur.description]
+
+            for row in cur:
+                new_contact = {}
+                for index, value in enumerate(row):
+                    new_contact[names[index]] = value
+
+                self.results.append(new_contact)
         finally:
             cur.close()
             conn.close()

src/mvt/ios/modules/mixed/interactionc.py

@@ -14,7 +14,7 @@ from mvt.common.module_types import (
     ModuleResults,
     ModuleSerializedResult,
 )
-from mvt.common.utils import convert_mactime_to_iso, sanitize_json_data
+from mvt.common.utils import convert_mactime_to_iso
 
 from ..base import IOSExtraction
 from .whatsapp_contacts import WhatsappContacts
@@ -547,22 +547,4 @@ class InteractionC(IOSExtraction):
 
         self._postprocess_results()
 
-        # Preserve every column from the underlying interaction row. The
-        # joined compatibility queries above intentionally expose a stable
-        # result schema but otherwise discard useful forensic fields.
-        conn = self._open_sqlite_db(self.file_path)
-        cur = conn.cursor()
-        try:
-            cur.execute("SELECT * FROM ZINTERACTIONS;")
-            names = [description[0] for description in cur.description]
-            interactions = {}
-            for row in cur:
-                raw = sanitize_json_data(dict(zip(names, row)))
-                interactions[raw.get("Z_PK")] = raw
-        finally:
-            cur.close()
-            conn.close()
-        for result in self.results:
-            result["interaction"] = interactions.get(result.get("table_id"), {})
-
         self.log.info("Extracted a total of %d InteractionC events", len(self.results))

src/mvt/ios/modules/mixed/net_datausage.py

@@ -7,8 +7,6 @@ import logging
 from typing import Optional
 
 from mvt.common.module_types import ModuleResults
-from mvt.common.utils import convert_mactime_to_iso, sanitize_json_data
-
 from ..net_base import NetBase
 
 DATAUSAGE_BACKUP_IDS = [
@@ -44,77 +42,6 @@ class Datausage(NetBase):
             results=results,
         )
 
-    def _extract_net_data(self):
-        assert self.file_path is not None
-        conn = self._open_sqlite_db(self.file_path)
-        cur = conn.cursor()
-        try:
-            cur.execute("PRAGMA table_info(ZPROCESS);")
-            process_columns = [row[1] for row in cur]
-            cur.execute("PRAGMA table_info(ZLIVEUSAGE);")
-            live_columns = [row[1] for row in cur]
-            selected = [
-                f'p."{column}" AS "process_{column}"' for column in process_columns
-            ] + [f'l."{column}" AS "live_{column}"' for column in live_columns]
-            cur.execute(
-                f"SELECT {', '.join(selected)} FROM ZLIVEUSAGE l "
-                "LEFT JOIN ZPROCESS p ON l.ZHASPROCESS = p.Z_PK;"
-            )
-            names = [description[0] for description in cur.description]
-            rows = [dict(zip(names, row)) for row in cur]
-
-            process_only = [
-                f'p."{column}" AS "process_{column}"' for column in process_columns
-            ] + [f'NULL AS "live_{column}"' for column in live_columns]
-            cur.execute(
-                f"SELECT {', '.join(process_only)} FROM ZPROCESS p "
-                "WHERE NOT EXISTS ("
-                "SELECT 1 FROM ZLIVEUSAGE l WHERE l.ZHASPROCESS = p.Z_PK);"
-            )
-            names = [description[0] for description in cur.description]
-            rows.extend(dict(zip(names, row)) for row in cur)
-        finally:
-            cur.close()
-            conn.close()
-
-        for raw_row in rows:
-            first_timestamp = raw_row.get("process_ZFIRSTTIMESTAMP")
-            process_timestamp = raw_row.get("process_ZTIMESTAMP")
-            live_timestamp = raw_row.get("live_ZTIMESTAMP")
-            first_isodate = (
-                convert_mactime_to_iso(first_timestamp)
-                if first_timestamp
-                else first_timestamp
-            )
-            isodate = (
-                convert_mactime_to_iso(process_timestamp)
-                if process_timestamp
-                else process_timestamp
-            )
-            live_isodate = (
-                convert_mactime_to_iso(live_timestamp)
-                if live_timestamp
-                else first_isodate
-            )
-            self.results.append(
-                {
-                    "first_isodate": first_isodate,
-                    "isodate": isodate,
-                    "proc_name": raw_row.get("process_ZPROCNAME"),
-                    "bundle_id": raw_row.get("process_ZBUNDLENAME"),
-                    "proc_id": raw_row.get("process_Z_PK"),
-                    "wifi_in": raw_row.get("live_ZWIFIIN"),
-                    "wifi_out": raw_row.get("live_ZWIFIOUT"),
-                    "wwan_in": raw_row.get("live_ZWWANIN"),
-                    "wwan_out": raw_row.get("live_ZWWANOUT"),
-                    "live_id": raw_row.get("live_Z_PK"),
-                    "live_proc_id": raw_row.get("live_ZHASPROCESS"),
-                    "live_isodate": live_isodate,
-                    "record": sanitize_json_data(raw_row),
-                }
-            )
-        self.log.info("Extracted information on %d processes", len(self.results))
-
     def run(self) -> None:
         self._find_ios_database(
             backup_ids=DATAUSAGE_BACKUP_IDS, root_paths=DATAUSAGE_ROOT_PATHS

src/mvt/ios/modules/mixed/safari_browserstate.py

@@ -15,11 +15,7 @@ from mvt.common.module_types import (
     ModuleResults,
     ModuleSerializedResult,
 )
-from mvt.common.utils import (
-    convert_mactime_to_iso,
-    keys_bytes_to_string,
-    sanitize_json_data,
-)
+from mvt.common.utils import convert_mactime_to_iso, keys_bytes_to_string
 
 from ..base import IOSExtraction
 
@@ -98,76 +94,85 @@ class SafariBrowserState(IOSExtraction):
     def _process_browser_state_db(self, db_path):
         self._recover_sqlite_db_if_needed(db_path)
         conn = self._open_sqlite_db(db_path)
+
         cur = conn.cursor()
         try:
             try:
                 cur.execute(
                     """
-                    SELECT tabs.*, tab_sessions.session_data AS session_data
+                    SELECT
+                        tabs.title,
+                        tabs.url,
+                        tabs.user_visible_url,
+                        tabs.last_viewed_time,
+                        tab_sessions.session_data
                     FROM tabs
-                    LEFT JOIN tab_sessions ON tabs.uuid = tab_sessions.tab_uuid
+                    JOIN tab_sessions ON tabs.uuid = tab_sessions.tab_uuid
                     ORDER BY tabs.last_viewed_time;
-                    """
+                """
                 )
-                names = [description[0] for description in cur.description]
             except sqlite3.OperationalError:
-                # Older databases store session_data directly in tabs.
+                # Old version iOS <12 likely
                 try:
-                    cur.execute("SELECT * FROM tabs ORDER BY last_viewed_time;")
-                    names = [description[0] for description in cur.description]
-                except sqlite3.OperationalError as exc:
-                    self.log.error("Error executing query: %s", exc)
+                    cur.execute(
+                        """
+                        SELECT
+                            title, url, user_visible_url, last_viewed_time, session_data
+                    FROM tabs
+                    ORDER BY last_viewed_time;
+                """
+                    )
+                except sqlite3.OperationalError as e:
+                    self.log.error(f"Error executing query: {e}")
                     return
 
             for row in cur:
-                raw = dict(zip(names, row))
                 session_entries = []
-                session_blob = raw.get("session_data")
-                if session_blob:
+
+                if row[4]:
                     # Skip a 4 byte header before the plist content.
-                    session_plist = session_blob[4:]
+                    session_plist = row[4][4:]
                     session_data = {}
                     try:
                         session_data = plistlib.load(io.BytesIO(session_plist))
                         session_data = keys_bytes_to_string(session_data)
-                    except (plistlib.InvalidFileException, ValueError, TypeError):
+                    except plistlib.InvalidFileException:
                         pass
 
-                    for session_entry in session_data.get("SessionHistory", {}).get(
-                        "SessionHistoryEntries", []
-                    ):
-                        self._session_history_count += 1
-                        entry_data = session_entry.get("SessionHistoryEntryData")
-                        session_entries.append(
-                            {
-                                "entry_title": session_entry.get(
-                                    "SessionHistoryEntryOriginalURL"
-                                ),
-                                "entry_url": session_entry.get(
-                                    "SessionHistoryEntryURL"
-                                ),
-                                "data_length": (
-                                    len(entry_data) if entry_data is not None else 0
-                                ),
-                            }
-                        )
+                    if "SessionHistoryEntries" in session_data.get("SessionHistory", {}):
+                        for session_entry in session_data["SessionHistory"].get(
+                            "SessionHistoryEntries"
+                        ):
+                            self._session_history_count += 1
+
+                            data_length = 0
+                            if "SessionHistoryEntryData" in session_entry:
+                                data_length = len(
+                                    session_entry.get("SessionHistoryEntryData")
+                                )
+
+                            session_entries.append(
+                                {
+                                    "entry_title": session_entry.get(
+                                        "SessionHistoryEntryOriginalURL"
+                                    ),
+                                    "entry_url": session_entry.get(
+                                        "SessionHistoryEntryURL"
+                                    ),
+                                    "data_length": data_length,
+                                }
+                            )
 
-                last_viewed = raw.get("last_viewed_time")
                 self.results.append(
                     {
-                        "tab_title": raw.get("title"),
-                        "tab_url": raw.get("url"),
-                        "tab_visible_url": raw.get("user_visible_url"),
-                        "last_viewed_timestamp": (
-                            convert_mactime_to_iso(last_viewed)
-                            if last_viewed is not None
-                            else ""
-                        ),
+                        "tab_title": row[0],
+                        "tab_url": row[1],
+                        "tab_visible_url": row[2],
+                        "last_viewed_timestamp": convert_mactime_to_iso(row[3]),
                         "session_data": session_entries,
                         "safari_browser_state_db": os.path.relpath(
                             db_path, self.target_path
                         ),
-                        "tab": sanitize_json_data(raw),
                     }
                 )
         finally:

src/mvt/ios/modules/mixed/safari_history.py

@@ -13,11 +13,7 @@ from mvt.common.module_types import (
     ModuleSerializedResult,
 )
 from mvt.common.url import URL
-from mvt.common.utils import (
-    convert_mactime_to_datetime,
-    convert_mactime_to_iso,
-    sanitize_json_data,
-)
+from mvt.common.utils import convert_mactime_to_datetime, convert_mactime_to_iso
 
 from ..base import IOSExtraction
 
@@ -137,41 +133,39 @@ class SafariHistory(IOSExtraction):
         self._recover_sqlite_db_if_needed(history_path)
         conn = self._open_sqlite_db(history_path)
         cur = conn.cursor()
-        try:
-            cur.execute("PRAGMA table_info(history_items);")
-            item_columns = [row[1] for row in cur]
-            cur.execute("PRAGMA table_info(history_visits);")
-            visit_columns = [row[1] for row in cur]
-            selected = [
-                f'i."{column}" AS "item_{column}"' for column in item_columns
-            ] + [f'v."{column}" AS "visit_{column}"' for column in visit_columns]
-            cur.execute(
-                f"SELECT {', '.join(selected)} FROM history_items i "
-                "JOIN history_visits v ON v.history_item = i.id "
-                "ORDER BY v.visit_time;"
+        cur.execute(
+            """
+            SELECT
+                history_items.id,
+                history_items.url,
+                history_visits.id,
+                history_visits.visit_time,
+                history_visits.redirect_source,
+                history_visits.redirect_destination
+            FROM history_items
+            JOIN history_visits ON history_visits.history_item = history_items.id
+            ORDER BY history_visits.visit_time;
+        """
+        )
+
+        for row in cur:
+            self.results.append(
+                {
+                    "id": row[0],
+                    "url": row[1],
+                    "visit_id": row[2],
+                    "timestamp": row[3],
+                    "isodate": convert_mactime_to_iso(row[3]),
+                    "redirect_source": row[4],
+                    "redirect_destination": row[5],
+                    "safari_history_db": os.path.relpath(
+                        history_path, self.target_path
+                    ),
+                }
             )
-            names = [description[0] for description in cur.description]
-            for row in cur:
-                raw = sanitize_json_data(dict(zip(names, row)))
-                timestamp = raw.get("visit_visit_time")
-                self.results.append(
-                    {
-                        "id": raw.get("item_id"),
-                        "url": raw.get("item_url"),
-                        "visit_id": raw.get("visit_id"),
-                        "timestamp": timestamp,
-                        "isodate": convert_mactime_to_iso(timestamp),
-                        "redirect_source": raw.get("visit_redirect_source"),
-                        "redirect_destination": raw.get("visit_redirect_destination"),
-                        "safari_history_db": os.path.relpath(
-                            history_path, self.target_path
-                        ),
-                        "record": raw,
-                    }
-                )
-        finally:
-            cur.close()
-            conn.close()
+
+        cur.close()
+        conn.close()
 
     def run(self) -> None:
         if self.is_backup:

src/mvt/ios/modules/mixed/tcc.py

@@ -12,7 +12,7 @@ from mvt.common.module_types import (
     ModuleResults,
     ModuleSerializedResult,
 )
-from mvt.common.utils import convert_unix_to_iso, sanitize_json_data
+from mvt.common.utils import convert_unix_to_iso
 
 from ..base import IOSExtraction
 
@@ -103,49 +103,126 @@ class TCC(IOSExtraction):
     def process_db(self, file_path):
         conn = self._open_sqlite_db(file_path)
         cur = conn.cursor()
+        db_version = "v3"
         try:
+            cur.execute(
+                """SELECT
+                service, client, client_type, auth_value,
+                auth_reason, last_modified
+            FROM access;"""
+            )
+        except sqlite3.OperationalError:
+            # v2 version
             try:
-                cur.execute("SELECT * FROM access;")
-            except sqlite3.OperationalError as exc:
-                self.log.error("Error parsing TCC database: %s", exc)
-                return
-
-            names = [description[0] for description in cur.description]
-            for row in cur:
-                record = sanitize_json_data(dict(zip(names, row)))
-                client_type = record.get("client_type")
-                record["client_type_value"] = client_type
-                record["client_type"] = (
-                    "bundle_id" if client_type == 0 else "absolute_path"
+                cur.execute(
+                    """SELECT
+                    service, client, client_type, allowed,
+                    prompt_count, last_modified
+                    FROM access;"""
                 )
-
-                if "auth_value" in record:
-                    auth_value = record["auth_value"]
-                    record["auth_value_value"] = auth_value
-                    record["auth_value"] = AUTH_VALUES.get(auth_value, "unknown")
-                    auth_reason = record.get("auth_reason")
-                    record["auth_reason_desc"] = AUTH_REASONS.get(
-                        auth_reason, "unknown"
+                db_version = "v2"
+            except sqlite3.OperationalError:
+                try:
+                    cur.execute(
+                        """SELECT
+                        service, client, client_type, allowed,
+                        prompt_count
+                        FROM access;"""
                     )
-                elif "allowed" in record:
-                    allowed = record["allowed"]
-                    record["allowed_value"] = AUTH_VALUE_OLD.get(allowed, "unknown")
+                    db_version = "v1"
+                except sqlite3.OperationalError as e:
+                    self.log.error(f"Error parsing TCC database: {e}")
 
-                if record.get("last_modified") is not None:
-                    record["last_modified_value"] = record["last_modified"]
-                    record["last_modified"] = convert_unix_to_iso(
-                        record["last_modified"]
+        for row in cur:
+            service = row[0]
+            client = row[1]
+            client_type = row[2]
+            client_type_desc = "bundle_id" if client_type == 0 else "absolute_path"
+            if db_version == "v3":
+                auth_value = row[3]
+                auth_value_desc = AUTH_VALUES.get(auth_value, "")
+                auth_reason = row[4]
+                auth_reason_desc = AUTH_REASONS.get(auth_reason, "unknown")
+                last_modified = convert_unix_to_iso(row[5])
+
+                if service in ["kTCCServiceMicrophone", "kTCCServiceCamera"]:
+                    device = (
+                        "microphone" if service == "kTCCServiceMicrophone" else "camera"
                     )
-                if record.get("last_reminded") is not None:
-                    record["last_reminded_value"] = record["last_reminded"]
-                    record["last_reminded"] = convert_unix_to_iso(
-                        record["last_reminded"]
+                    self.log.info(
+                        'Found client "%s" with access %s to %s on %s by %s',
+                        client,
+                        auth_value_desc,
+                        device,
+                        last_modified,
+                        auth_reason_desc,
                     )
 
-                self.results.append(record)
-        finally:
-            cur.close()
-            conn.close()
+                self.results.append(
+                    {
+                        "service": service,
+                        "client": client,
+                        "client_type": client_type_desc,
+                        "auth_value": auth_value_desc,
+                        "auth_reason_desc": auth_reason_desc,
+                        "last_modified": last_modified,
+                    }
+                )
+            else:
+                allowed_value = row[3]
+                allowed_desc = AUTH_VALUE_OLD.get(allowed_value, "")
+                prompt_count = row[4]
+
+                if db_version == "v2":
+                    last_modified = convert_unix_to_iso(row[5])
+                    if service in ["kTCCServiceMicrophone", "kTCCServiceCamera"]:
+                        device = "camera"
+                        if service == "kTCCServiceMicrophone":
+                            device = "microphone"
+
+                        self.log.info(
+                            'Found client "%s" with access %s to %s at %s',
+                            client,
+                            allowed_desc,
+                            device,
+                            last_modified,
+                        )
+
+                    self.results.append(
+                        {
+                            "service": service,
+                            "client": client,
+                            "client_type": client_type_desc,
+                            "allowed_value": allowed_desc,
+                            "prompt_count": prompt_count,
+                            "last_modified": last_modified,
+                        }
+                    )
+                else:
+                    if service in ["kTCCServiceMicrophone", "kTCCServiceCamera"]:
+                        device = "camera"
+                        if service == "kTCCServiceMicrophone":
+                            device = "microphone"
+
+                        self.log.info(
+                            'Found client "%s" with access %s to %s',
+                            client,
+                            allowed_desc,
+                            device,
+                        )
+
+                    self.results.append(
+                        {
+                            "service": service,
+                            "client": client,
+                            "client_type": client_type_desc,
+                            "allowed_value": allowed_desc,
+                            "prompt_count": prompt_count,
+                        }
+                    )
+
+        cur.close()
+        conn.close()
 
     def run(self) -> None:
         self._find_ios_database(backup_ids=TCC_BACKUP_IDS, root_paths=TCC_ROOT_PATHS)

src/mvt/ios/modules/mixed/webkit_resource_load_statistics.py

@@ -13,7 +13,7 @@ from mvt.common.module_types import (
     ModuleResults,
     ModuleSerializedResult,
 )
-from mvt.common.utils import convert_unix_to_iso, sanitize_json_data
+from mvt.common.utils import convert_unix_to_iso
 
 from ..base import IOSExtraction
 
@@ -85,45 +85,52 @@ class WebkitResourceLoadStatistics(IOSExtraction):
 
         try:
             try:
-                cur.execute("SELECT * FROM ObservedDomains;")
+                cur.execute("PRAGMA table_info(ObservedDomains);")
+                available_columns = {row[1] for row in cur}
+                required_columns = [
+                    "domainID",
+                    "registrableDomain",
+                    "lastSeen",
+                    "hadUserInteraction",
+                ]
+                if not set(required_columns).issubset(available_columns):
+                    return
+
+                optional_columns = [
+                    column
+                    for column in [
+                        "mostRecentUserInteractionTime",
+                        "mostRecentWebPushInteractionTime",
+                    ]
+                    if column in available_columns
+                ]
+                selected_columns = required_columns + optional_columns
+                cur.execute(
+                    f"SELECT {', '.join(selected_columns)} FROM ObservedDomains;"
+                )
             except sqlite3.OperationalError:
                 return
 
-            names = [description[0] for description in cur.description]
-            required_columns = {
-                "domainID",
-                "registrableDomain",
-                "lastSeen",
-                "hadUserInteraction",
-            }
-            if not required_columns.issubset(names):
-                return
             for row in cur:
-                raw = sanitize_json_data(dict(zip(names, row)))
-                last_seen = raw.get("lastSeen")
                 result = {
-                    "domain_id": raw.get("domainID"),
-                    "registrable_domain": raw.get("registrableDomain"),
-                    "last_seen": last_seen,
-                    "had_user_interaction": bool(raw.get("hadUserInteraction")),
-                    "last_seen_isodate": convert_unix_to_iso(last_seen),
+                    "domain_id": row[0],
+                    "registrable_domain": row[1],
+                    "last_seen": row[2],
+                    "had_user_interaction": bool(row[3]),
+                    "last_seen_isodate": convert_unix_to_iso(row[2]),
                     "domain": domain,
                     "path": path,
-                    "record": raw,
                 }
-                for column, field in (
-                    (
-                        "mostRecentUserInteractionTime",
-                        "most_recent_user_interaction_time",
-                    ),
-                    (
-                        "mostRecentWebPushInteractionTime",
-                        "most_recent_web_push_interaction_time",
-                    ),
-                ):
-                    if column not in raw:
-                        continue
-                    timestamp = raw[column]
+                for index, column in enumerate(optional_columns, start=4):
+                    field = {
+                        "mostRecentUserInteractionTime": (
+                            "most_recent_user_interaction_time"
+                        ),
+                        "mostRecentWebPushInteractionTime": (
+                            "most_recent_web_push_interaction_time"
+                        ),
+                    }[column]
+                    timestamp = row[index]
                     result[field] = timestamp
                     if timestamp is not None and timestamp >= 0:
                         result[f"{field}_isodate"] = convert_unix_to_iso(timestamp)

src/mvt/ios/modules/sysdiagnose/__init__.py

@@ -3,4 +3,9 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+from mvt.common.module import MVTModule
+
 from .base import SysdiagnoseExtraction
+from .sysdiagnose_info import SysdiagnoseInfo
+
+SYSDIAGNOSE_MODULES: list[type[MVTModule]] = [SysdiagnoseInfo]

src/mvt/ios/modules/sysdiagnose/sysdiagnose_info.py

@@ -0,0 +1,228 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2026 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import json
+import logging
+import os
+import plistlib
+import re
+import sqlite3
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from mvt.common.module_types import ModuleResults
+from mvt.common.utils import convert_datetime_to_iso
+from mvt.ios.versions import (
+    find_version_by_build,
+    get_device_desc_from_id,
+    is_ios_version_outdated,
+)
+
+from .base import SysdiagnoseExtraction
+
+# The fields dumpsys prints in the remotectl dump state and the mobile
+# activation request which are worth a log line of their own.
+LOGGED_FIELDS = (
+    "ProductName",
+    "ProductType",
+    "SerialNumber",
+    "OSVersion",
+    "RegionCode",
+    "IMEI",
+    "BuildVersion",
+)
+
+
+class SysdiagnoseInfo(SysdiagnoseExtraction):
+    """Extract details about the device and the sysdiagnose itself.
+
+    The fields come from four files of the archive: the remotectl dump state
+    (product type, OS version, serial number, region and the rest of its
+    Properties block), the mobile activation request (UDID, IMEI, MEID and the
+    OS build), the App Store daemon database (the Apple account name and email)
+    and sysdiagnose.log (the archive's original file name and creation time).
+
+    Newer iOS versions no longer include the App Store daemon database in a
+    sysdiagnose; it is still read for the analysis of older archives.
+    """
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[ModuleResults] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+        self.results: dict = results if results is not None else {}
+
+    def _copy_sqlite_db(self, file_path: str, directory: str) -> str:
+        """Copy a database and its WAL sidecars out of the archive.
+
+        A database dumped mid-transaction keeps its latest rows in the -wal
+        file next to it, which SQLite only reads when both sit in the same
+        directory under the same name.
+        """
+        available_files = self.tar_files if self.tar else self.files
+        for suffix in ("", "-wal", "-shm"):
+            if suffix and f"{file_path}{suffix}" not in available_files:
+                continue
+            copy_path = os.path.join(directory, f"{Path(file_path).name}{suffix}")
+            with open(copy_path, "wb") as handle:
+                handle.write(self._get_file_content(f"{file_path}{suffix}"))
+
+        return os.path.join(directory, Path(file_path).name)
+
+    def _process_appstored(self, file_path: str) -> None:
+        self.log.info("Found App Store daemon database at: %s", file_path)
+        with tempfile.TemporaryDirectory(prefix="mvt_sqlite_") as directory:
+            db_path = Path(self._copy_sqlite_db(file_path, directory)).resolve()
+            conn = sqlite3.connect(f"{db_path.as_uri()}?mode=ro", uri=True)
+            try:
+                self._read_appstored(conn)
+            finally:
+                conn.close()
+
+    def _read_appstored(self, conn: sqlite3.Connection) -> None:
+        cur = conn.cursor()
+        # The account name sits in an opaque structure of every asset row.
+        try:
+            rows = cur.execute("SELECT sinfs_data FROM asset;").fetchall()
+        except sqlite3.DatabaseError as exc:
+            self.log.debug("Unable to read the asset table: %s", exc)
+            rows = []
+
+        for (sinfs_data,) in rows:
+            try:
+                sinf = plistlib.loads(sinfs_data)[0]["sinf"]
+            except (plistlib.InvalidFileException, IndexError, KeyError, TypeError):
+                continue
+            match = re.search(rb"name(.*?)\x00", sinf)
+            if match:
+                self.results["Account Name"] = match.group(1).decode(
+                    "utf-8", errors="replace"
+                )
+                break
+
+        try:
+            row = cur.execute(
+                "SELECT store_account_name FROM job_software "
+                "WHERE store_account_name IS NOT NULL LIMIT 1;"
+            ).fetchone()
+        except sqlite3.DatabaseError as exc:
+            self.log.debug("Unable to read the job_software table: %s", exc)
+            return
+
+        if row:
+            self.results["Email Address"] = row[0]
+
+    def _process_activation_log(self, file_path: str) -> None:
+        self.log.info("Found mobile activation request at: %s", file_path)
+        content = self._get_file_content(file_path)
+        match = re.search(rb"BODY:\s+({.+?})\s", content, re.MULTILINE)
+        if not match:
+            return
+
+        try:
+            body = json.loads(match.group(1))
+        except json.JSONDecodeError as exc:
+            self.log.warning("Unable to parse the activation request body: %s", exc)
+            return
+
+        self.results.update(
+            {
+                "SerialNumber": body.get("serial-number"),
+                "ProductType": body.get("productType"),
+                "ProductName": body.get("productName"),
+                "IMEI": body.get("imei"),
+                "ProductVersion": body.get("os-version"),
+                "UniqueIdentifier": body.get("udid"),
+                "MEID": body.get("meid"),
+                "BuildVersion": body.get("os-build"),
+            }
+        )
+
+    def _process_dumpstate(self, file_path: str) -> None:
+        self.log.info("Found remotectl dump state at: %s", file_path)
+        content = self._get_file_content(file_path).decode("utf-8", errors="replace")
+        in_properties = False
+        for line in content.splitlines():
+            if not in_properties:
+                in_properties = line == "\tProperties: {"
+                continue
+
+            if line == "\t}":
+                break
+
+            key, separator, value = line.partition("=>")
+            if separator:
+                self.results[key.strip()] = value.strip()
+
+    def _process_sysdiagnose_log(self, file_path: str) -> None:
+        self.log.info("Found sysdiagnose.log at: %s", file_path)
+        content = self._get_file_content(file_path).decode("utf-8", errors="replace")
+        match = re.search(r"sysdiagnose_\S+?\.tar\.gz", content)
+        if not match:
+            self.log.info("Could not find the original output path in sysdiagnose.log")
+            return
+
+        file_name = os.path.basename(match.group(0))
+        try:
+            created = datetime.strptime(
+                "_".join(file_name.split("_")[1:3]), "%Y.%m.%d_%H-%M-%S%z"
+            )
+        except ValueError:
+            self.log.warning("Unexpected sysdiagnose file name: %s", file_name)
+            return
+
+        self.results["OriginalFilename"] = file_name
+        self.results["CreatedTimestamp"] = convert_datetime_to_iso(created)
+
+    def run(self) -> None:
+        for file_path in self._get_files_by_pattern(
+            "*/logs/appinstallation/appstored.sqlitedb"
+        ):
+            self._process_appstored(file_path)
+
+        for file_path in self._get_files_by_pattern(
+            "*/logs/MobileActivation/collection_oob_request.txt"
+        ):
+            self._process_activation_log(file_path)
+
+        for file_path in self._get_files_by_pattern("*/remotectl_dumpstate.txt"):
+            self._process_dumpstate(file_path)
+
+        for file_path in self._get_files_by_pattern("*/sysdiagnose.log"):
+            self._process_sysdiagnose_log(file_path)
+
+        # The activation request names the product "iPhone OS"; the model
+        # description is what an analyst wants to read.
+        product_name = get_device_desc_from_id(self.results.get("ProductType", ""))
+        if product_name:
+            self.results["ProductName"] = product_name
+
+        for field in LOGGED_FIELDS:
+            if field not in self.results:
+                continue
+            value = self.results[field]
+            if field == "BuildVersion" and value:
+                self.log.info("%s: %s - %s", field, value, find_version_by_build(value))
+            else:
+                self.log.info("%s: %s", field, value)
+
+        if self.results.get("BuildVersion"):
+            is_ios_version_outdated(self.results["BuildVersion"], self.log)

tests/common/test_command_modules.py

@@ -12,13 +12,16 @@ from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES
 from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES
 from mvt.ios.modules.fs import FS_MODULES
 from mvt.ios.modules.mixed import MIXED_MODULES
+from mvt.ios.modules.sysdiagnose import SYSDIAGNOSE_MODULES
 
 
 def test_the_check_iocs_lists_are_the_families_of_their_platform():
     # The CLI reads these same lists, so nothing composing one elsewhere can
     # drift from what the command runs. This pins what the lists are composed
     # of.
-    assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES
+    assert IOS_CHECK_IOCS_MODULES == (
+        IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES + SYSDIAGNOSE_MODULES
+    )
     assert ANDROID_CHECK_IOCS_MODULES == (
         ANDROID_BACKUP_MODULES
         + BUGREPORT_MODULES

tests/common/test_utils.py

@@ -18,7 +18,6 @@ from mvt.common.utils import (
     generate_hashes_from_path,
     get_sha256_from_file_path,
     init_logging,
-    sanitize_json_data,
     set_verbose_logging,
 )
 
@@ -109,18 +108,6 @@ class TestCustomJSONEncoder:
         )
 
 
-def test_sanitize_json_data_preserves_nested_binary_and_dates():
-    value = {
-        "blob": b"\x00\xff",
-        "nested": [datetime(2023, 11, 13, 12, 21, 49, 727467)],
-    }
-
-    assert sanitize_json_data(value) == {
-        "blob": "AP8=",
-        "nested": ["2023-11-13 12:21:49.727467"],
-    }
-
-
 class TestInitLogging:
     def test__init_logging_is_idempotent(self):
         # Loaded module packages may import an MVT CLI module, which calls

tests/ios_backup/test_calendar.py

@@ -21,8 +21,6 @@ class TestCalendarModule:
         assert len(m.timeline) == 4
         assert len(m.alertstore.alerts) == 0
         assert m.results[0]["summary"] == "Super interesting meeting"
-        assert m.results[0]["record"]["calendar_app_link"] is None
-        assert "participant_proposed_start_date" in m.results[0]["record"]
 
     def test_calendar_with_explicit_file_path(self):
         backup_path = get_ios_backup_folder()

tests/ios_backup/test_calls.py

@@ -1,38 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2026 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import sqlite3
-
-from mvt.common.module import run_module
-from mvt.ios.modules.mixed.calls import Calls
-
-
-def test_calls_preserves_complete_source_record(tmp_path):
-    db_path = tmp_path / "CallHistory.storedata"
-    conn = sqlite3.connect(db_path)
-    conn.execute(
-        """
-        CREATE TABLE ZCALLRECORD (
-            ZDATE REAL, ZDURATION REAL, ZLOCATION TEXT, ZADDRESS BLOB,
-            ZSERVICE_PROVIDER TEXT, ZORIGINATED INTEGER, ZANSWERED INTEGER,
-            ZMETADATA BLOB
-        );
-        """
-    )
-    conn.execute(
-        "INSERT INTO ZCALLRECORD VALUES (?, ?, ?, ?, ?, ?, ?, ?);",
-        (700000000, 42, "Berlin", b"+491234", "carrier", 1, 0, b"\x00\xff"),
-    )
-    conn.commit()
-    conn.close()
-
-    module = Calls(file_path=str(db_path))
-    run_module(module)
-
-    assert len(module.results) == 1
-    assert module.results[0]["number"] == "+491234"
-    assert module.results[0]["call"]["ZORIGINATED"] == 1
-    assert module.results[0]["call"]["ZANSWERED"] == 0
-    assert module.results[0]["call"]["ZMETADATA"] == "AP8="

tests/ios_backup/test_contacts.py

@@ -1,42 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2026 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import sqlite3
-
-from mvt.common.module import run_module
-from mvt.ios.modules.mixed.contacts import Contacts
-
-
-def test_contacts_preserves_complete_rows_and_people_without_values(tmp_path):
-    db_path = tmp_path / "AddressBook.sqlitedb"
-    conn = sqlite3.connect(db_path)
-    conn.executescript(
-        """
-        CREATE TABLE ABPerson (
-            ROWID INTEGER PRIMARY KEY, First TEXT, Middle TEXT, Last TEXT,
-            Organization TEXT, Note TEXT
-        );
-        CREATE TABLE ABMultiValue (
-            ROWID INTEGER PRIMARY KEY, record_id INTEGER, value TEXT,
-            label INTEGER
-        );
-        INSERT INTO ABPerson VALUES (1, 'Alice', NULL, 'Example', NULL, 'note');
-        INSERT INTO ABPerson VALUES (2, 'Bob', NULL, 'NoValue', NULL, 'retained');
-        INSERT INTO ABMultiValue VALUES (10, 1, '+491234', 3);
-        """
-    )
-    conn.close()
-
-    module = Contacts(file_path=str(db_path))
-    run_module(module)
-
-    assert len(module.results) == 2
-    alice = next(result for result in module.results if result["first"] == "Alice")
-    bob = next(result for result in module.results if result["first"] == "Bob")
-    assert alice["contact"]["person_Note"] == "note"
-    assert alice["contact"]["multivalue_label"] == 3
-    assert alice["First"] == "Alice"
-    assert bob["value"] is None
-    assert bob["contact"]["person_Note"] == "retained"

tests/ios_backup/test_datausage.py

@@ -20,8 +20,6 @@ class TestDatausageModule:
         assert m.results[0]["isodate"][0:19] == "2019-08-27 15:08:09"
         assert len(m.results) == 42
         assert len(m.timeline) == 60
-        assert "live_ZBILLCYCLEEND" in m.results[0]["record"]
-        assert "process_Z_ENT" in m.results[0]["record"]
         assert (
             len(m.alertstore.alerts) == 1
         )  # We now have a detection for missing processes.

tests/ios_backup/test_interactionc.py

@@ -20,10 +20,6 @@ class TestInteractionCModule:
         run_module(m)
 
         assert len(m.results) == 3
-        assert all(
-            result["interaction"]["Z_PK"] == result["table_id"] for result in m.results
-        )
-        assert all("ZMECHANISM" in result["interaction"] for result in m.results)
 
         incoming = next(
             r for r in m.results if r["sender_identifier"] == "100000000000001@lid"

tests/ios_backup/test_safari_browserstate.py

@@ -5,7 +5,6 @@
 
 import logging
 import shutil
-import sqlite3
 
 import pytest
 
@@ -53,7 +52,6 @@ class TestSafariBrowserStateModule:
         assert len(m.results) == 1
         assert len(m.timeline) == 1
         assert len(m.alertstore.alerts) == 0
-        assert "tab" in m.results[0]
 
     def test_parsing_backup_with_profile(self, backup_with_safari_profile):
         m = SafariBrowserState(target_path=backup_with_safari_profile)
@@ -76,30 +74,3 @@ class TestSafariBrowserStateModule:
         assert len(m.alertstore.alerts) == 1
         assert len(m.results) == 1
         assert m.results[0]["tab_url"] == "https://en.wikipedia.org/wiki/NSO_Group"
-
-    def test_tabs_without_session_rows_are_not_dropped(self, tmp_path):
-        db_path = tmp_path / "BrowserState.db"
-        conn = sqlite3.connect(db_path)
-        conn.executescript(
-            """
-            CREATE TABLE tabs (
-                uuid TEXT, title TEXT, url TEXT, user_visible_url TEXT,
-                last_viewed_time REAL, private_browsing INTEGER
-            );
-            CREATE TABLE tab_sessions (tab_uuid TEXT, session_data BLOB);
-            INSERT INTO tabs VALUES (
-                'tab-1', 'Example', 'https://example.test',
-                'https://example.test', 700000000, 1
-            );
-            """
-        )
-        conn.close()
-        module = SafariBrowserState(target_path=str(tmp_path))
-
-        module._process_browser_state_db(str(db_path))
-
-        assert len(module.results) == 1
-        assert module.results[0]["tab_url"] == "https://example.test"
-        assert module.results[0]["session_data"] == []
-        assert module.results[0]["tab"]["private_browsing"] == 1
-        assert module.results[0]["tab"]["session_data"] is None

tests/ios_backup/test_safari_history.py

@@ -105,7 +105,6 @@ class TestSafariHistoryModule:
         assert len(m.results) == 2
         assert {result["url"] for result in m.results} == {DEFAULT_URL, PROFILE_URL}
         assert len({result["safari_history_db"] for result in m.results}) == 2
-        assert all("visit_history_item" in result["record"] for result in m.results)
 
     def test_parsing_fs_dump_with_profile(self, fs_dump_with_safari_profile):
         m = SafariHistory(target_path=fs_dump_with_safari_profile)

tests/ios_backup/test_tcc.py

@@ -22,9 +22,6 @@ class TestTCCModule:
         assert m.results[0]["service"] == "kTCCServiceUbiquity"
         assert m.results[0]["client"] == "com.apple.Preferences"
         assert m.results[0]["auth_value"] == "allowed"
-        assert m.results[0]["auth_value_value"] == 2
-        assert "policy_id" in m.results[0]
-        assert "csreq" in m.results[0]
 
     def test_tcc_detection(self, indicator_file):
         m = TCC(target_path=get_ios_backup_folder())

tests/ios_backup/test_webkit_resource_load_statistics.py

@@ -34,7 +34,6 @@ class TestWebkitResourceLoadStatisticsModule:
             "most_recent_web_push_interaction_time" not in result
             for result in m.results
         )
-        assert all("record" in result for result in m.results)
 
     def test_webkit_full_timestamp_schema(self, tmp_path):
         db_path = tmp_path / "observations.db"
@@ -69,4 +68,3 @@ class TestWebkitResourceLoadStatisticsModule:
         assert "most_recent_user_interaction_time_isodate" in result
         assert result["most_recent_web_push_interaction_time"] == -1.0
         assert "most_recent_web_push_interaction_time_isodate" not in result
-        assert result["record"]["domainID"] == 1

tests/ios_sysdiagnose/__init__.py

@@ -0,0 +1,4 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2026 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/

tests/ios_sysdiagnose/test_sysdiagnose_info.py

@@ -0,0 +1,161 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2026 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import json
+import plistlib
+import sqlite3
+import tarfile
+
+from mvt.common.module import run_module
+from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
+from mvt.ios.modules.sysdiagnose.sysdiagnose_info import SysdiagnoseInfo
+from mvt.ios.versions import get_device_desc_from_id
+
+# The name sysdiagnose gives its archive: the time it ran, then the OS and build.
+ARCHIVE_NAME = "sysdiagnose_2024.01.02_03-04-05+0200_iPhone-OS_iPhone_21C62"
+
+DUMPSTATE = (
+    "Found device: ...\n"
+    "\tProperties: {\n"
+    "\t\tProductType => iPhone12,1\n"
+    "\t\tOSVersion => 17.2\n"
+    "\t\tSerialNumber => C0FFEE000000\n"
+    "\t\tRegionCode => LL\n"
+    "\t}\n"
+    "\tServices: {\n"
+    "\t\tcom.apple.example => ignored\n"
+    "\t}\n"
+)
+
+ACTIVATION_BODY = {
+    "serial-number": "C0FFEE000000",
+    "productType": "iPhone12,1",
+    "productName": "iPhone OS",
+    "imei": "000000000000000",
+    "os-version": "17.2",
+    "os-build": "21C62",
+    "udid": "00000000-0000000000000000",
+    "meid": "00000000000000",
+}
+
+
+def make_sysdiagnose(tmp_path, activation_body=None):
+    folder = tmp_path / ARCHIVE_NAME
+    folder.mkdir()
+    (folder / "sysdiagnose.log").write_text(
+        f"Output available at '/private/var/tmp/{ARCHIVE_NAME}.tar.gz'\n",
+        encoding="utf-8",
+    )
+    (folder / "remotectl_dumpstate.txt").write_text(DUMPSTATE, encoding="utf-8")
+
+    activation = folder / "logs" / "MobileActivation"
+    activation.mkdir(parents=True)
+    body = json.dumps(
+        activation_body if activation_body is not None else ACTIVATION_BODY
+    )
+    (activation / "collection_oob_request.txt").write_text(
+        f"HEADERS: {{}}\nBODY: {body}\nEND\n", encoding="utf-8"
+    )
+
+    appinstallation = folder / "logs" / "appinstallation"
+    appinstallation.mkdir(parents=True)
+    conn = sqlite3.connect(appinstallation / "appstored.sqlitedb")
+    conn.execute("CREATE TABLE asset (sinfs_data BLOB)")
+    conn.execute(
+        "INSERT INTO asset VALUES (?)",
+        (plistlib.dumps([{"sinf": b"\x00\x10nameExample Person\x00\x00rest"}]),),
+    )
+    conn.execute("CREATE TABLE job_software (store_account_name TEXT)")
+    conn.execute("INSERT INTO job_software VALUES (NULL)")
+    conn.execute("INSERT INTO job_software VALUES ('person@example.com')")
+    conn.commit()
+    conn.close()
+    return folder
+
+
+def run_command(target, results_path=None):
+    command = CmdIOSCheckSysdiagnose(target_path=str(target), results_path=results_path)
+    command.run()
+    (module,) = [m for m in command.executed if isinstance(m, SysdiagnoseInfo)]
+    return module
+
+
+def test_device_details_from_a_sysdiagnose_folder(tmp_path):
+    results_path = tmp_path / "results"
+    results_path.mkdir()
+    module = run_command(make_sysdiagnose(tmp_path), str(results_path))
+
+    assert module.results["SerialNumber"] == "C0FFEE000000"
+    assert module.results["ProductType"] == "iPhone12,1"
+    assert module.results["ProductName"] == get_device_desc_from_id("iPhone12,1")
+    assert module.results["ProductName"] != "iPhone OS"
+    assert module.results["OSVersion"] == "17.2"
+    assert module.results["BuildVersion"] == "21C62"
+    assert module.results["UniqueIdentifier"] == "00000000-0000000000000000"
+    assert module.results["RegionCode"] == "LL"
+    assert "com.apple.example" not in module.results
+    assert module.results["Account Name"] == "Example Person"
+    assert module.results["Email Address"] == "person@example.com"
+    assert module.results["OriginalFilename"] == f"{ARCHIVE_NAME}.tar.gz"
+    assert module.results["CreatedTimestamp"] == "2024-01-02 01:04:05.000000"
+    assert (results_path / "sysdiagnose_info.json").exists()
+
+
+def test_device_details_from_a_sysdiagnose_archive(tmp_path):
+    folder = make_sysdiagnose(tmp_path)
+    archive_path = tmp_path / f"{ARCHIVE_NAME}.tar.gz"
+    with tarfile.open(archive_path, "w:gz") as archive:
+        archive.add(folder, arcname=ARCHIVE_NAME)
+
+    module = run_command(archive_path)
+
+    assert module.results["SerialNumber"] == "C0FFEE000000"
+    assert module.results["Account Name"] == "Example Person"
+    assert module.results["OriginalFilename"] == f"{ARCHIVE_NAME}.tar.gz"
+
+
+def test_wal_sidecars_are_copied_beside_the_database(tmp_path):
+    folder = tmp_path / ARCHIVE_NAME
+    (folder / "logs").mkdir(parents=True)
+    (folder / "logs" / "db.sqlite").write_bytes(b"main")
+    (folder / "logs" / "db.sqlite-wal").write_bytes(b"wal")
+    module = SysdiagnoseInfo()
+    module.from_sysdiagnose_folder(
+        str(folder),
+        [f"{ARCHIVE_NAME}/logs/db.sqlite", f"{ARCHIVE_NAME}/logs/db.sqlite-wal"],
+    )
+    copies = tmp_path / "copies"
+    copies.mkdir()
+
+    db_path = module._copy_sqlite_db(f"{ARCHIVE_NAME}/logs/db.sqlite", str(copies))
+
+    assert db_path == str(copies / "db.sqlite")
+    assert (copies / "db.sqlite").read_bytes() == b"main"
+    assert (copies / "db.sqlite-wal").read_bytes() == b"wal"
+    assert not (copies / "db.sqlite-shm").exists()
+
+
+def test_a_sysdiagnose_without_the_files_yields_nothing(tmp_path):
+    folder = tmp_path / ARCHIVE_NAME
+    folder.mkdir()
+    (folder / "other.txt").write_text("nothing here", encoding="utf-8")
+
+    module = SysdiagnoseInfo()
+    module.from_sysdiagnose_folder(str(folder), [f"{ARCHIVE_NAME}/other.txt"])
+    run_module(module)
+
+    assert module.results == {}
+
+
+def test_a_malformed_activation_request_is_skipped(tmp_path):
+    folder = make_sysdiagnose(tmp_path)
+    (folder / "logs" / "MobileActivation" / "collection_oob_request.txt").write_text(
+        "BODY: {not json}\n", encoding="utf-8"
+    )
+
+    module = run_command(folder)
+
+    assert "IMEI" not in module.results
+    assert module.results["SerialNumber"] == "C0FFEE000000"

tests/test_check_ios_sysdiagnose.py

@@ -1,3 +1,5 @@
+import logging
+
 from click.testing import CliRunner
 
 from mvt.ios.cli import check_sysdiagnose
@@ -50,8 +52,13 @@ def test_check_sysdiagnose_runs_explicitly_scoped_custom_module(tmp_path):
     assert (output_path / "custom_sysdiagnose_module.json").exists()
 
 
-def test_check_sysdiagnose_requires_an_explicitly_scoped_module(tmp_path):
-    result = CliRunner().invoke(check_sysdiagnose, [str(_create_sysdiagnose_folder(tmp_path))])
+def test_check_sysdiagnose_warns_without_a_custom_module(tmp_path, caplog):
+    # The built-in SysdiagnoseInfo alone performs no check, so the run goes
+    # ahead but says so.
+    with caplog.at_level(logging.WARNING, logger="mvt"):
+        result = CliRunner().invoke(
+            check_sysdiagnose, [str(_create_sysdiagnose_folder(tmp_path))]
+        )
 
-    assert result.exit_code != 0
-    assert "No custom modules support mvt-ios check-sysdiagnose" in result.output
+    assert result.exit_code == 0
+    assert "No forensic sysdiagnose modules have been loaded" in caplog.text

tests/test_cmd_check_sysdiagnose.py

@@ -45,6 +45,11 @@ def _create_sysdiagnose_archive(tmp_path, folder):
     return archive_path
 
 
+def _test_module(command):
+    (module,) = [m for m in command.executed if isinstance(m, SysdiagnoseTestModule)]
+    return module
+
+
 def _run_command(path):
     command = CmdIOSCheckSysdiagnose(
         target_path=str(path), custom_modules=[SysdiagnoseTestModule]
@@ -56,10 +61,10 @@ def _run_command(path):
 def test_check_sysdiagnose_from_folder(tmp_path):
     command = _run_command(_create_sysdiagnose_folder(tmp_path))
 
-    assert command.executed[0].results == [
+    assert _test_module(command).results == [
         {"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
     ]
-    assert command.executed[0].ips_files == [
+    assert _test_module(command).ips_files == [
         {"file_path": str(tmp_path / "sysdiagnose" / "report.ips"), "bug_type": 210}
     ]
 
@@ -68,14 +73,12 @@ def test_check_sysdiagnose_from_archive_closes_archive(tmp_path):
     folder = _create_sysdiagnose_folder(tmp_path)
     command = _run_command(_create_sysdiagnose_archive(tmp_path, folder))
 
-    assert command.executed[0].results == [
+    assert _test_module(command).results == [
         {"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
     ]
-    assert command.executed[0].ips_files == [
+    assert _test_module(command).ips_files == [
         {
-            "file_path": str(
-                Path(command.extracted_sysdiagnose_path) / "report.ips"
-            ),
+            "file_path": str(Path(command.extracted_sysdiagnose_path) / "report.ips"),
             "bug_type": 210,
         }
     ]