From f98f4f6cd21c8a8d8fdf56aea0121851d1bc2618 Mon Sep 17 00:00:00 2001 From: Yi-111-a Date: Tue, 29 Sep 2026 23:59:15 +0800 Subject: [PATCH 1/2] fix(manifest): flag backup files listed in the manifest but not stored An incomplete iTunes backup still lists in its Manifest.db the files it failed to acquire, so a module which found nothing for one of them looks exactly like a module which found nothing on a device that never had the artifact. The Manifest module now records a "missing" flag on those records and reports the total, so a gap in the acquisition is visible in manifest.json and in the command output. The stored file IDs are collected by walking the backup folder once, which keeps the check off the per-entry filesystem lookup path: on a 20k entry manifest the module runs in the same time as before the change. --- docs/ios/records.md | 2 + src/mvt/ios/modules/backup/manifest.py | 25 +++++++++++ src/mvt/ios/modules/base.py | 35 +++++++++++++++ tests/ios_backup/test_manifest.py | 62 ++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) diff --git a/docs/ios/records.md b/docs/ios/records.md index 856d0db9..6de89b20 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -204,6 +204,8 @@ This JSON file is created by mvt-ios' `Manifest` module. The module extracts rec If indicators are provided through the command-line, they are checked against the original relative path in case. In some cases, there might be records of files created containing a domain name in their name, for example in the case of browser cache folders. Any matches are stored in *manifest_detected.json*. +An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it. + --- ### `os_analytics_ad_daily.json` diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index cc74cbb6..e4ef17b5 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -147,6 +147,12 @@ class Manifest(IOSExtraction): ) names = [description[0] for description in cur.description] + # An incomplete backup still lists the files it failed to acquire in + # its manifest. Only regular files are stored in the backup folder, so + # directories and symlinks (flags 2 and 4) are not looked up. + stored_file_ids = self._get_stored_backup_file_ids() + missing_files = 0 + for file_entry in cur: file_data = {} for index, value in enumerate(file_entry): @@ -160,6 +166,18 @@ class Manifest(IOSExtraction): "created": "", } + if file_data["flags"] == 1 and file_data["fileID"] not in stored_file_ids: + # Without this, a module which found nothing for one of these + # files would look like a negative result rather than a gap in + # the acquisition. + cleaned_metadata["missing"] = True + missing_files += 1 + self.log.debug( + "File %s is listed in the manifest but was not found in the " + "backup folder", + cleaned_metadata["relative_path"], + ) + if file_data["file"]: try: file_plist = plistlib.load(io.BytesIO(file_data["file"])) @@ -197,3 +215,10 @@ class Manifest(IOSExtraction): conn.close() self.log.info("Extracted a total of %d file metadata items", len(self.results)) + + if missing_files: + self.log.info( + "Found %d files listed in the manifest but missing from the backup " + "folder. The backup might be incomplete.", + missing_files, + ) diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index a5c3354b..6cce6318 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -216,6 +216,41 @@ class IOSExtraction(MVTModule): return None + def _get_stored_backup_file_ids(self) -> set[str]: + """List the IDs of the files actually stored in the backup folder. + + A backup folder stores each file under a two character subfolder named + after the first two characters of its file ID. Walking the folders once + is cheaper than a filesystem lookup per file ID, and it keeps callers + which compare a whole manifest against the folder from paying a + `resolve()` for every entry. + + :returns: The file IDs found in the backup folder, empty if there is + no backup folder to walk. + """ + if not self.target_path: + return set() + + file_ids: set[str] = set() + try: + with os.scandir(self.target_path) as entries: + for entry in entries: + if not entry.is_dir(): + continue + with os.scandir(entry.path) as sub_entries: + for sub_entry in sub_entries: + if sub_entry.is_file(): + file_ids.add(sub_entry.name) + except OSError as exc: + self.log.debug( + "Unable to list the files stored in the backup folder %s: %s", + self.target_path, + exc, + ) + return set() + + return file_ids + def _get_fs_files_from_patterns(self, root_paths: list) -> Iterator[str]: if not self.target_path: return diff --git a/tests/ios_backup/test_manifest.py b/tests/ios_backup/test_manifest.py index aaa747d0..8601a667 100644 --- a/tests/ios_backup/test_manifest.py +++ b/tests/ios_backup/test_manifest.py @@ -5,8 +5,11 @@ import gc import logging +import shutil import warnings +import pytest + from mvt.common.indicators import Indicators from mvt.common.module import run_module from mvt.ios.modules.base import IOSExtraction @@ -14,6 +17,30 @@ from mvt.ios.modules.backup.manifest import Manifest from ..utils import get_ios_backup_folder +# fileID of HomeDomain::Library/SMS/sms.db in the test backup. It is one of the +# few files the test backup actually stores. +SMS_FILE_ID = "3d0d7e5fb2ce288813306e4d4636395e047a3d28" + + +@pytest.fixture +def backup_without_stored_files(tmp_path): + """A copy of the test backup with every stored file removed. + + The test backup only ships a handful of the files its manifest lists, so + dropping what it does store leaves a backup where every regular file the + manifest mentions is missing from the folder. + """ + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + + for file_id_folder in backup_path.iterdir(): + if not file_id_folder.is_dir(): + continue + for backup_file in file_id_folder.iterdir(): + backup_file.unlink() + + return str(backup_path) + class TestIOSExtraction: def test_get_backup_files_from_manifest_closes_connection(self): @@ -41,6 +68,41 @@ class TestManifestModule: assert len(m.timeline) == 5881 assert len(m.alertstore.alerts) == 0 + def test_manifest_flags_the_files_missing_from_an_incomplete_backup(self): + m = Manifest(target_path=get_ios_backup_folder()) + run_module(m) + + missing = [result for result in m.results if result.get("missing")] + # The test backup only stores a handful of the files its manifest + # lists, and every one of the rest is a regular file (flags 1). + assert len(missing) == 1079 + assert all(result["flags"] == 1 for result in missing) + + stored = [result for result in m.results if result["file_id"] == SMS_FILE_ID] + assert len(stored) == 1 + assert "missing" not in stored[0] + + def test_manifest_flags_every_stored_file_removed_from_the_backup_folder( + self, backup_without_stored_files + ): + m = Manifest(target_path=backup_without_stored_files) + run_module(m) + + missing = [result for result in m.results if result.get("missing")] + assert len(missing) == 1089 + + def test_manifest_flags_a_file_removed_from_the_backup_folder(self, tmp_path): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).unlink() + + m = Manifest(target_path=str(backup_path)) + run_module(m) + + removed = [result for result in m.results if result["file_id"] == SMS_FILE_ID] + assert len(removed) == 1 + assert removed[0]["missing"] is True + def test_detection(self, indicator_file): m = Manifest(target_path=get_ios_backup_folder()) ind = Indicators(log=logging.getLogger()) From a80b0fff2e3d0bed8d5fa16776d997bed5f36261 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Tue, 29 Sep 2026 18:40:02 +0200 Subject: [PATCH 2/2] fix(manifest): validate backup inventory before marking files missing --- docs/ios/records.md | 2 + src/mvt/ios/modules/backup/manifest.py | 6 +- src/mvt/ios/modules/base.py | 28 ++++++-- tests/ios_backup/test_manifest.py | 90 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 8 deletions(-) diff --git a/docs/ios/records.md b/docs/ios/records.md index 6de89b20..495db368 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -206,6 +206,8 @@ If indicators are provided through the command-line, they are checked against th An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it. +Files must be stored at their expected paths inside the backup folder. If the module cannot finish listing the backup files, it logs a warning and skips the missing-file check; the manifest metadata is still extracted. + --- ### `os_analytics_ad_daily.json` diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index e4ef17b5..d0a6680d 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -166,7 +166,11 @@ class Manifest(IOSExtraction): "created": "", } - if file_data["flags"] == 1 and file_data["fileID"] not in stored_file_ids: + if ( + stored_file_ids is not None + and file_data["flags"] == 1 + and file_data["fileID"] not in stored_file_ids + ): # Without this, a module which found nothing for one of these # files would look like a negative result rather than a gap in # the acquisition. diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index 6cce6318..e5281fda 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -216,7 +216,7 @@ class IOSExtraction(MVTModule): return None - def _get_stored_backup_file_ids(self) -> set[str]: + def _get_stored_backup_file_ids(self) -> Optional[set[str]]: """List the IDs of the files actually stored in the backup folder. A backup folder stores each file under a two character subfolder named @@ -225,29 +225,43 @@ class IOSExtraction(MVTModule): which compare a whole manifest against the folder from paying a `resolve()` for every entry. - :returns: The file IDs found in the backup folder, empty if there is - no backup folder to walk. + :returns: The file IDs found at their expected paths within the backup + folder, or None if the inventory could not be completed. """ if not self.target_path: - return set() + return None file_ids: set[str] = set() try: + backup_root = Path(self.target_path).resolve() with os.scandir(self.target_path) as entries: for entry in entries: + if len(entry.name) != 2 or any( + char not in "0123456789abcdef" for char in entry.name + ): + continue if not entry.is_dir(): continue + if not Path(entry.path).resolve().is_relative_to(backup_root): + continue with os.scandir(entry.path) as sub_entries: for sub_entry in sub_entries: + if sub_entry.name[:2] != entry.name: + continue + if sub_entry.is_symlink() and not Path( + sub_entry.path + ).resolve().is_relative_to(backup_root): + continue if sub_entry.is_file(): file_ids.add(sub_entry.name) except OSError as exc: - self.log.debug( - "Unable to list the files stored in the backup folder %s: %s", + self.log.warning( + "Unable to list the files stored in the backup folder %s: %s. " + "Skipping the missing-file check.", self.target_path, exc, ) - return set() + return None return file_ids diff --git a/tests/ios_backup/test_manifest.py b/tests/ios_backup/test_manifest.py index 8601a667..9a52ee78 100644 --- a/tests/ios_backup/test_manifest.py +++ b/tests/ios_backup/test_manifest.py @@ -5,8 +5,10 @@ import gc import logging +import os import shutil import warnings +from pathlib import Path import pytest @@ -43,6 +45,32 @@ def backup_without_stored_files(tmp_path): class TestIOSExtraction: + @pytest.mark.parametrize("link_directory", [False, True], ids=["file", "directory"]) + @pytest.mark.parametrize("inside_backup", [False, True], ids=["outside", "inside"]) + def test_stored_file_inventory_matches_symlink_resolution( + self, tmp_path, link_directory, inside_backup + ): + backup_path = tmp_path / "backup" + backup_path.mkdir() + destination = (backup_path if inside_backup else tmp_path) / "contents" + destination.mkdir() + (destination / SMS_FILE_ID).write_bytes(b"backup file") + bucket = backup_path / SMS_FILE_ID[:2] + try: + if link_directory: + bucket.symlink_to(destination, target_is_directory=True) + else: + bucket.mkdir() + (bucket / SMS_FILE_ID).symlink_to(destination / SMS_FILE_ID) + except OSError: + pytest.skip("creating symbolic links is not permitted on this system") + + m = IOSExtraction(target_path=str(backup_path)) + assert bool(m._get_backup_file_from_id(SMS_FILE_ID)) is inside_backup + assert m._get_stored_backup_file_ids() == ( + {SMS_FILE_ID} if inside_backup else set() + ) + def test_get_backup_files_from_manifest_closes_connection(self): m = IOSExtraction(target_path=get_ios_backup_folder()) @@ -103,6 +131,68 @@ class TestManifestModule: assert len(removed) == 1 assert removed[0]["missing"] is True + @pytest.mark.parametrize("failed_folder", ["", "3d"], ids=["root", "bucket"]) + def test_manifest_skips_missing_check_when_inventory_fails( + self, monkeypatch, caplog, failed_folder + ): + backup_path = Path(get_ios_backup_folder()) + failed_path = backup_path / failed_folder + scandir = os.scandir + + def fail_listing(path): + if Path(path) == failed_path: + raise PermissionError("cannot list backup folder") + return scandir(path) + + monkeypatch.setattr(os, "scandir", fail_listing) + m = Manifest(target_path=str(backup_path)) + with caplog.at_level(logging.INFO): + m.run() + + assert len(m.results) == 3721 + assert m._get_backup_file_from_id(SMS_FILE_ID) is not None + assert all("missing" not in result for result in m.results) + assert "Skipping the missing-file check" in caplog.text + assert "The backup might be incomplete" not in caplog.text + + def test_manifest_ignores_unreadable_unrelated_folders(self, tmp_path, monkeypatch): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + unrelated = backup_path / "notes" + unrelated.mkdir() + scandir = os.scandir + + def fail_listing(path): + if Path(path) == unrelated: + raise PermissionError("cannot list unrelated folder") + return scandir(path) + + monkeypatch.setattr(os, "scandir", fail_listing) + m = Manifest(target_path=str(backup_path)) + m.run() + + assert sum(bool(result.get("missing")) for result in m.results) == 1079 + stored = next( + result for result in m.results if result["file_id"] == SMS_FILE_ID + ) + assert "missing" not in stored + + @pytest.mark.parametrize("wrong_folder", ["ab", "notes"]) + def test_manifest_flags_files_in_the_wrong_folder(self, tmp_path, wrong_folder): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + destination = backup_path / wrong_folder + destination.mkdir(exist_ok=True) + (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).rename(destination / SMS_FILE_ID) + + m = Manifest(target_path=str(backup_path)) + m.run() + + assert m._get_backup_file_from_id(SMS_FILE_ID) is None + moved = next(result for result in m.results if result["file_id"] == SMS_FILE_ID) + assert moved["missing"] is True + assert sum(bool(result.get("missing")) for result in m.results) == 1080 + def test_detection(self, indicator_file): m = Manifest(target_path=get_ios_backup_folder()) ind = Indicators(log=logging.getLogger())