diff --git a/docs/ios/records.md b/docs/ios/records.md index 856d0db9..6de89b20 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -204,6 +204,8 @@ This JSON file is created by mvt-ios' `Manifest` module. The module extracts rec If indicators are provided through the command-line, they are checked against the original relative path in case. In some cases, there might be records of files created containing a domain name in their name, for example in the case of browser cache folders. Any matches are stored in *manifest_detected.json*. +An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it. + --- ### `os_analytics_ad_daily.json` diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index cc74cbb6..e4ef17b5 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -147,6 +147,12 @@ class Manifest(IOSExtraction): ) names = [description[0] for description in cur.description] + # An incomplete backup still lists the files it failed to acquire in + # its manifest. Only regular files are stored in the backup folder, so + # directories and symlinks (flags 2 and 4) are not looked up. + stored_file_ids = self._get_stored_backup_file_ids() + missing_files = 0 + for file_entry in cur: file_data = {} for index, value in enumerate(file_entry): @@ -160,6 +166,18 @@ class Manifest(IOSExtraction): "created": "", } + if file_data["flags"] == 1 and file_data["fileID"] not in stored_file_ids: + # Without this, a module which found nothing for one of these + # files would look like a negative result rather than a gap in + # the acquisition. + cleaned_metadata["missing"] = True + missing_files += 1 + self.log.debug( + "File %s is listed in the manifest but was not found in the " + "backup folder", + cleaned_metadata["relative_path"], + ) + if file_data["file"]: try: file_plist = plistlib.load(io.BytesIO(file_data["file"])) @@ -197,3 +215,10 @@ class Manifest(IOSExtraction): conn.close() self.log.info("Extracted a total of %d file metadata items", len(self.results)) + + if missing_files: + self.log.info( + "Found %d files listed in the manifest but missing from the backup " + "folder. The backup might be incomplete.", + missing_files, + ) diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index a5c3354b..6cce6318 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -216,6 +216,41 @@ class IOSExtraction(MVTModule): return None + def _get_stored_backup_file_ids(self) -> set[str]: + """List the IDs of the files actually stored in the backup folder. + + A backup folder stores each file under a two character subfolder named + after the first two characters of its file ID. Walking the folders once + is cheaper than a filesystem lookup per file ID, and it keeps callers + which compare a whole manifest against the folder from paying a + `resolve()` for every entry. + + :returns: The file IDs found in the backup folder, empty if there is + no backup folder to walk. + """ + if not self.target_path: + return set() + + file_ids: set[str] = set() + try: + with os.scandir(self.target_path) as entries: + for entry in entries: + if not entry.is_dir(): + continue + with os.scandir(entry.path) as sub_entries: + for sub_entry in sub_entries: + if sub_entry.is_file(): + file_ids.add(sub_entry.name) + except OSError as exc: + self.log.debug( + "Unable to list the files stored in the backup folder %s: %s", + self.target_path, + exc, + ) + return set() + + return file_ids + def _get_fs_files_from_patterns(self, root_paths: list) -> Iterator[str]: if not self.target_path: return diff --git a/tests/ios_backup/test_manifest.py b/tests/ios_backup/test_manifest.py index aaa747d0..8601a667 100644 --- a/tests/ios_backup/test_manifest.py +++ b/tests/ios_backup/test_manifest.py @@ -5,8 +5,11 @@ import gc import logging +import shutil import warnings +import pytest + from mvt.common.indicators import Indicators from mvt.common.module import run_module from mvt.ios.modules.base import IOSExtraction @@ -14,6 +17,30 @@ from mvt.ios.modules.backup.manifest import Manifest from ..utils import get_ios_backup_folder +# fileID of HomeDomain::Library/SMS/sms.db in the test backup. It is one of the +# few files the test backup actually stores. +SMS_FILE_ID = "3d0d7e5fb2ce288813306e4d4636395e047a3d28" + + +@pytest.fixture +def backup_without_stored_files(tmp_path): + """A copy of the test backup with every stored file removed. + + The test backup only ships a handful of the files its manifest lists, so + dropping what it does store leaves a backup where every regular file the + manifest mentions is missing from the folder. + """ + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + + for file_id_folder in backup_path.iterdir(): + if not file_id_folder.is_dir(): + continue + for backup_file in file_id_folder.iterdir(): + backup_file.unlink() + + return str(backup_path) + class TestIOSExtraction: def test_get_backup_files_from_manifest_closes_connection(self): @@ -41,6 +68,41 @@ class TestManifestModule: assert len(m.timeline) == 5881 assert len(m.alertstore.alerts) == 0 + def test_manifest_flags_the_files_missing_from_an_incomplete_backup(self): + m = Manifest(target_path=get_ios_backup_folder()) + run_module(m) + + missing = [result for result in m.results if result.get("missing")] + # The test backup only stores a handful of the files its manifest + # lists, and every one of the rest is a regular file (flags 1). + assert len(missing) == 1079 + assert all(result["flags"] == 1 for result in missing) + + stored = [result for result in m.results if result["file_id"] == SMS_FILE_ID] + assert len(stored) == 1 + assert "missing" not in stored[0] + + def test_manifest_flags_every_stored_file_removed_from_the_backup_folder( + self, backup_without_stored_files + ): + m = Manifest(target_path=backup_without_stored_files) + run_module(m) + + missing = [result for result in m.results if result.get("missing")] + assert len(missing) == 1089 + + def test_manifest_flags_a_file_removed_from_the_backup_folder(self, tmp_path): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).unlink() + + m = Manifest(target_path=str(backup_path)) + run_module(m) + + removed = [result for result in m.results if result["file_id"] == SMS_FILE_ID] + assert len(removed) == 1 + assert removed[0]["missing"] is True + def test_detection(self, indicator_file): m = Manifest(target_path=get_ios_backup_folder()) ind = Indicators(log=logging.getLogger())