diff --git a/docs/ios/records.md b/docs/ios/records.md index 6de89b20..495db368 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -206,6 +206,8 @@ If indicators are provided through the command-line, they are checked against th An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it. +Files must be stored at their expected paths inside the backup folder. If the module cannot finish listing the backup files, it logs a warning and skips the missing-file check; the manifest metadata is still extracted. + --- ### `os_analytics_ad_daily.json` diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index e4ef17b5..d0a6680d 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -166,7 +166,11 @@ class Manifest(IOSExtraction): "created": "", } - if file_data["flags"] == 1 and file_data["fileID"] not in stored_file_ids: + if ( + stored_file_ids is not None + and file_data["flags"] == 1 + and file_data["fileID"] not in stored_file_ids + ): # Without this, a module which found nothing for one of these # files would look like a negative result rather than a gap in # the acquisition. diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index 6cce6318..e5281fda 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -216,7 +216,7 @@ class IOSExtraction(MVTModule): return None - def _get_stored_backup_file_ids(self) -> set[str]: + def _get_stored_backup_file_ids(self) -> Optional[set[str]]: """List the IDs of the files actually stored in the backup folder. A backup folder stores each file under a two character subfolder named @@ -225,29 +225,43 @@ class IOSExtraction(MVTModule): which compare a whole manifest against the folder from paying a `resolve()` for every entry. - :returns: The file IDs found in the backup folder, empty if there is - no backup folder to walk. + :returns: The file IDs found at their expected paths within the backup + folder, or None if the inventory could not be completed. """ if not self.target_path: - return set() + return None file_ids: set[str] = set() try: + backup_root = Path(self.target_path).resolve() with os.scandir(self.target_path) as entries: for entry in entries: + if len(entry.name) != 2 or any( + char not in "0123456789abcdef" for char in entry.name + ): + continue if not entry.is_dir(): continue + if not Path(entry.path).resolve().is_relative_to(backup_root): + continue with os.scandir(entry.path) as sub_entries: for sub_entry in sub_entries: + if sub_entry.name[:2] != entry.name: + continue + if sub_entry.is_symlink() and not Path( + sub_entry.path + ).resolve().is_relative_to(backup_root): + continue if sub_entry.is_file(): file_ids.add(sub_entry.name) except OSError as exc: - self.log.debug( - "Unable to list the files stored in the backup folder %s: %s", + self.log.warning( + "Unable to list the files stored in the backup folder %s: %s. " + "Skipping the missing-file check.", self.target_path, exc, ) - return set() + return None return file_ids diff --git a/tests/ios_backup/test_manifest.py b/tests/ios_backup/test_manifest.py index 8601a667..9a52ee78 100644 --- a/tests/ios_backup/test_manifest.py +++ b/tests/ios_backup/test_manifest.py @@ -5,8 +5,10 @@ import gc import logging +import os import shutil import warnings +from pathlib import Path import pytest @@ -43,6 +45,32 @@ def backup_without_stored_files(tmp_path): class TestIOSExtraction: + @pytest.mark.parametrize("link_directory", [False, True], ids=["file", "directory"]) + @pytest.mark.parametrize("inside_backup", [False, True], ids=["outside", "inside"]) + def test_stored_file_inventory_matches_symlink_resolution( + self, tmp_path, link_directory, inside_backup + ): + backup_path = tmp_path / "backup" + backup_path.mkdir() + destination = (backup_path if inside_backup else tmp_path) / "contents" + destination.mkdir() + (destination / SMS_FILE_ID).write_bytes(b"backup file") + bucket = backup_path / SMS_FILE_ID[:2] + try: + if link_directory: + bucket.symlink_to(destination, target_is_directory=True) + else: + bucket.mkdir() + (bucket / SMS_FILE_ID).symlink_to(destination / SMS_FILE_ID) + except OSError: + pytest.skip("creating symbolic links is not permitted on this system") + + m = IOSExtraction(target_path=str(backup_path)) + assert bool(m._get_backup_file_from_id(SMS_FILE_ID)) is inside_backup + assert m._get_stored_backup_file_ids() == ( + {SMS_FILE_ID} if inside_backup else set() + ) + def test_get_backup_files_from_manifest_closes_connection(self): m = IOSExtraction(target_path=get_ios_backup_folder()) @@ -103,6 +131,68 @@ class TestManifestModule: assert len(removed) == 1 assert removed[0]["missing"] is True + @pytest.mark.parametrize("failed_folder", ["", "3d"], ids=["root", "bucket"]) + def test_manifest_skips_missing_check_when_inventory_fails( + self, monkeypatch, caplog, failed_folder + ): + backup_path = Path(get_ios_backup_folder()) + failed_path = backup_path / failed_folder + scandir = os.scandir + + def fail_listing(path): + if Path(path) == failed_path: + raise PermissionError("cannot list backup folder") + return scandir(path) + + monkeypatch.setattr(os, "scandir", fail_listing) + m = Manifest(target_path=str(backup_path)) + with caplog.at_level(logging.INFO): + m.run() + + assert len(m.results) == 3721 + assert m._get_backup_file_from_id(SMS_FILE_ID) is not None + assert all("missing" not in result for result in m.results) + assert "Skipping the missing-file check" in caplog.text + assert "The backup might be incomplete" not in caplog.text + + def test_manifest_ignores_unreadable_unrelated_folders(self, tmp_path, monkeypatch): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + unrelated = backup_path / "notes" + unrelated.mkdir() + scandir = os.scandir + + def fail_listing(path): + if Path(path) == unrelated: + raise PermissionError("cannot list unrelated folder") + return scandir(path) + + monkeypatch.setattr(os, "scandir", fail_listing) + m = Manifest(target_path=str(backup_path)) + m.run() + + assert sum(bool(result.get("missing")) for result in m.results) == 1079 + stored = next( + result for result in m.results if result["file_id"] == SMS_FILE_ID + ) + assert "missing" not in stored + + @pytest.mark.parametrize("wrong_folder", ["ab", "notes"]) + def test_manifest_flags_files_in_the_wrong_folder(self, tmp_path, wrong_folder): + backup_path = tmp_path / "backup" + shutil.copytree(get_ios_backup_folder(), backup_path) + destination = backup_path / wrong_folder + destination.mkdir(exist_ok=True) + (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).rename(destination / SMS_FILE_ID) + + m = Manifest(target_path=str(backup_path)) + m.run() + + assert m._get_backup_file_from_id(SMS_FILE_ID) is None + moved = next(result for result in m.results if result["file_id"] == SMS_FILE_ID) + assert moved["missing"] is True + assert sum(bool(result.get("missing")) for result in m.results) == 1080 + def test_detection(self, indicator_file): m = Manifest(target_path=get_ios_backup_folder()) ind = Indicators(log=logging.getLogger())