fix(manifest): validate backup inventory before marking files missing

docs/ios/records.md

@@ -206,6 +206,8 @@ If indicators are provided through the command-line, they are checked against th
 
 An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it.
 
+Files must be stored at their expected paths inside the backup folder. If the module cannot finish listing the backup files, it logs a warning and skips the missing-file check; the manifest metadata is still extracted.
+
 ---
 
 ### `os_analytics_ad_daily.json`

src/mvt/ios/modules/backup/manifest.py

@@ -166,7 +166,11 @@ class Manifest(IOSExtraction):
                 "created": "",
             }
 
-            if file_data["flags"] == 1 and file_data["fileID"] not in stored_file_ids:
+            if (
+                stored_file_ids is not None
+                and file_data["flags"] == 1
+                and file_data["fileID"] not in stored_file_ids
+            ):
                 # Without this, a module which found nothing for one of these
                 # files would look like a negative result rather than a gap in
                 # the acquisition.

src/mvt/ios/modules/base.py

@@ -216,7 +216,7 @@ class IOSExtraction(MVTModule):
 
         return None
 
-    def _get_stored_backup_file_ids(self) -> set[str]:
+    def _get_stored_backup_file_ids(self) -> Optional[set[str]]:
         """List the IDs of the files actually stored in the backup folder.
 
         A backup folder stores each file under a two character subfolder named
@@ -225,29 +225,43 @@ class IOSExtraction(MVTModule):
         which compare a whole manifest against the folder from paying a
         `resolve()` for every entry.
 
-        :returns: The file IDs found in the backup folder, empty if there is
-                  no backup folder to walk.
+        :returns: The file IDs found at their expected paths within the backup
+                  folder, or None if the inventory could not be completed.
         """
         if not self.target_path:
-            return set()
+            return None
 
         file_ids: set[str] = set()
         try:
+            backup_root = Path(self.target_path).resolve()
             with os.scandir(self.target_path) as entries:
                 for entry in entries:
+                    if len(entry.name) != 2 or any(
+                        char not in "0123456789abcdef" for char in entry.name
+                    ):
+                        continue
                     if not entry.is_dir():
                         continue
+                    if not Path(entry.path).resolve().is_relative_to(backup_root):
+                        continue
                     with os.scandir(entry.path) as sub_entries:
                         for sub_entry in sub_entries:
+                            if sub_entry.name[:2] != entry.name:
+                                continue
+                            if sub_entry.is_symlink() and not Path(
+                                sub_entry.path
+                            ).resolve().is_relative_to(backup_root):
+                                continue
                             if sub_entry.is_file():
                                 file_ids.add(sub_entry.name)
         except OSError as exc:
-            self.log.debug(
-                "Unable to list the files stored in the backup folder %s: %s",
+            self.log.warning(
+                "Unable to list the files stored in the backup folder %s: %s. "
+                "Skipping the missing-file check.",
                 self.target_path,
                 exc,
             )
-            return set()
+            return None
 
         return file_ids
 

tests/ios_backup/test_manifest.py

@@ -5,8 +5,10 @@
 
 import gc
 import logging
+import os
 import shutil
 import warnings
+from pathlib import Path
 
 import pytest
 
@@ -43,6 +45,32 @@ def backup_without_stored_files(tmp_path):
 
 
 class TestIOSExtraction:
+    @pytest.mark.parametrize("link_directory", [False, True], ids=["file", "directory"])
+    @pytest.mark.parametrize("inside_backup", [False, True], ids=["outside", "inside"])
+    def test_stored_file_inventory_matches_symlink_resolution(
+        self, tmp_path, link_directory, inside_backup
+    ):
+        backup_path = tmp_path / "backup"
+        backup_path.mkdir()
+        destination = (backup_path if inside_backup else tmp_path) / "contents"
+        destination.mkdir()
+        (destination / SMS_FILE_ID).write_bytes(b"backup file")
+        bucket = backup_path / SMS_FILE_ID[:2]
+        try:
+            if link_directory:
+                bucket.symlink_to(destination, target_is_directory=True)
+            else:
+                bucket.mkdir()
+                (bucket / SMS_FILE_ID).symlink_to(destination / SMS_FILE_ID)
+        except OSError:
+            pytest.skip("creating symbolic links is not permitted on this system")
+
+        m = IOSExtraction(target_path=str(backup_path))
+        assert bool(m._get_backup_file_from_id(SMS_FILE_ID)) is inside_backup
+        assert m._get_stored_backup_file_ids() == (
+            {SMS_FILE_ID} if inside_backup else set()
+        )
+
     def test_get_backup_files_from_manifest_closes_connection(self):
         m = IOSExtraction(target_path=get_ios_backup_folder())
 
@@ -103,6 +131,68 @@ class TestManifestModule:
         assert len(removed) == 1
         assert removed[0]["missing"] is True
 
+    @pytest.mark.parametrize("failed_folder", ["", "3d"], ids=["root", "bucket"])
+    def test_manifest_skips_missing_check_when_inventory_fails(
+        self, monkeypatch, caplog, failed_folder
+    ):
+        backup_path = Path(get_ios_backup_folder())
+        failed_path = backup_path / failed_folder
+        scandir = os.scandir
+
+        def fail_listing(path):
+            if Path(path) == failed_path:
+                raise PermissionError("cannot list backup folder")
+            return scandir(path)
+
+        monkeypatch.setattr(os, "scandir", fail_listing)
+        m = Manifest(target_path=str(backup_path))
+        with caplog.at_level(logging.INFO):
+            m.run()
+
+        assert len(m.results) == 3721
+        assert m._get_backup_file_from_id(SMS_FILE_ID) is not None
+        assert all("missing" not in result for result in m.results)
+        assert "Skipping the missing-file check" in caplog.text
+        assert "The backup might be incomplete" not in caplog.text
+
+    def test_manifest_ignores_unreadable_unrelated_folders(self, tmp_path, monkeypatch):
+        backup_path = tmp_path / "backup"
+        shutil.copytree(get_ios_backup_folder(), backup_path)
+        unrelated = backup_path / "notes"
+        unrelated.mkdir()
+        scandir = os.scandir
+
+        def fail_listing(path):
+            if Path(path) == unrelated:
+                raise PermissionError("cannot list unrelated folder")
+            return scandir(path)
+
+        monkeypatch.setattr(os, "scandir", fail_listing)
+        m = Manifest(target_path=str(backup_path))
+        m.run()
+
+        assert sum(bool(result.get("missing")) for result in m.results) == 1079
+        stored = next(
+            result for result in m.results if result["file_id"] == SMS_FILE_ID
+        )
+        assert "missing" not in stored
+
+    @pytest.mark.parametrize("wrong_folder", ["ab", "notes"])
+    def test_manifest_flags_files_in_the_wrong_folder(self, tmp_path, wrong_folder):
+        backup_path = tmp_path / "backup"
+        shutil.copytree(get_ios_backup_folder(), backup_path)
+        destination = backup_path / wrong_folder
+        destination.mkdir(exist_ok=True)
+        (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).rename(destination / SMS_FILE_ID)
+
+        m = Manifest(target_path=str(backup_path))
+        m.run()
+
+        assert m._get_backup_file_from_id(SMS_FILE_ID) is None
+        moved = next(result for result in m.results if result["file_id"] == SMS_FILE_ID)
+        assert moved["missing"] is True
+        assert sum(bool(result.get("missing")) for result in m.results) == 1080
+
     def test_detection(self, indicator_file):
         m = Manifest(target_path=get_ios_backup_folder())
         ind = Indicators(log=logging.getLogger())