Merge pull request #964 from forest-savage1234/docs/python-minimum-q093-20260929

Correct the Python minimum in installation documentation

docs/install.md

@@ -1,6 +1,6 @@
 # Installation
 
-Before proceeding, please note that MVT requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
+Before proceeding, please note that MVT requires Python 3.10 or newer to run. While it should be available on most operating systems, please make sure of that before proceeding.
 
 ## Dependencies on Linux
 

docs/ios/records.md

@@ -204,6 +204,10 @@ This JSON file is created by mvt-ios' `Manifest` module. The module extracts rec
 
 If indicators are provided through the command-line, they are checked against the original relative path in case. In some cases, there might be records of files created containing a domain name in their name, for example in the case of browser cache folders. Any matches are stored in *manifest_detected.json*.
 
+An incomplete backup still lists in its manifest the files it failed to acquire. Those records carry a `"missing": true` field, and the module reports how many of them it found. Use it to tell a module which returned nothing because the artifact was not acquired apart from one which found nothing on a device that never had it.
+
+Files must be stored at their expected paths inside the backup folder. If the module cannot finish listing the backup files, it logs a warning and skips the missing-file check; the manifest metadata is still extracted.
+
 ---
 
 ### `os_analytics_ad_daily.json`

src/mvt/android/modules/bugreport/dumpsys_accessibility.py

@@ -48,17 +48,12 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
         )
         self.parse(content)
 
-        listed = unnamed = enabled_count = 0
+        listed = unnamed = 0
         for result in self.results:
             if result.get("component"):
                 listed += 1
-                enabled = result.get("enabled")
-                if enabled is True:
-                    enabled_count += 1
                 self.log.info(
-                    'Found accessibility service "%s" (enabled: %s)',
-                    result.get("component"),
-                    "not stated" if enabled is None else enabled,
+                    'Found accessibility service "%s"', result.get("component")
                 )
                 continue
             # The operator gets this per user as a LOW alert from
@@ -67,9 +62,8 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
             unnamed += result.get("unnamed_service_count") or 0
 
         self.log.info(
-            "Identified a total of %d accessibility services, %d reported enabled "
-            "(%d more stated by the dump without a component name)",
+            "Identified a total of %d accessibility services (%d more stated by "
+            "the dump without a component name)",
             listed,
-            enabled_count,
             unnamed,
         )

src/mvt/ios/data/ios_versions.json

@@ -1279,5 +1279,9 @@
     {
         "version": "27.0",
         "build": "24A437"
+    },
+    {
+        "version": "27.0.1",
+        "build": "24A446"
     }
 ]

src/mvt/ios/modules/backup/manifest.py

@@ -147,6 +147,12 @@ class Manifest(IOSExtraction):
             )
         names = [description[0] for description in cur.description]
 
+        # An incomplete backup still lists the files it failed to acquire in
+        # its manifest. Only regular files are stored in the backup folder, so
+        # directories and symlinks (flags 2 and 4) are not looked up.
+        stored_file_ids = self._get_stored_backup_file_ids()
+        missing_files = 0
+
         for file_entry in cur:
             file_data = {}
             for index, value in enumerate(file_entry):
@@ -160,6 +166,22 @@ class Manifest(IOSExtraction):
                 "created": "",
             }
 
+            if (
+                stored_file_ids is not None
+                and file_data["flags"] == 1
+                and file_data["fileID"] not in stored_file_ids
+            ):
+                # Without this, a module which found nothing for one of these
+                # files would look like a negative result rather than a gap in
+                # the acquisition.
+                cleaned_metadata["missing"] = True
+                missing_files += 1
+                self.log.debug(
+                    "File %s is listed in the manifest but was not found in the "
+                    "backup folder",
+                    cleaned_metadata["relative_path"],
+                )
+
             if file_data["file"]:
                 try:
                     file_plist = plistlib.load(io.BytesIO(file_data["file"]))
@@ -197,3 +219,10 @@ class Manifest(IOSExtraction):
         conn.close()
 
         self.log.info("Extracted a total of %d file metadata items", len(self.results))
+
+        if missing_files:
+            self.log.info(
+                "Found %d files listed in the manifest but missing from the backup "
+                "folder. The backup might be incomplete.",
+                missing_files,
+            )

src/mvt/ios/modules/base.py

@@ -216,6 +216,55 @@ class IOSExtraction(MVTModule):
 
         return None
 
+    def _get_stored_backup_file_ids(self) -> Optional[set[str]]:
+        """List the IDs of the files actually stored in the backup folder.
+
+        A backup folder stores each file under a two character subfolder named
+        after the first two characters of its file ID. Walking the folders once
+        is cheaper than a filesystem lookup per file ID, and it keeps callers
+        which compare a whole manifest against the folder from paying a
+        `resolve()` for every entry.
+
+        :returns: The file IDs found at their expected paths within the backup
+                  folder, or None if the inventory could not be completed.
+        """
+        if not self.target_path:
+            return None
+
+        file_ids: set[str] = set()
+        try:
+            backup_root = Path(self.target_path).resolve()
+            with os.scandir(self.target_path) as entries:
+                for entry in entries:
+                    if len(entry.name) != 2 or any(
+                        char not in "0123456789abcdef" for char in entry.name
+                    ):
+                        continue
+                    if not entry.is_dir():
+                        continue
+                    if not Path(entry.path).resolve().is_relative_to(backup_root):
+                        continue
+                    with os.scandir(entry.path) as sub_entries:
+                        for sub_entry in sub_entries:
+                            if sub_entry.name[:2] != entry.name:
+                                continue
+                            if sub_entry.is_symlink() and not Path(
+                                sub_entry.path
+                            ).resolve().is_relative_to(backup_root):
+                                continue
+                            if sub_entry.is_file():
+                                file_ids.add(sub_entry.name)
+        except OSError as exc:
+            self.log.warning(
+                "Unable to list the files stored in the backup folder %s: %s. "
+                "Skipping the missing-file check.",
+                self.target_path,
+                exc,
+            )
+            return None
+
+        return file_ids
+
     def _get_fs_files_from_patterns(self, root_paths: list) -> Iterator[str]:
         if not self.target_path:
             return

tests/android/test_artifact_dumpsys_accessibility.py

@@ -46,23 +46,6 @@ class TestDumpsysAccessibilityArtifact:
         # "not stated" into "not installed", so it reads None here.
         assert da.results[0]["installed"] is None
 
-    def test_parsing_installed_and_enabled(self):
-        da = DumpsysAccessibilityArtifact()
-        with open(get_artifact("android_data/dumpsys_accessibility_enabled.txt")) as f:
-            da.parse(f.read())
-
-        assert len(da.results) == 5
-        assert all(result["component"] for result in da.results)
-        assert all(result["installed"] is True for result in da.results)
-        enabled = [result for result in da.results if result["enabled"] is True]
-        assert len(enabled) == 1
-        assert enabled[0]["package_name"] == "com.samsung.accessibility"
-        assert (
-            enabled[0]["component"]
-            == "com.samsung.accessibility/.universalswitch.UniversalSwitchService"
-        )
-        assert sum(result["enabled"] is False for result in da.results) == 4
-
     def test_accessibility_service_alert(self):
         da = DumpsysAccessibilityArtifact()
         file = get_artifact("android_data/dumpsys_accessibility_v14_or_later.txt")

tests/android/test_bugreport_dumpsys_accessibility.py

@@ -1,73 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2026 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import logging
-
-from mvt.android.modules.bugreport.dumpsys_accessibility import DumpsysAccessibility
-
-from ..utils import get_artifact
-
-
-def _run(content, monkeypatch, caplog):
-    module = DumpsysAccessibility()
-    dumpstate = f"DUMP OF SERVICE accessibility:\n{content}".encode()
-    monkeypatch.setattr(module, "_get_dumpstate_file", lambda: dumpstate)
-    with caplog.at_level(logging.INFO):
-        module.run()
-
-
-def test_logs_mixed_enabled_states(monkeypatch, caplog):
-    with open(get_artifact("android_data/dumpsys_accessibility_enabled.txt")) as handle:
-        _run(handle.read(), monkeypatch, caplog)
-
-    assert (
-        'Found accessibility service "com.samsung.accessibility/'
-        '.universalswitch.UniversalSwitchService" (enabled: True)' in caplog.messages
-    )
-    assert sum("(enabled: False)" in message for message in caplog.messages) == 4
-    assert (
-        "Identified a total of 5 accessibility services, 1 reported enabled "
-        "(0 more stated by the dump without a component name)" in caplog.messages
-    )
-
-
-def test_logs_enabled_count_alongside_unnamed_count(monkeypatch, caplog):
-    with open(
-        get_artifact("android_data/dumpsys_accessibility_v14_or_later.txt")
-    ) as handle:
-        _run(handle.read(), monkeypatch, caplog)
-
-    assert (
-        "Identified a total of 1 accessibility services, 1 reported enabled "
-        "(1 more stated by the dump without a component name)" in caplog.messages
-    )
-    assert (
-        sum("Found accessibility service" in message for message in caplog.messages)
-        == 1
-    )
-
-
-def test_logs_unknown_enabled_state_without_treating_it_as_disabled(
-    monkeypatch, caplog
-):
-    _run(
-        """User state[attributes:{id=0, installedServiceCount=2}
-  installed services: {
-    0 : com.example/.Service
-  }
-""",
-        monkeypatch,
-        caplog,
-    )
-
-    assert (
-        'Found accessibility service "com.example/.Service" (enabled: not stated)'
-        in caplog.messages
-    )
-    assert not any("(enabled: False)" in message for message in caplog.messages)
-    assert (
-        "Identified a total of 1 accessibility services, 0 reported enabled "
-        "(1 more stated by the dump without a component name)" in caplog.messages
-    )

tests/artifacts/android_data/dumpsys_accessibility_enabled.txt

@@ -1,16 +0,0 @@
-ACCESSIBILITY MANAGER (dumpsys accessibility)
-
-currentUserId=0
-User state[
-     attributes:{id=0, touchExplorationEnabled=false, installedServiceCount=5}
-     installed services: {
-      0 : com.google.android.apps.accessibility.voiceaccess/.JustSpeakService  (A11yTool)
-      1 : com.microsoft.appmanager/com.microsoft.mmx.screenmirroringsrc.accessibility.ScreenMirroringAccessibilityService
-      2 : com.samsung.accessibility/.assistantmenu.serviceframework.AssistantMenuService  (A11yTool)
-      3 : com.samsung.accessibility/.universalswitch.UniversalSwitchService  (A11yTool)
-      4 : com.samsung.android.accessibility.talkback/com.samsung.android.marvin.talkback.TalkBackService  (A11yTool)
-     }
-     Bound services:{}
-     Enabled services:{{com.samsung.accessibility/.universalswitch.UniversalSwitchService}}
-     Binding services:{}
-     Crashed services:{}

tests/ios_backup/test_manifest.py

@@ -5,7 +5,12 @@
 
 import gc
 import logging
+import os
+import shutil
 import warnings
+from pathlib import Path
+
+import pytest
 
 from mvt.common.indicators import Indicators
 from mvt.common.module import run_module
@@ -14,8 +19,58 @@ from mvt.ios.modules.backup.manifest import Manifest
 
 from ..utils import get_ios_backup_folder
 
+# fileID of HomeDomain::Library/SMS/sms.db in the test backup. It is one of the
+# few files the test backup actually stores.
+SMS_FILE_ID = "3d0d7e5fb2ce288813306e4d4636395e047a3d28"
+
+
+@pytest.fixture
+def backup_without_stored_files(tmp_path):
+    """A copy of the test backup with every stored file removed.
+
+    The test backup only ships a handful of the files its manifest lists, so
+    dropping what it does store leaves a backup where every regular file the
+    manifest mentions is missing from the folder.
+    """
+    backup_path = tmp_path / "backup"
+    shutil.copytree(get_ios_backup_folder(), backup_path)
+
+    for file_id_folder in backup_path.iterdir():
+        if not file_id_folder.is_dir():
+            continue
+        for backup_file in file_id_folder.iterdir():
+            backup_file.unlink()
+
+    return str(backup_path)
+
 
 class TestIOSExtraction:
+    @pytest.mark.parametrize("link_directory", [False, True], ids=["file", "directory"])
+    @pytest.mark.parametrize("inside_backup", [False, True], ids=["outside", "inside"])
+    def test_stored_file_inventory_matches_symlink_resolution(
+        self, tmp_path, link_directory, inside_backup
+    ):
+        backup_path = tmp_path / "backup"
+        backup_path.mkdir()
+        destination = (backup_path if inside_backup else tmp_path) / "contents"
+        destination.mkdir()
+        (destination / SMS_FILE_ID).write_bytes(b"backup file")
+        bucket = backup_path / SMS_FILE_ID[:2]
+        try:
+            if link_directory:
+                bucket.symlink_to(destination, target_is_directory=True)
+            else:
+                bucket.mkdir()
+                (bucket / SMS_FILE_ID).symlink_to(destination / SMS_FILE_ID)
+        except OSError:
+            pytest.skip("creating symbolic links is not permitted on this system")
+
+        m = IOSExtraction(target_path=str(backup_path))
+        assert bool(m._get_backup_file_from_id(SMS_FILE_ID)) is inside_backup
+        assert m._get_stored_backup_file_ids() == (
+            {SMS_FILE_ID} if inside_backup else set()
+        )
+
     def test_get_backup_files_from_manifest_closes_connection(self):
         m = IOSExtraction(target_path=get_ios_backup_folder())
 
@@ -41,6 +96,103 @@ class TestManifestModule:
         assert len(m.timeline) == 5881
         assert len(m.alertstore.alerts) == 0
 
+    def test_manifest_flags_the_files_missing_from_an_incomplete_backup(self):
+        m = Manifest(target_path=get_ios_backup_folder())
+        run_module(m)
+
+        missing = [result for result in m.results if result.get("missing")]
+        # The test backup only stores a handful of the files its manifest
+        # lists, and every one of the rest is a regular file (flags 1).
+        assert len(missing) == 1079
+        assert all(result["flags"] == 1 for result in missing)
+
+        stored = [result for result in m.results if result["file_id"] == SMS_FILE_ID]
+        assert len(stored) == 1
+        assert "missing" not in stored[0]
+
+    def test_manifest_flags_every_stored_file_removed_from_the_backup_folder(
+        self, backup_without_stored_files
+    ):
+        m = Manifest(target_path=backup_without_stored_files)
+        run_module(m)
+
+        missing = [result for result in m.results if result.get("missing")]
+        assert len(missing) == 1089
+
+    def test_manifest_flags_a_file_removed_from_the_backup_folder(self, tmp_path):
+        backup_path = tmp_path / "backup"
+        shutil.copytree(get_ios_backup_folder(), backup_path)
+        (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).unlink()
+
+        m = Manifest(target_path=str(backup_path))
+        run_module(m)
+
+        removed = [result for result in m.results if result["file_id"] == SMS_FILE_ID]
+        assert len(removed) == 1
+        assert removed[0]["missing"] is True
+
+    @pytest.mark.parametrize("failed_folder", ["", "3d"], ids=["root", "bucket"])
+    def test_manifest_skips_missing_check_when_inventory_fails(
+        self, monkeypatch, caplog, failed_folder
+    ):
+        backup_path = Path(get_ios_backup_folder())
+        failed_path = backup_path / failed_folder
+        scandir = os.scandir
+
+        def fail_listing(path):
+            if Path(path) == failed_path:
+                raise PermissionError("cannot list backup folder")
+            return scandir(path)
+
+        monkeypatch.setattr(os, "scandir", fail_listing)
+        m = Manifest(target_path=str(backup_path))
+        with caplog.at_level(logging.INFO):
+            m.run()
+
+        assert len(m.results) == 3721
+        assert m._get_backup_file_from_id(SMS_FILE_ID) is not None
+        assert all("missing" not in result for result in m.results)
+        assert "Skipping the missing-file check" in caplog.text
+        assert "The backup might be incomplete" not in caplog.text
+
+    def test_manifest_ignores_unreadable_unrelated_folders(self, tmp_path, monkeypatch):
+        backup_path = tmp_path / "backup"
+        shutil.copytree(get_ios_backup_folder(), backup_path)
+        unrelated = backup_path / "notes"
+        unrelated.mkdir()
+        scandir = os.scandir
+
+        def fail_listing(path):
+            if Path(path) == unrelated:
+                raise PermissionError("cannot list unrelated folder")
+            return scandir(path)
+
+        monkeypatch.setattr(os, "scandir", fail_listing)
+        m = Manifest(target_path=str(backup_path))
+        m.run()
+
+        assert sum(bool(result.get("missing")) for result in m.results) == 1079
+        stored = next(
+            result for result in m.results if result["file_id"] == SMS_FILE_ID
+        )
+        assert "missing" not in stored
+
+    @pytest.mark.parametrize("wrong_folder", ["ab", "notes"])
+    def test_manifest_flags_files_in_the_wrong_folder(self, tmp_path, wrong_folder):
+        backup_path = tmp_path / "backup"
+        shutil.copytree(get_ios_backup_folder(), backup_path)
+        destination = backup_path / wrong_folder
+        destination.mkdir(exist_ok=True)
+        (backup_path / SMS_FILE_ID[:2] / SMS_FILE_ID).rename(destination / SMS_FILE_ID)
+
+        m = Manifest(target_path=str(backup_path))
+        m.run()
+
+        assert m._get_backup_file_from_id(SMS_FILE_ID) is None
+        moved = next(result for result in m.results if result["file_id"] == SMS_FILE_ID)
+        assert moved["missing"] is True
+        assert sum(bool(result.get("missing")) for result in m.results) == 1080
+
     def test_detection(self, indicator_file):
         m = Manifest(target_path=get_ios_backup_folder())
         ind = Indicators(log=logging.getLogger())