Freeze versions and bump version (#632)

* Freeze versions and bump version
* Drops support for python below 3.10

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10']  # , '3.11']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
       - uses: actions/checkout@v4
@@ -35,4 +35,4 @@ jobs:
         if: github.event_name == 'pull_request'
         with:
           pytest-coverage-path: ./pytest-coverage.txt
-          junitxml-path: ./pytest.xml
+          junitxml-path: ./pytest.xml

.github/workflows/update-ios-data.yml

@@ -21,6 +21,7 @@ jobs:
           title: '[auto] Update iOS releases and versions'
           commit-message: Add new iOS versions and build numbers
           branch: auto/add-new-ios-releases
+          draft: true
           body: |
             This is an automated pull request to update the iOS releases and version numbers.
           add-paths: |

Dockerfile

@@ -103,7 +103,7 @@ RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
 
 
 # Create main image
-FROM ubuntu:22.04 as main
+FROM ubuntu:24.04 as main
 
 LABEL org.opencontainers.image.url="https://mvt.re"
 LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
@@ -135,8 +135,7 @@ COPY --from=build-usbmuxd /build /
 COPY . mvt/
 RUN apt-get update \
    && apt-get install -y git python3-pip \
-   && PIP_NO_CACHE_DIR=1 pip3 install --upgrade pip \
-   && PIP_NO_CACHE_DIR=1 pip3 install ./mvt \
+   && PIP_NO_CACHE_DIR=1 pip3 install --break-system-packages ./mvt \
    && apt-get remove -y python3-pip git && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/* \
    && rm -rf mvt

pyproject.toml

@@ -19,25 +19,25 @@ classifiers = [
     "Programming Language :: Python"
 ]
 dependencies = [
-    "click >=8.1.3",
-    "rich >=12.6.0",
-    "tld >=0.12.6",
-    "requests >=2.28.1",
-    "simplejson >=3.17.6",
-    "packaging >=21.3",
-    "appdirs >=1.4.4",
-    "iOSbackup >=0.9.923",
-    "adb-shell[usb] >=0.4.3",
-    "libusb1 >=3.0.0",
-    "cryptography >=42.0.5",
-    "pyyaml >=6.0",
-    "pyahocorasick >= 2.0.0",
-    "betterproto >=1.2.0",
-    "pydantic >= 2.10.0",
-    "pydantic-settings >= 2.7.0",
-    'backports.zoneinfo; python_version < "3.9"',
+    "click==8.2.1",
+    "rich==14.0.0",
+    "tld==0.13.1",
+    "requests==2.32.2",
+    "simplejson==3.20.1",
+    "packaging==25.0",
+    "appdirs==1.4.4",
+    "iOSbackup==0.9.925",
+    "adb-shell[usb]==0.4.4",
+    "libusb1==3.3.1",
+    "cryptography==45.0.3",
+    "PyYAML>=6.0.2",
+    "pyahocorasick==2.1.0",
+    "betterproto==1.2.5",
+    "pydantic==2.11.5",
+    "pydantic-settings==2.9.1",
+    "NSKeyedUnArchiver==1.5",
 ]
-requires-python = ">= 3.8"
+requires-python = ">= 3.10"
 
 [project.urls]
 homepage = "https://docs.mvt.re/en/latest/"
@@ -103,4 +103,4 @@ where = ["src"]
 mvt = ["ios/data/*.json"]
 
 [tool.setuptools.dynamic]
-version = {attr = "mvt.common.version.MVT_VERSION"}
+version = {attr = "mvt.common.version.MVT_VERSION"}

src/mvt/android/artifacts/dumpsys_adb.py

@@ -4,13 +4,14 @@
 #   https://license.mvt.re/1.1/
 
 import base64
+import binascii
 import hashlib
 
 from .artifact import AndroidArtifact
 
 
 class DumpsysADBArtifact(AndroidArtifact):
-    multiline_fields = ["user_keys"]
+    multiline_fields = ["user_keys", "keystore"]
 
     def indented_dump_parser(self, dump_data):
         """
@@ -67,14 +68,38 @@ class DumpsysADBArtifact(AndroidArtifact):
 
         return res
 
+    def parse_xml(self, xml_data):
+        """
+        Parse XML data from dumpsys ADB output
+        """
+        import xml.etree.ElementTree as ET
+
+        keystore = []
+        keystore_root = ET.fromstring(xml_data)
+        for adb_key in keystore_root.findall("adbKey"):
+            key_info = self.calculate_key_info(adb_key.get("key").encode("utf-8"))
+            key_info["last_connected"] = adb_key.get("lastConnection")
+            keystore.append(key_info)
+
+        return keystore
+
     @staticmethod
     def calculate_key_info(user_key: bytes) -> str:
-        key_base64, user = user_key.split(b" ", 1)
-        key_raw = base64.b64decode(key_base64)
-        key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
-        key_fingerprint_colon = ":".join(
-            [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
-        )
+        if b" " in user_key:
+            key_base64, user = user_key.split(b" ", 1)
+        else:
+            key_base64, user = user_key, b""
+
+        try:
+            key_raw = base64.b64decode(key_base64)
+            key_fingerprint = hashlib.md5(key_raw).hexdigest().upper()
+            key_fingerprint_colon = ":".join(
+                [key_fingerprint[i : i + 2] for i in range(0, len(key_fingerprint), 2)]
+            )
+        except binascii.Error:
+            # Impossible to parse base64
+            key_fingerprint_colon = ""
+
         return {
             "user": user.decode("utf-8"),
             "fingerprint": key_fingerprint_colon,
@@ -115,8 +140,24 @@ class DumpsysADBArtifact(AndroidArtifact):
         if parsed.get("debugging_manager") is None:
             self.log.error("Unable to find expected ADB entries in dumpsys output")  # noqa
             return
+
+        # Keystore can be in different levels, as the basic parser
+        # is not always consistent due to different dumpsys formats.
+        if parsed.get("keystore"):
+            keystore_data = b"\n".join(parsed["keystore"])
+        elif parsed["debugging_manager"].get("keystore"):
+            keystore_data = b"\n".join(parsed["debugging_manager"]["keystore"])
         else:
-            parsed = parsed["debugging_manager"]
+            keystore_data = None
+
+        # Keystore is in XML format on some devices and we need to parse it
+        if keystore_data and keystore_data.startswith(b"<?xml"):
+            parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
+        else:
+            # Keystore is not XML format
+            parsed["debugging_manager"]["keystore"] = keystore_data
+
+        parsed = parsed["debugging_manager"]
 
         # Calculate key fingerprints for better readability
         key_info = []

src/mvt/android/artifacts/tombstone_crashes.py

@@ -62,7 +62,7 @@ class TombstoneCrashResult(pydantic.BaseModel):
     process_name: Optional[str] = None
     binary_path: Optional[str] = None
     selinux_label: Optional[str] = None
-    uid: Optional[int] = None
+    uid: int
     signal_info: SignalInfo
     cause: Optional[str] = None
     extra: Optional[str] = None
@@ -124,7 +124,9 @@ class TombstoneCrashArtifact(AndroidArtifact):
         Parse Android tombstone crash files from a protobuf object.
         """
         tombstone_pb = Tombstone().parse(data)
-        tombstone_dict = tombstone_pb.to_dict(betterproto.Casing.SNAKE)
+        tombstone_dict = tombstone_pb.to_dict(
+            betterproto.Casing.SNAKE, include_default_values=True
+        )
 
         # Add some extra metadata
         tombstone_dict["timestamp"] = self._parse_timestamp_string(

src/mvt/android/cmd_check_androidqf.py

@@ -12,8 +12,6 @@ from typing import List, Optional
 from mvt.common.command import Command
 
 from .modules.androidqf import ANDROIDQF_MODULES
-from .modules.bugreport import BUGREPORT_MODULES
-from .modules.bugreport.base import BugReportModule
 
 log = logging.getLogger(__name__)
 
@@ -41,11 +39,7 @@ class CmdAndroidCheckAndroidQF(Command):
         )
 
         self.name = "check-androidqf"
-
-        # We can load AndroidQF and bugreport modules here, as
-        # AndroidQF dump will contain a bugreport.
-        self.modules = ANDROIDQF_MODULES + BUGREPORT_MODULES
-        # TODO: Check how to namespace and deduplicate modules.
+        self.modules = ANDROIDQF_MODULES
 
         self.format: Optional[str] = None
         self.archive: Optional[zipfile.ZipFile] = None
@@ -60,44 +54,12 @@ class CmdAndroidCheckAndroidQF(Command):
                 for fname in subfiles:
                     file_path = os.path.relpath(os.path.join(root, fname), parent_path)
                     self.files.append(file_path)
-
         elif os.path.isfile(self.target_path):
             self.format = "zip"
             self.archive = zipfile.ZipFile(self.target_path)
             self.files = self.archive.namelist()
 
-    def load_bugreport(self):
-        # Refactor this file list loading
-        # First we need to find the bugreport file location
-        bugreport_zip_path = None
-        for file_name in self.files:
-            if file_name.endswith("bugreport.zip"):
-                bugreport_zip_path = file_name
-                break
-        else:
-            self.log.warning("No bugreport.zip found in the AndroidQF dump")
-            return None
-
-        if self.format == "zip":
-            # Create handle to the bugreport.zip file inside the AndroidQF dump
-            handle = self.archive.open(bugreport_zip_path)
-            bugreport_zip = zipfile.ZipFile(handle)
-        else:
-            # Load the bugreport.zip file from the extracted AndroidQF dump on disk.
-            parent_path = Path(self.target_path).absolute().parent.as_posix()
-            bug_report_path = os.path.join(parent_path, bugreport_zip_path)
-            bugreport_zip = zipfile.ZipFile(bug_report_path)
-
-        return bugreport_zip
-
     def module_init(self, module):
-        if isinstance(module, BugReportModule):
-            bugreport_archive = self.load_bugreport()
-            if not bugreport_archive:
-                return
-            module.from_zip(bugreport_archive, bugreport_archive.namelist())
-            return
-
         if self.format == "zip":
             module.from_zip_file(self.archive, self.files)
         else:

src/mvt/common/cmd_check_iocs.py

@@ -65,6 +65,10 @@ class CmdCheckIOCS(Command):
                 m = iocs_module.from_json(
                     file_path, log=logging.getLogger(iocs_module.__module__)
                 )
+                if not m:
+                    log.warning("No result from this module, skipping it")
+                    continue
+
                 if self.iocs.total_ioc_count > 0:
                     m.indicators = self.iocs
                     m.indicators.log = m.log

src/mvt/common/logo.py

@@ -29,7 +29,7 @@ def check_updates() -> None:
         if latest_version:
             rich_print(
                 f"\t\t[bold]Version {latest_version} is available! "
-                "Upgrade mvt with `pip3 install -U mvt`[/bold]"
+                "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
             )
 
     # Then we check for indicators files updates.

src/mvt/common/module.py

@@ -69,10 +69,14 @@ class MVTModule:
     @classmethod
     def from_json(cls, json_path: str, log: logging.Logger):
         with open(json_path, "r", encoding="utf-8") as handle:
-            results = json.load(handle)
-            if log:
-                log.info('Loaded %d results from "%s"', len(results), json_path)
-            return cls(results=results, log=log)
+            try:
+                results = json.load(handle)
+                if log:
+                    log.info('Loaded %d results from "%s"', len(results), json_path)
+                return cls(results=results, log=log)
+            except json.decoder.JSONDecodeError as err:
+                log.error('Error to decode the json "%s" file: "%s"', json_path, err)
+                return None
 
     @classmethod
     def get_slug(cls) -> str:

src/mvt/common/version.py

@@ -3,4 +3,4 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.6.0"
+MVT_VERSION = "2.6.1"

src/mvt/ios/data/ios_versions.json

@@ -891,6 +891,10 @@
         "version": "15.8.2",
         "build": "19H384"
     },
+    {
+        "version": "15.8.4",
+        "build": "19H390"
+    },
     {
         "build": "20A362",
         "version": "16.0"
@@ -992,6 +996,10 @@
         "version": "16.7.8",
         "build": "20H343"
     },
+    {
+        "version": "16.7.11",
+        "build": "20H360"
+    },
     {
         "version": "17.0",
         "build": "21A327"
@@ -1076,6 +1084,10 @@
         "version": "17.6.1",
         "build": "21G101"
     },
+    {
+        "version": "17.7.7",
+        "build": "21H433"
+    },
     {
         "version": "18",
         "build": "22A3354"
@@ -1103,5 +1115,21 @@
     {
         "version": "18.3",
         "build": "22D63"
+    },
+    {
+        "version": "18.3.1",
+        "build": "22D72"
+    },
+    {
+        "version": "18.4",
+        "build": "22E240"
+    },
+    {
+        "version": "18.4.1",
+        "build": "22E252"
+    },
+    {
+        "version": "18.5",
+        "build": "22F76"
     }
 ]

tests/android/test_artifact_dumpsys_adb.py

@@ -29,3 +29,28 @@ class TestDumpsysADBArtifact:
             user_key["fingerprint"] == "F0:A1:3D:8C:B3:F4:7B:09:9F:EE:8B:D8:38:2E:BD:C6"
         )
         assert user_key["user"] == "user@linux"
+
+    def test_parsing_adb_xml(self):
+        da_adb = DumpsysADBArtifact()
+        file = get_artifact("android_data/dumpsys_adb_xml.txt")
+        with open(file, "rb") as f:
+            data = f.read()
+
+        da_adb.parse(data)
+
+        assert len(da_adb.results) == 1
+
+        adb_data = da_adb.results[0]
+        assert "user_keys" in adb_data
+        assert len(adb_data["user_keys"]) == 1
+
+        # Check key and fingerprint parsed successfully.
+        expected_fingerprint = "F0:0B:27:08:E3:68:7B:FA:4C:79:A2:B4:BF:0E:CF:70"
+        user_key = adb_data["user_keys"][0]
+        user_key["fingerprint"] == expected_fingerprint
+        assert user_key["user"] == "user@laptop"
+
+        key_store_entry = adb_data["keystore"][0]
+        assert key_store_entry["user"] == "user@laptop"
+        assert key_store_entry["fingerprint"] == expected_fingerprint
+        assert key_store_entry["last_connected"] == "1628501829898"

tests/artifacts/android_data/dumpsys_adb_xml.txt

@@ -0,0 +1,16 @@
+-------------------------------------------------------------------------------
+DUMP OF SERVICE adb:
+ADB MANAGER STATE (dumpsys adb):
+{
+  debugging_manager={
+    connected_to_adb=true
+    user_keys=QAAAAAcgbytJst31DsaSP7hn8QcBXKR9NPVPK9MZssFVSNIP user@laptop
+
+    keystore=<?xml version='1.0' encoding='utf-8' standalone='yes' ?>
+    <keyStore version="1">
+    <adbKey key="QAAAAAcgbytJst31DsaSP7hn8QcBXKR9NPVPK9MZssFVSNIP user@laptop" lastConnection="1628501829898" />
+    </keyStore>
+
+  }
+}
+--------- 0.012s was the duration of dumpsys adb, ending at: 2025-02-04 20:25:58