Fix betterproto2 migration: update generated proto code and callers

The dependency switch from betterproto to betterproto2 was incomplete.
This updates all affected files to use the betterproto2 API:

- tombstone.py: rewrite generated code to use betterproto2.field() with
  explicit TYPE_* constants, repeated/optional/group flags, and map_meta()
  for map fields
- tombstone_crashes.py: update import and fix to_dict() call to use
  keyword-only casing= argument required by betterproto2
- pyproject.toml: replace betterproto[compiler] dev dep with betterproto2-compiler
- Makefile: update protoc plugin flag to --python_betterproto2_out

.github/workflows/add-issue-to-project.yml

@@ -11,7 +11,7 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.5.0
+      - uses: actions/add-to-project@v1
         with:
           # You can target a project in a different organization
           # to the issue

README.md

@@ -4,9 +4,6 @@
 
 # Mobile Verification Toolkit
 
-> [!IMPORTANT]  
-> Soon we will merge the v3 pull request which will result in breaking changes. If you rely on mvt output in other script make sure to the the branch before we merge. More details: https://github.com/mvt-project/mvt/issues/757
-
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
 [![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
 [![CI](https://github.com/mvt-project/mvt/actions/workflows/tests.yml/badge.svg)](https://github.com/mvt-project/mvt/actions/workflows/tests.yml)

src/mvt/android/artifacts/dumpsys_accessibility.py

@@ -22,13 +22,13 @@ class DumpsysAccessibilityArtifact(AndroidArtifact):
 
     def parse(self, content: str) -> None:
         """
-        Parse the Dumpsys Accessibility section.
-        Adds results to self.results (List[Dict[str, Any]])
+        Parse the Dumpsys Accessibility section/
+        Adds results to self.results (List[Dict[str, str]])
 
         :param content: content of the accessibility section (string)
         """
 
-        # Parse installed services
+        # "Old" syntax
         in_services = False
         for line in content.splitlines():
             if line.strip().startswith("installed services:"):
@@ -39,6 +39,7 @@ class DumpsysAccessibilityArtifact(AndroidArtifact):
                 continue
 
             if line.strip() == "}":
+                # At end of installed services
                 break
 
             service = line.split(":")[1].strip()
@@ -47,66 +48,21 @@ class DumpsysAccessibilityArtifact(AndroidArtifact):
                 {
                     "package_name": service.split("/")[0],
                     "service": service,
-                    "enabled": False,
                 }
             )
 
-        # Parse enabled services from both old and new formats.
-        #
-        # Old format (multi-line block):
-        #   enabled services: {
-        #     0 : com.example/.MyService
-        #   }
-        #
-        # New format (single line, AOSP >= 14):
-        #   Enabled services:{{com.example/com.example.MyService}, {com.other/com.other.Svc}}
-        enabled_services = set()
+        # "New" syntax - AOSP >= 14 (?)
+        # Looks like:
+        # Enabled services:{{com.azure.authenticator/com.microsoft.brooklyn.module.accessibility.BrooklynAccessibilityService}, {com.agilebits.onepassword/com.agilebits.onepassword.filling.accessibility.FillingAccessibilityService}}
 
-        in_enabled = False
         for line in content.splitlines():
-            stripped = line.strip()
+            if line.strip().startswith("Enabled services:"):
+                matches = re.finditer(r"{([^{]+?)}", line)
 
-            if in_enabled:
-                if stripped == "}":
-                    in_enabled = False
-                    continue
-                service = line.split(":")[1].strip()
-                enabled_services.add(service)
-                continue
-
-            if re.match(r"enabled services:\s*\{\s*$", stripped, re.IGNORECASE):
-                # Old multi-line format: "enabled services: {"
-                in_enabled = True
-                continue
-
-            if re.match(r"enabled services:\s*\{", stripped, re.IGNORECASE):
-                # New single-line format: "Enabled services:{{pkg/svc}, {pkg2/svc2}}"
-                matches = re.finditer(r"\{([^{}]+)\}", stripped)
                 for match in matches:
-                    enabled_services.add(match.group(1).strip())
+                    # Each match is in format: <package_name>/<service>
+                    package_name, _, service = match.group(1).partition("/")
 
-        # Mark installed services that are enabled.
-        # Installed service names may include trailing annotations like
-        # "(A11yTool)" that are absent from the enabled services list,
-        # so strip annotations before comparing.
-        def _strip_annotation(s: str) -> str:
-            return re.sub(r"\s+\(.*?\)\s*$", "", s)
-
-        installed_stripped = {
-            _strip_annotation(r["service"]): r for r in self.results
-        }
-        for enabled in enabled_services:
-            if enabled in installed_stripped:
-                installed_stripped[enabled]["enabled"] = True
-
-        # Add enabled services not found in the installed list
-        for service in enabled_services:
-            if service not in installed_stripped:
-                package_name, _, _ = service.partition("/")
-                self.results.append(
-                    {
-                        "package_name": package_name,
-                        "service": service,
-                        "enabled": True,
-                    }
-                )
+                    self.results.append(
+                        {"package_name": package_name, "service": service}
+                    )

src/mvt/android/modules/bugreport/dumpsys_accessibility.py

@@ -49,14 +49,9 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
 
         for result in self.results:
             self.log.info(
-                'Found installed accessibility service "%s" (enabled: %s)',
-                result.get("service"),
-                result.get("enabled", False),
+                'Found installed accessibility service "%s"', result.get("service")
             )
 
-        enabled_count = sum(1 for r in self.results if r.get("enabled"))
         self.log.info(
-            "Identified a total of %d accessibility services, %d enabled",
-            len(self.results),
-            enabled_count,
+            "Identified a total of %d accessibility services", len(self.results)
         )

src/mvt/common/indicators.py

@@ -100,17 +100,6 @@ class Indicators:
         key, value = indicator.get("pattern", "").strip("[]").split("=")
         key = key.strip()
 
-        # Normalize hash algorithm keys so that both the STIX2-spec-compliant
-        # form (e.g. file:hashes.'SHA-256', which requires quotes around
-        # algorithm names that contain hyphens) and the non-standard lowercase
-        # form (e.g. file:hashes.sha256) are accepted.  Strip single quotes and
-        # hyphens from the algorithm name only, then lowercase it.
-        for sep in ("hashes.", "cert."):
-            if sep in key:
-                prefix, _, algo = key.partition(sep)
-                key = prefix + sep + algo.replace("'", "").replace("-", "").lower()
-                break
-
         if key == "domain-name:value":
             # We force domain names to lower case.
             self._add_indicator(

src/mvt/ios/modules/mixed/sms.py

@@ -123,11 +123,6 @@ class SMS(IOSExtraction):
                 """
                 )
                 items = list(cur)
-            elif "no such table" in str(exc):
-                self.log.info(
-                    "No SMS tables found in the database, skipping: %s", exc
-                )
-                return
             else:
                 raise exc
         names = [description[0] for description in cur.description]

src/mvt/ios/modules/mixed/sms_attachments.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from base64 import b64encode
 from typing import Optional, Union
 
@@ -80,29 +79,21 @@ class SMSAttachments(IOSExtraction):
 
         conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
-        try:
-            cur.execute(
-                """
-                SELECT
-                    attachment.ROWID as "attachment_id",
-                    attachment.*,
-                    message.service as "service",
-                    handle.id as "phone_number"
-                FROM attachment
-                LEFT JOIN message_attachment_join ON
-                    message_attachment_join.attachment_id = attachment.ROWID
-                LEFT JOIN message ON
-                    message.ROWID = message_attachment_join.message_id
-                LEFT JOIN handle ON handle.ROWID = message.handle_id;
+        cur.execute(
             """
-            )
-        except sqlite3.OperationalError as exc:
-            self.log.info(
-                "No SMS attachment tables found in the database, skipping: %s", exc
-            )
-            cur.close()
-            conn.close()
-            return
+            SELECT
+                attachment.ROWID as "attachment_id",
+                attachment.*,
+                message.service as "service",
+                handle.id as "phone_number"
+            FROM attachment
+            LEFT JOIN message_attachment_join ON
+                message_attachment_join.attachment_id = attachment.ROWID
+            LEFT JOIN message ON
+                message.ROWID = message_attachment_join.message_id
+            LEFT JOIN handle ON handle.ROWID = message.handle_id;
+        """
+        )
         names = [description[0] for description in cur.description]
 
         for item in cur:

tests/android/test_artifact_dumpsys_accessibility.py

@@ -25,9 +25,6 @@ class TestDumpsysAccessibilityArtifact:
             da.results[0]["service"]
             == "com.android.settings/com.samsung.android.settings.development.gpuwatch.GPUWatchInterceptor"
         )
-        # All services are installed but none enabled in this fixture
-        for result in da.results:
-            assert result["enabled"] is False
 
     def test_parsing_v14_aosp_format(self):
         da = DumpsysAccessibilityArtifact()
@@ -39,32 +36,7 @@ class TestDumpsysAccessibilityArtifact:
         da.parse(data)
         assert len(da.results) == 1
         assert da.results[0]["package_name"] == "com.malware.accessibility"
-        assert (
-            da.results[0]["service"]
-            == "com.malware.accessibility/com.malware.service.malwareservice"
-        )
-        assert da.results[0]["enabled"] is True
-
-    def test_parsing_installed_and_enabled(self):
-        da = DumpsysAccessibilityArtifact()
-        file = get_artifact("android_data/dumpsys_accessibility_enabled.txt")
-        with open(file) as f:
-            data = f.read()
-
-        assert len(da.results) == 0
-        da.parse(data)
-        assert len(da.results) == 5
-
-        enabled = [r for r in da.results if r["enabled"]]
-        assert len(enabled) == 1
-        assert enabled[0]["package_name"] == "com.samsung.accessibility"
-        assert (
-            enabled[0]["service"]
-            == "com.samsung.accessibility/.universalswitch.UniversalSwitchService  (A11yTool)"
-        )
-
-        not_enabled = [r for r in da.results if not r["enabled"]]
-        assert len(not_enabled) == 4
+        assert da.results[0]["service"] == "com.malware.service.malwareservice"
 
     def test_ioc_check(self, indicator_file):
         da = DumpsysAccessibilityArtifact()

tests/artifacts/android_data/dumpsys_accessibility_enabled.txt

@@ -1,16 +0,0 @@
-ACCESSIBILITY MANAGER (dumpsys accessibility)
-
-currentUserId=0
-User state[
-     attributes:{id=0, touchExplorationEnabled=false, installedServiceCount=5}
-     installed services: {
-      0 : com.google.android.apps.accessibility.voiceaccess/.JustSpeakService  (A11yTool)
-      1 : com.microsoft.appmanager/com.microsoft.mmx.screenmirroringsrc.accessibility.ScreenMirroringAccessibilityService
-      2 : com.samsung.accessibility/.assistantmenu.serviceframework.AssistantMenuService  (A11yTool)
-      3 : com.samsung.accessibility/.universalswitch.UniversalSwitchService  (A11yTool)
-      4 : com.samsung.android.accessibility.talkback/com.samsung.android.marvin.talkback.TalkBackService  (A11yTool)
-     }
-     Bound services:{}
-     Enabled services:{{com.samsung.accessibility/.universalswitch.UniversalSwitchService}}
-     Binding services:{}
-     Crashed services:{}

tests/artifacts/generate_stix.py

@@ -82,7 +82,7 @@ def generate_test_stix_file(file_path):
     for h in sha256:
         i = Indicator(
             indicator_types=["malicious-activity"],
-            pattern="[file:hashes.'SHA-256'='{}']".format(h),
+            pattern="[file:hashes.sha256='{}']".format(h),
             pattern_type="stix",
         )
         res.append(i)
@@ -91,7 +91,7 @@ def generate_test_stix_file(file_path):
     for h in sha1:
         i = Indicator(
             indicator_types=["malicious-activity"],
-            pattern="[file:hashes.'SHA-1'='{}']".format(h),
+            pattern="[file:hashes.sha1='{}']".format(h),
             pattern_type="stix",
         )
         res.append(i)

tests/common/test_indicators.py

@@ -94,78 +94,6 @@ class TestIndicators:
         )
         assert ind.check_file_hash("da0611a300a9ce9aa7a09d1212f203fca5856794")
 
-    def test_parse_stix2_hash_key_variants(self, tmp_path):
-        """STIX2 spec requires single-quoted algorithm names that contain hyphens,
-        e.g. file:hashes.'SHA-256'. Verify MVT accepts both spec-compliant and
-        non-standard lowercase spellings for MD5, SHA-1 and SHA-256."""
-        import json
-
-        sha256_hash = "570cd76bf49cf52e0cb347a68bdcf0590b2eaece134e1b1eba7e8d66261bdbe6"
-        sha1_hash = "da0611a300a9ce9aa7a09d1212f203fca5856794"
-        md5_hash = "d41d8cd98f00b204e9800998ecf8427e"
-
-        variants = [
-            # (pattern_key, expected_bucket)
-            ("file:hashes.'SHA-256'", "files_sha256"),
-            ("file:hashes.SHA-256", "files_sha256"),
-            ("file:hashes.SHA256", "files_sha256"),
-            ("file:hashes.sha256", "files_sha256"),
-            ("file:hashes.'SHA-1'", "files_sha1"),
-            ("file:hashes.SHA-1", "files_sha1"),
-            ("file:hashes.SHA1", "files_sha1"),
-            ("file:hashes.sha1", "files_sha1"),
-            ("file:hashes.MD5", "files_md5"),
-            ("file:hashes.'MD5'", "files_md5"),
-            ("file:hashes.md5", "files_md5"),
-        ]
-
-        hash_for = {
-            "files_sha256": sha256_hash,
-            "files_sha1": sha1_hash,
-            "files_md5": md5_hash,
-        }
-
-        for pattern_key, bucket in variants:
-            h = hash_for[bucket]
-            stix = {
-                "type": "bundle",
-                "id": "bundle--test",
-                "objects": [
-                    {
-                        "type": "malware",
-                        "id": "malware--test",
-                        "name": "TestMalware",
-                        "is_family": False,
-                    },
-                    {
-                        "type": "indicator",
-                        "id": "indicator--test",
-                        "indicator_types": ["malicious-activity"],
-                        "pattern": f"[{pattern_key}='{h}']",
-                        "pattern_type": "stix",
-                        "valid_from": "2024-01-01T00:00:00Z",
-                    },
-                    {
-                        "type": "relationship",
-                        "id": "relationship--test",
-                        "relationship_type": "indicates",
-                        "source_ref": "indicator--test",
-                        "target_ref": "malware--test",
-                    },
-                ],
-            }
-            stix_file = tmp_path / "test.stix2"
-            stix_file.write_text(json.dumps(stix))
-
-            ind = Indicators(log=logging)
-            ind.load_indicators_files([str(stix_file)], load_default=False)
-            assert len(ind.ioc_collections[0][bucket]) == 1, (
-                f"Pattern key '{pattern_key}' was not parsed into '{bucket}'"
-            )
-            assert ind.check_file_hash(h) is not None, (
-                f"check_file_hash failed for pattern key '{pattern_key}'"
-            )
-
     def test_check_android_property(self, indicator_file):
         ind = Indicators(log=logging)
         ind.load_indicators_files([indicator_file], load_default=False)