Merge branch 'main' into fix/stix2-hash-key-normalization

The STIX2 specification requires single quotes around hash algorithm
names that contain hyphens (e.g. file:hashes.'SHA-256'). MVT only
accepted a non-standard lowercase form (file:hashes.sha256), silently
dropping any indicators using the spec-correct spelling.

Normalize hash algorithm keys in _process_indicator by stripping quotes
and hyphens from the algorithm portion before matching, so all of the
following are accepted for SHA-256, SHA-1 and MD5:

  file:hashes.'SHA-256'   (STIX2 spec)
  file:hashes.SHA-256
  file:hashes.SHA256
  file:hashes.sha256      (previously the only accepted form)

The same normalization is applied to app:cert.* keys.

Update generate_stix.py to use the spec-compliant quoted forms, and add
test_parse_stix2_hash_key_variants to cover all spelling variants.

.github/workflows/add-issue-to-project.yml

@@ -11,7 +11,7 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.5.0
+      - uses: actions/add-to-project@v1
         with:
           # You can target a project in a different organization
           # to the issue

README.md

@@ -4,9 +4,6 @@
 
 # Mobile Verification Toolkit
 
-> [!IMPORTANT]  
-> Soon we will merge the v3 pull request which will result in breaking changes. If you rely on mvt output in other script make sure to the the branch before we merge. More details: https://github.com/mvt-project/mvt/issues/757
-
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
 [![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
 [![CI](https://github.com/mvt-project/mvt/actions/workflows/tests.yml/badge.svg)](https://github.com/mvt-project/mvt/actions/workflows/tests.yml)

src/mvt/common/command.py

@@ -11,7 +11,7 @@ from datetime import datetime
 from typing import Optional
 
 from mvt.common.indicators import Indicators
-from mvt.common.module import EncryptedBackupError, MVTModule, run_module, save_timeline
+from mvt.common.module import MVTModule, run_module, save_timeline
 from mvt.common.utils import (
     convert_datetime_to_iso,
     generate_hashes_from_path,
@@ -244,14 +244,7 @@ class Command:
             except NotImplementedError:
                 pass
 
-            try:
-                run_module(m)
-            except EncryptedBackupError:
-                self.log.critical(
-                    "The backup appears to be encrypted. "
-                    "Please decrypt it first using `mvt-ios decrypt-backup`."
-                )
-                return
+            run_module(m)
 
             self.executed.append(m)
 

src/mvt/common/module.py

@@ -21,10 +21,6 @@ class DatabaseCorruptedError(Exception):
     pass
 
 
-class EncryptedBackupError(Exception):
-    pass
-
-
 class InsufficientPrivileges(Exception):
     pass
 
@@ -173,8 +169,6 @@ def run_module(module: MVTModule) -> None:
 
     try:
         exec_or_profile("module.run()", globals(), locals())
-    except EncryptedBackupError:
-        raise
     except NotImplementedError:
         module.log.exception(
             "The run() procedure of module %s was not implemented yet!",

src/mvt/ios/modules/backup/manifest.py

@@ -8,10 +8,9 @@ import io
 import logging
 import os
 import plistlib
-import sqlite3
 from typing import Optional
 
-from mvt.common.module import DatabaseNotFoundError, EncryptedBackupError
+from mvt.common.module import DatabaseNotFoundError
 from mvt.common.url import URL
 from mvt.common.utils import convert_datetime_to_iso, convert_unix_to_iso
 
@@ -128,14 +127,7 @@ class Manifest(IOSExtraction):
         conn = self._open_sqlite_db(manifest_db_path)
         cur = conn.cursor()
 
-        try:
-            cur.execute("SELECT * FROM Files;")
-        except sqlite3.DatabaseError:
-            conn.close()
-            raise EncryptedBackupError(
-                "Manifest.db is not a valid SQLite database. "
-                "The backup may be encrypted."
-            )
+        cur.execute("SELECT * FROM Files;")
         names = [description[0] for description in cur.description]
 
         for file_entry in cur:

src/mvt/ios/modules/mixed/sms.py

@@ -123,11 +123,6 @@ class SMS(IOSExtraction):
                 """
                 )
                 items = list(cur)
-            elif "no such table" in str(exc):
-                self.log.info(
-                    "No SMS tables found in the database, skipping: %s", exc
-                )
-                return
             else:
                 raise exc
         names = [description[0] for description in cur.description]

src/mvt/ios/modules/mixed/sms_attachments.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import sqlite3
 from base64 import b64encode
 from typing import Optional, Union
 
@@ -80,29 +79,21 @@ class SMSAttachments(IOSExtraction):
 
         conn = self._open_sqlite_db(self.file_path)
         cur = conn.cursor()
-        try:
-            cur.execute(
-                """
-                SELECT
-                    attachment.ROWID as "attachment_id",
-                    attachment.*,
-                    message.service as "service",
-                    handle.id as "phone_number"
-                FROM attachment
-                LEFT JOIN message_attachment_join ON
-                    message_attachment_join.attachment_id = attachment.ROWID
-                LEFT JOIN message ON
-                    message.ROWID = message_attachment_join.message_id
-                LEFT JOIN handle ON handle.ROWID = message.handle_id;
+        cur.execute(
             """
-            )
-        except sqlite3.OperationalError as exc:
-            self.log.info(
-                "No SMS attachment tables found in the database, skipping: %s", exc
-            )
-            cur.close()
-            conn.close()
-            return
+            SELECT
+                attachment.ROWID as "attachment_id",
+                attachment.*,
+                message.service as "service",
+                handle.id as "phone_number"
+            FROM attachment
+            LEFT JOIN message_attachment_join ON
+                message_attachment_join.attachment_id = attachment.ROWID
+            LEFT JOIN message ON
+                message.ROWID = message_attachment_join.message_id
+            LEFT JOIN handle ON handle.ROWID = message.handle_id;
+        """
+        )
         names = [description[0] for description in cur.description]
 
         for item in cur: