Update deprecated functions and other small changes (#533)

* also search for STIX2 files in directories in MVT_STIX2

* update datetime deprecations

* add variable declaration in __init__

* add str to return typed in cmd_download_apks.py

* change dictionary creations to dictionary literals

* replace call to set() with set literal

* fix incorrect docstrings

* remove whitespace according to PEP8: E203

* remove whitespace according to PEP8: E203

* remove unreachable return statement

* use Union[] instead of | operator for python 3.8/9 compatability

* Fix ruff formating of files

* Revert "also search for STIX2 files in directories in MVT_STIX2"

This reverts commit 287a11a2ee. We
have this change as a seperate PR in #527.

---------

Co-authored-by: Janik Besendorf <jb@reporter-ohne-grenzen.de>
Co-authored-by: Donncha Ó Cearbhaill <donncha.ocearbhaill@amnesty.org>

src/mvt/android/cmd_download_apks.py

@@ -6,7 +6,7 @@
 import json
 import logging
 import os
-from typing import Callable, Optional
+from typing import Callable, Optional, Union
 
 from rich.progress import track
 
@@ -52,7 +52,9 @@ class DownloadAPKs(AndroidExtraction):
             packages = json.load(handle)
             return cls(packages=packages)
 
-    def pull_package_file(self, package_name: str, remote_path: str) -> None:
+    def pull_package_file(
+        self, package_name: str, remote_path: str
+    ) -> Union[str, None]:
         """Pull files related to specific package from the device.
 
         :param package_name: Name of the package to download

src/mvt/android/modules/androidqf/base.py

@@ -32,6 +32,7 @@ class AndroidQFModule(MVTModule):
             log=log,
             results=results,
         )
+        self.parent_path = None
         self._path: str = target_path
         self.files: List[str] = []
         self.archive: Optional[zipfile.ZipFile] = None

src/mvt/android/modules/bugreport/base.py

@@ -91,5 +91,3 @@ class BugReportModule(MVTModule):
                 return None
 
             return self._get_file_content(dumpstate_logs[0])
-
-        return None

src/mvt/common/updates.py

@@ -64,7 +64,7 @@ class IndicatorsUpdates:
         return 0
 
     def set_latest_check(self) -> None:
-        timestamp = int(datetime.utcnow().timestamp())
+        timestamp = int(datetime.now().timestamp())
         with open(self.latest_check_path, "w", encoding="utf-8") as handle:
             handle.write(str(timestamp))
 
@@ -80,7 +80,7 @@ class IndicatorsUpdates:
         return 0
 
     def set_latest_update(self) -> None:
-        timestamp = int(datetime.utcnow().timestamp())
+        timestamp = int(datetime.now().timestamp())
         with open(self.latest_update_path, "w", encoding="utf-8") as handle:
             handle.write(str(timestamp))
 
@@ -195,7 +195,7 @@ class IndicatorsUpdates:
         return latest_commit_ts
 
     def should_check(self) -> Tuple[bool, int]:
-        now = datetime.utcnow()
+        now = datetime.now()
         latest_check_ts = self.get_latest_check()
         latest_check_dt = datetime.fromtimestamp(latest_check_ts)
 

src/mvt/common/url.py

@@ -334,8 +334,6 @@ class URL:
     def get_domain(self) -> str:
         """Get the domain from a URL.
 
-        :param url: URL to parse
-        :type url: str
         :returns: Domain name extracted from URL
         :rtype: str
 
@@ -349,8 +347,6 @@ class URL:
     def get_top_level(self) -> str:
         """Get only the top-level domain from a URL.
 
-        :param url: URL to parse
-        :type url: str
         :returns: Top-level domain name extracted from URL
         :rtype: str
 

src/mvt/common/utils.py

@@ -53,8 +53,8 @@ def convert_chrometime_to_datetime(timestamp: int) -> datetime.datetime:
 def convert_datetime_to_iso(date_time: datetime.datetime) -> str:
     """Converts datetime to ISO string.
 
-    :param datetime: datetime, naive or timezone aware
+    :param date_time: datetime, naive or timezone aware
-    :type datetime: datetime.datetime
+    :type date_time: datetime.datetime
     :returns: ISO datetime string in YYYY-mm-dd HH:MM:SS.ms format.
     :rtype: str
 
@@ -78,7 +78,7 @@ def convert_unix_to_utc_datetime(
     :returns: datetime.
 
     """
-    return datetime.datetime.utcfromtimestamp(float(timestamp))
+    return datetime.datetime.fromtimestamp(float(timestamp), tz=datetime.timezone.utc)
 
 
 def convert_unix_to_iso(timestamp: Union[int, float, str]) -> str:

src/mvt/ios/modules/backup/manifest.py

@@ -65,9 +65,11 @@ class Manifest(IOSExtraction):
         if "modified" not in record or "status_changed" not in record:
             return records
 
-        for timestamp in set(
+        for timestamp in {
-            [record["created"], record["modified"], record["status_changed"]]
+            record["created"],
-        ):
+            record["modified"],
+            record["status_changed"],
+        }:
             macb = ""
             macb += "M" if timestamp == record["modified"] else "-"
             macb += "-"

src/mvt/ios/modules/fs/analytics.py

@@ -129,8 +129,7 @@ class Analytics(IOSExtraction):
                 data["isodate"] = isodate
             elif row[0]:
                 isodate = convert_mactime_to_iso(row[0], False)
-                data = {}
+                data = {"isodate": isodate}
-                data["isodate"] = isodate
             elif row[1]:
                 isodate = ""
                 data = plistlib.loads(row[1])

src/mvt/ios/modules/mixed/shortcuts.py

@@ -119,9 +119,10 @@ class Shortcuts(IOSExtraction):
                 action_data = plistlib.load(io.BytesIO(shortcut.pop("action_data", [])))
                 actions = []
                 for action_entry in action_data:
-                    action = {}
+                    action = {
-                    action["identifier"] = action_entry["WFWorkflowActionIdentifier"]
+                        "identifier": action_entry["WFWorkflowActionIdentifier"],
-                    action["parameters"] = action_entry["WFWorkflowActionParameters"]
+                        "parameters": action_entry["WFWorkflowActionParameters"],
+                    }
 
                     # URLs might be in multiple fields, do a simple regex search
                     # across the parameters.