From 57647583cc2ed793d96e55cf57f81a53745e09ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 04:17:03 +0100 Subject: [PATCH 01/24] Add new iOS versions and build numbers (#569) Co-authored-by: DonnchaC --- src/mvt/ios/data/ios_versions.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index b297997..efc8584 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1083,5 +1083,9 @@ { "version": "18.0.1", "build": "22A3370" + }, + { + "version": "18.1", + "build": "22B83" } ] \ No newline at end of file From 7e4f0aec4d71771b9912ffe5289746d81c517daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 31 Oct 2024 19:59:29 +0100 Subject: [PATCH 02/24] Fix error to due extra equal character in Files detection --- src/mvt/android/modules/androidqf/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mvt/android/modules/androidqf/files.py b/src/mvt/android/modules/androidqf/files.py index 18dee9d..fde7ca1 100644 --- a/src/mvt/android/modules/androidqf/files.py +++ b/src/mvt/android/modules/androidqf/files.py @@ -74,7 +74,7 @@ class Files(AndroidQFModule): for result in self.results: ioc = self.indicators.check_file_path(result["path"]) if ioc: - result["matched_indicator"] == ioc + result["matched_indicator"] = ioc self.detected.append(result) continue From 22fce280af76e99ff4c6af71aeafd8039747a510 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:02:09 +0100 Subject: [PATCH 03/24] Add new iOS versions and build numbers (#572) Co-authored-by: DonnchaC --- src/mvt/ios/data/ios_versions.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index efc8584..7200c9c 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1087,5 +1087,9 @@ { "version": "18.1", "build": "22B83" + }, + { + "version": "18.1.1", + "build": "22B91" } ] \ No newline at end of file From 9d81b5bfa8930f1fd6549fb58f7ef591642af83d Mon Sep 17 00:00:00 2001 From: tes Date: Wed, 11 Dec 2024 16:47:19 -0300 Subject: [PATCH 04/24] Add a module to parse uninstalled apps from dumpsys data, for both bugreport and AndroidQF output, and match them against package name IoCs. --- .../artifacts/dumpsys_platform_compat.py | 44 +++++++++++++++++ src/mvt/android/modules/androidqf/__init__.py | 2 + .../androidqf/dumpsys_platform_compat.py | 44 +++++++++++++++++ src/mvt/android/modules/bugreport/__init__.py | 2 + .../modules/bugreport/platform_compat.py | 48 +++++++++++++++++++ src/mvt/common/indicators.py | 1 + .../test_artifact_dumpsys_platform_compat.py | 40 ++++++++++++++++ .../test_dumpsys_platform_compat.py | 23 +++++++++ .../android_data/bugreport/dumpstate.txt | 19 +++++++- .../android_data/dumpsys_platform_compat.txt | 16 +++++++ tests/artifacts/androidqf/dumpsys.txt | 18 +++++++ tests/common/test_utils.py | 2 +- 12 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 src/mvt/android/artifacts/dumpsys_platform_compat.py create mode 100644 src/mvt/android/modules/androidqf/dumpsys_platform_compat.py create mode 100644 src/mvt/android/modules/bugreport/platform_compat.py create mode 100644 tests/android/test_artifact_dumpsys_platform_compat.py create mode 100644 tests/android_androidqf/test_dumpsys_platform_compat.py create mode 100644 tests/artifacts/android_data/dumpsys_platform_compat.txt diff --git a/src/mvt/android/artifacts/dumpsys_platform_compat.py b/src/mvt/android/artifacts/dumpsys_platform_compat.py new file mode 100644 index 0000000..60152f6 --- /dev/null +++ b/src/mvt/android/artifacts/dumpsys_platform_compat.py @@ -0,0 +1,44 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from .artifact import AndroidArtifact + + +class DumpsysPlatformCompatArtifact(AndroidArtifact): + """ + Parser for uninstalled apps listed in platform_compat section. + """ + + def check_indicators(self) -> None: + if not self.indicators: + return + + for result in self.results: + ioc = self.indicators.check_app_id(result["package_name"]) + if ioc: + result["matched_indicator"] = ioc + self.detected.append(result) + continue + + def parse(self, data: str) -> None: + for line in data.splitlines(): + if not line.startswith("ChangeId(168419799; name=DOWNSCALED;"): + continue + + if line.strip() == "": + break + + # Look for rawOverrides field + if "rawOverrides={" in line: + # Extract the content inside the braces for rawOverrides + overrides_field = line.split("rawOverrides={", 1)[1].split("};", 1)[0] + + for entry in overrides_field.split(", "): + # Extract app name + uninstall_app = entry.split("=")[0].strip() + + self.results.append( + {"package_name": uninstall_app} + ) \ No newline at end of file diff --git a/src/mvt/android/modules/androidqf/__init__.py b/src/mvt/android/modules/androidqf/__init__.py index c1c7548..cdb0af8 100644 --- a/src/mvt/android/modules/androidqf/__init__.py +++ b/src/mvt/android/modules/androidqf/__init__.py @@ -14,6 +14,7 @@ from .dumpsys_receivers import DumpsysReceivers from .dumpsys_adb import DumpsysADBState from .getprop import Getprop from .packages import Packages +from .dumpsys_platform_compat import DumpsysPlatformCompat from .processes import Processes from .settings import Settings from .sms import SMS @@ -29,6 +30,7 @@ ANDROIDQF_MODULES = [ DumpsysBatteryHistory, DumpsysADBState, Packages, + DumpsysPlatformCompat, Processes, Getprop, Settings, diff --git a/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py b/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py new file mode 100644 index 0000000..b0e5e0a --- /dev/null +++ b/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py @@ -0,0 +1,44 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +from typing import Optional + +from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact + +from .base import AndroidQFModule + + +class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, AndroidQFModule): + """This module extracts details on uninstalled apps.""" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def run(self) -> None: + dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt") + if not dumpsys_file: + return + + data = self._get_file_content(dumpsys_file[0]).decode("utf-8", errors="replace") + content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:") + self.parse(content) + + self.log.info("Found %d uninstalled apps", len(self.results)) \ No newline at end of file diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 119c958..73dd852 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -11,6 +11,7 @@ from .battery_history import BatteryHistory from .dbinfo import DBInfo from .getprop import Getprop from .packages import Packages +from .platform_compat import PlatformCompat from .receivers import Receivers from .adb_state import DumpsysADBState @@ -23,6 +24,7 @@ BUGREPORT_MODULES = [ DBInfo, Getprop, Packages, + PlatformCompat, Receivers, DumpsysADBState, ] diff --git a/src/mvt/android/modules/bugreport/platform_compat.py b/src/mvt/android/modules/bugreport/platform_compat.py new file mode 100644 index 0000000..c08bab4 --- /dev/null +++ b/src/mvt/android/modules/bugreport/platform_compat.py @@ -0,0 +1,48 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +from typing import Optional + +from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact + +from mvt.android.modules.bugreport.base import BugReportModule + + +class PlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): + """This module extracts details on uninstalled apps.""" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def run(self) -> None: + data = self._get_dumpstate_file() + if not data: + self.log.error( + "Unable to find dumpstate file. " + "Did you provide a valid bug report archive?" + ) + return + + data = data.decode("utf-8", errors="replace") + content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:") + self.parse(content) + + self.log.info("Found %d uninstalled apps", len(self.results)) \ No newline at end of file diff --git a/src/mvt/common/indicators.py b/src/mvt/common/indicators.py index a73938d..56ab1f8 100644 --- a/src/mvt/common/indicators.py +++ b/src/mvt/common/indicators.py @@ -324,6 +324,7 @@ class Indicators: def get_iocs(self, ioc_type: str) -> Iterator[Dict[str, Any]]: for ioc_collection in self.ioc_collections: + for ioc in ioc_collection.get(ioc_type, []): yield { "value": ioc, diff --git a/tests/android/test_artifact_dumpsys_platform_compat.py b/tests/android/test_artifact_dumpsys_platform_compat.py new file mode 100644 index 0000000..e2321a4 --- /dev/null +++ b/tests/android/test_artifact_dumpsys_platform_compat.py @@ -0,0 +1,40 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +import logging + +from mvt.android.artifacts.dumpsys_platform_compat import DumpsysPlatformCompatArtifact +from mvt.common.indicators import Indicators + +from ..utils import get_artifact + + +class TestDumpsysPlatformCompatArtifact: + def test_parsing(self): + dbi = DumpsysPlatformCompatArtifact() + file = get_artifact("android_data/dumpsys_platform_compat.txt") + with open(file) as f: + data = f.read() + + assert len(dbi.results) == 0 + dbi.parse(data) + assert len(dbi.results) == 2 + assert dbi.results[0]["package_name"] == "org.torproject.torbrowser" + assert dbi.results[1]["package_name"] == "org.article19.circulo.next" + + def test_ioc_check(self, indicator_file): + dbi = DumpsysPlatformCompatArtifact() + file = get_artifact("android_data/dumpsys_platform_compat.txt") + with open(file) as f: + data = f.read() + dbi.parse(data) + + ind = Indicators(log=logging.getLogger()) + ind.parse_stix2(indicator_file) + ind.ioc_collections[0]["app_ids"].append("org.torproject.torbrowser") + ind.ioc_collections[0]["app_ids"].append("org.article19.circulo.next") + dbi.indicators = ind + assert len(dbi.detected) == 0 + dbi.check_indicators() + assert len(dbi.detected) == 2 diff --git a/tests/android_androidqf/test_dumpsys_platform_compat.py b/tests/android_androidqf/test_dumpsys_platform_compat.py new file mode 100644 index 0000000..8123432 --- /dev/null +++ b/tests/android_androidqf/test_dumpsys_platform_compat.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from pathlib import Path + +from mvt.android.modules.androidqf.dumpsys_platform_compat import DumpsysPlatformCompat +from mvt.common.module import run_module + +from ..utils import get_android_androidqf, list_files + + +class TestDumpsysPlatformCompatModule: + def test_parsing(self): + data_path = get_android_androidqf() + m = DumpsysPlatformCompat(target_path=data_path) + files = list_files(data_path) + parent_path = Path(data_path).absolute().parent.as_posix() + m.from_folder(parent_path, files) + run_module(m) + assert len(m.results) == 2 + assert len(m.detected) == 0 diff --git a/tests/artifacts/android_data/bugreport/dumpstate.txt b/tests/artifacts/android_data/bugreport/dumpstate.txt index b30d30b..c888ed5 100644 --- a/tests/artifacts/android_data/bugreport/dumpstate.txt +++ b/tests/artifacts/android_data/bugreport/dumpstate.txt @@ -246,6 +246,23 @@ Packages: com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivity com.instagram.share.handleractivity.ClipsShareHandlerActivity com.instagram.direct.share.handler.DirectMultipleExternalMediaShareActivityInterop - +--------- 0.053s was the duration of dumpsys appops, ending at: 2022-03-29 23:14:27 +------------------------------------------------------------------------------- +DUMP OF SERVICE platform_compat: +ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable) +ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable) +ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30) +ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30) +ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled) +ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35) +ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33) +ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35) +ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33) +ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31) +ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable) +ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled) +ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34) +ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled) +ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable) diff --git a/tests/artifacts/android_data/dumpsys_platform_compat.txt b/tests/artifacts/android_data/dumpsys_platform_compat.txt new file mode 100644 index 0000000..13cc17c --- /dev/null +++ b/tests/artifacts/android_data/dumpsys_platform_compat.txt @@ -0,0 +1,16 @@ +DUMP OF SERVICE platform_compat: +ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable) +ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable) +ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30) +ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30) +ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled) +ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35) +ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33) +ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35) +ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33) +ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31) +ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable) +ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled) +ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34) +ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled) +ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable) \ No newline at end of file diff --git a/tests/artifacts/androidqf/dumpsys.txt b/tests/artifacts/androidqf/dumpsys.txt index 2f21ffa..0deb4cf 100644 --- a/tests/artifacts/androidqf/dumpsys.txt +++ b/tests/artifacts/androidqf/dumpsys.txt @@ -379,4 +379,22 @@ Daily stats: Update com.google.android.projection.gearhead vers=99632623 Update com.google.android.projection.gearhead vers=99632623 Update com.google.android.projection.gearhead vers=99632623 +--------- 0.053s was the duration of dumpsys batterystats, ending at: 2024-03-21 11:07:22 +------------------------------------------------------------------------------- +DUMP OF SERVICE platform_compat: +ChangeId(180326845; name=OVERRIDE_MIN_ASPECT_RATIO_MEDIUM; disabled; overridable) +ChangeId(189969744; name=DOWNSCALE_65; disabled; overridable) +ChangeId(183372781; name=ENABLE_RAW_SYSTEM_GALLERY_ACCESS; enableSinceTargetSdk=30) +ChangeId(150939131; name=ADD_CONTENT_OBSERVER_FLAGS; enableSinceTargetSdk=30) +ChangeId(226439802; name=SCHEDULE_EXACT_ALARM_DENIED_BY_DEFAULT; disabled) +ChangeId(270674727; name=ENABLE_STRICT_FORMATTER_VALIDATION; enableSinceTargetSdk=35) +ChangeId(183155436; name=ALWAYS_USE_CONTEXT_USER; enableSinceTargetSdk=33) +ChangeId(303742236; name=ROLE_MANAGER_USER_HANDLE_AWARE; enableSinceTargetSdk=35) +ChangeId(203800354; name=MEDIA_CONTROL_SESSION_ACTIONS; enableSinceTargetSdk=33) +ChangeId(144027538; name=BLOCK_GPS_STATUS_USAGE; enableSinceTargetSdk=31) +ChangeId(189969749; name=DOWNSCALE_35; disabled; overridable) +ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled) +ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34) +ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled) +ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable) diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index c97b9c0..d1058e5 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -75,7 +75,7 @@ class TestHashes: # This needs to be updated when we add or edit files in AndroidQF folder assert ( hashes[1]["sha256"] - == "1bd255f656a7f9d5647a730f0f0cc47053115576f11532d41bf28c16635b193d" + == "9fb6396b64cfff30e2a459a64496d3c1386926d09edd68be2d878de45fa7b3a9" ) From 4bcc0e5f27b9b3d3425cf17f41603dbbf6f3ccfb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:43:59 +0100 Subject: [PATCH 05/24] Add new iOS versions and build numbers (#583) Co-authored-by: DonnchaC --- src/mvt/ios/data/ios_versions.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index 7200c9c..a9d3f16 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1091,5 +1091,9 @@ { "version": "18.1.1", "build": "22B91" + }, + { + "version": "18.2", + "build": "22C152" } ] \ No newline at end of file From d3fcc686ffc7705fabb8e9b3d01be63748aa79a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Fri, 13 Dec 2024 22:38:40 +0100 Subject: [PATCH 06/24] Update contribution guidelines --- CONTRIBUTING.md | 62 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ee1666..308f00e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,19 +1,65 @@ -# Contributing +# Contributing to Mobile Verification Toolkit (MVT) -Thank you for your interest in contributing to Mobile Verification Toolkit (MVT)! Your help is very much appreciated. +We greatly appreciate contributions to MVT! + +Your involvement, whether through identifying issues, improving functionality, or enhancing documentation, is very much appreciated. To ensure smooth collaboration and a welcoming environment, we've outlined some key guidelines for contributing below. + +## Getting started + +Contributing to an open-source project like MVT might seem overwhelming at first, but we're here to support you! + + Whether you're a technologist, a frontline human rights defender, a field researcher, or someone new to consensual spyware forensics, there are many ways to make meaningful contributions. + + Here's how you can get started: + +1. **Explore the codebase:** + - Browse the repository to get familar with MVT. Many MVT modules are simple in functionality and easy to understand. + - Look for `TODO:` or `FIXME:` comments in the code for areas that need attention. + +2. **Check Github issues:** + - Look for issues tagged with ["help wanted"](https://github.com/mvt-project/mvt/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) or ["good first issue"](https://github.com/mvt-project/mvt/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) to find tasks that are beginner-friendly or where input from the community would be helpful. + +3. **Ask for guidance:** + + - If you're unsure where to start, feel free to open a [discussion](https://github.com/mvt-project/mvt/discussions) or comment on an issue. + +## How to contribute: + +1. **Report issues:** + + - Found a bug? Please check existing issues to see if it's already reported. If not, open a new issue. Mobile operating systems and databases are constantly evolving, an new errors may appear spontaniously in new app versions. + + **Please provide as much information as possible about the prodblem including: any error messages, steps to reproduce the problem, and any logs or screenshots that can help.** -## Where to start +2. **Suggest features:** + - If you have an idea for new functionality, create a feature request issue and describe your proposal. -Starting to contribute to a somewhat complex project like MVT might seem intimidating. Unless you have specific ideas of new functionality you would like to submit, some good starting points are searching for `TODO:` and `FIXME:` comments throughout the code. Alternatively you can check if any GitHub issues existed marked with the ["help wanted"](https://github.com/mvt-project/mvt/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) tag. +3. **Submit code:** + - Fork the repository and create a new branch for your changes. + - Ensure your changes align with the code style guidelines (see below). + - Open a pull request (PR) with a clear description of your changes and link it to any relevant issues. +4. **Documentation contributions:** + - Improving documentation is just as valuable as contributing code! If you notice gaps or inaccuracies in the documentation, feel free to submit changes or suggest updates. ## Code style +Please follow these code style guidelines for consistency and readability: -When contributing code to +- **Indentation**: use 4 spaces per tab. +- **Quotes**: Use double quotes (`"`) by default. Use single quotes (`'`) for nested strings instead of escaping (`\"`), or when using f-formatting. +- **Maximum line length**: + - Aim for lines no longer than 80 characters. + - Exceptions are allowed for long log lines or strings, which may extend up to 100 characters. + - Wrap lines that exceed 100 characters. -- **Indentation**: we use 4-spaces tabs. +Follow [PEP 8 guidelines](https://peps.python.org/pep-0008/) for indentation and overall Python code style. All MVT code is automatically linted with [Ruff](https://github.com/astral-sh/ruff) before merging. -- **Quotes**: we use double quotes (`"`) as a default. Single quotes (`'`) can be favored with nested strings instead of escaping (`\"`), or when using f-formatting. +Please check your code before opening a pull request by running `make ruff` -- **Maximum line length**: we strongly encourage to respect a 80 characters long lines and to follow [PEP8 indentation guidelines](https://peps.python.org/pep-0008/#indentation) when having to wrap. However, if breaking at 80 is not possible or is detrimental to the readability of the code, exceptions are tolerated. For example, long log lines, or long strings can be extended to 100 characters long. Please hard wrap anything beyond 100 characters. + +## Community and support + +We aim to create a supportive and collaborative environment for all contributors. If you run into any challenges, feel free to reach out through the discussions or issues section of the repository. + +Your contributions, big or small, help improve MVT and are always appreciated. \ No newline at end of file From 5b2fe3baec0115dafae6991ce5089878720dc4c5 Mon Sep 17 00:00:00 2001 From: Tek Date: Sat, 14 Dec 2024 10:04:47 +0100 Subject: [PATCH 07/24] Reorganize code in iOS app module (#586) --- src/mvt/ios/modules/mixed/applications.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/mvt/ios/modules/mixed/applications.py b/src/mvt/ios/modules/mixed/applications.py index 3eb9593..8c15130 100644 --- a/src/mvt/ios/modules/mixed/applications.py +++ b/src/mvt/ios/modules/mixed/applications.py @@ -17,6 +17,12 @@ from mvt.ios.modules.base import IOSExtraction APPLICATIONS_DB_PATH = [ "private/var/containers/Bundle/Application/*/iTunesMetadata.plist" ] +KNOWN_APP_INSTALLERS = [ + "com.apple.AppStore", + "com.apple.AppStore.ProductPageExtension", + "com.apple.dmd", + "dmd", +] class Applications(IOSExtraction): @@ -80,12 +86,10 @@ class Applications(IOSExtraction): self.detected.append(result) continue # Some apps installed from apple store with sourceApp "com.apple.AppStore.ProductPageExtension" - if result.get("sourceApp", "com.apple.AppStore") not in [ - "com.apple.AppStore", - "com.apple.AppStore.ProductPageExtension", - "com.apple.dmd", - "dmd", - ]: + if ( + result.get("sourceApp", "com.apple.AppStore") + not in KNOWN_APP_INSTALLERS + ): self.log.warning( "Suspicious app not installed from the App Store or MDM: %s", result["softwareVersionBundleId"], From 3da61c8da8706b1c10e8e12c75cd6bcf87932610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Sun, 15 Dec 2024 23:22:36 +0100 Subject: [PATCH 08/24] Fix ruff checks --- src/mvt/android/artifacts/dumpsys_platform_compat.py | 6 ++---- .../android/modules/androidqf/dumpsys_platform_compat.py | 2 +- src/mvt/android/modules/bugreport/platform_compat.py | 2 +- src/mvt/common/indicators.py | 1 - 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_platform_compat.py b/src/mvt/android/artifacts/dumpsys_platform_compat.py index 60152f6..e1037f0 100644 --- a/src/mvt/android/artifacts/dumpsys_platform_compat.py +++ b/src/mvt/android/artifacts/dumpsys_platform_compat.py @@ -29,7 +29,7 @@ class DumpsysPlatformCompatArtifact(AndroidArtifact): if line.strip() == "": break - + # Look for rawOverrides field if "rawOverrides={" in line: # Extract the content inside the braces for rawOverrides @@ -39,6 +39,4 @@ class DumpsysPlatformCompatArtifact(AndroidArtifact): # Extract app name uninstall_app = entry.split("=")[0].strip() - self.results.append( - {"package_name": uninstall_app} - ) \ No newline at end of file + self.results.append({"package_name": uninstall_app}) diff --git a/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py b/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py index b0e5e0a..869c476 100644 --- a/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py +++ b/src/mvt/android/modules/androidqf/dumpsys_platform_compat.py @@ -41,4 +41,4 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, AndroidQFModule): content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:") self.parse(content) - self.log.info("Found %d uninstalled apps", len(self.results)) \ No newline at end of file + self.log.info("Found %d uninstalled apps", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/platform_compat.py b/src/mvt/android/modules/bugreport/platform_compat.py index c08bab4..fadac92 100644 --- a/src/mvt/android/modules/bugreport/platform_compat.py +++ b/src/mvt/android/modules/bugreport/platform_compat.py @@ -45,4 +45,4 @@ class PlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:") self.parse(content) - self.log.info("Found %d uninstalled apps", len(self.results)) \ No newline at end of file + self.log.info("Found %d uninstalled apps", len(self.results)) diff --git a/src/mvt/common/indicators.py b/src/mvt/common/indicators.py index 56ab1f8..a73938d 100644 --- a/src/mvt/common/indicators.py +++ b/src/mvt/common/indicators.py @@ -324,7 +324,6 @@ class Indicators: def get_iocs(self, ioc_type: str) -> Iterator[Dict[str, Any]]: for ioc_collection in self.ioc_collections: - for ioc in ioc_collection.get(ioc_type, []): yield { "value": ioc, From 154e6dab153271dbd1cfd7a5e247ef475a3090d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Tue, 24 Dec 2024 23:30:18 +0000 Subject: [PATCH 09/24] Add config file parser for MVT --- src/mvt/common/config.py | 91 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/mvt/common/config.py diff --git a/src/mvt/common/config.py b/src/mvt/common/config.py new file mode 100644 index 0000000..4341d87 --- /dev/null +++ b/src/mvt/common/config.py @@ -0,0 +1,91 @@ +import os +import uuid +import yaml +import json + +from typing import Tuple, Type +from appdirs import user_config_dir +from pydantic import AnyHttpUrl, Field +from pydantic_settings import ( + BaseSettings, + InitSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +MVT_CONFIG_FOLDER = user_config_dir("mvt") +MVT_CONFIG_PATH = os.path.join(MVT_CONFIG_FOLDER, "config.yaml") + + +class MVTSettings(BaseSettings): + model_config = SettingsConfigDict( + env_prefix="MVT_", env_nested_delimiter="_", extra="ignore" + ) + # Allow to decided if want to load environment variables + load_env: bool = Field(True, exclude=True) + + # General settings + PYPI_UPDATE_URL: AnyHttpUrl = Field( + "https://pypi.org/pypi/mvt/json", + validate_default=False, + ) + INDICATORS_UPDATE_URL: AnyHttpUrl = Field( + default="https://raw.githubusercontent.com/mvt-project/mvt-indicators/main/indicators.yaml", + validate_default=False, + ) + NETWORK_ACCESS_ALLOWED: bool = True + NETWORK_TIMEOUT: int = 15 + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: InitSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + sources = ( + YamlConfigSettingsSource(settings_cls, MVT_CONFIG_PATH), + init_settings, + ) + # Load env variables if enabled + if init_settings.init_kwargs.get("load_env", True): + sources = (env_settings,) + sources + return sources + + def save_settings( + self, + ) -> None: + """ + Save the current settings to a file. + """ + if not os.path.isdir(MVT_CONFIG_FOLDER): + os.makedirs(MVT_CONFIG_FOLDER) + + # Dump the settings to the YAML file + model_serializable = json.loads(self.model_dump_json(exclude_defaults=True)) + with open(MVT_CONFIG_PATH, "w") as config_file: + config_file.write(yaml.dump(model_serializable, default_flow_style=False)) + + @classmethod + def initialise(cls) -> "MVTSettings": + """ + Initialise the settings file. + + We first initialise the settings (without env variable) and then persist + them to file. This way we can update the config file with the default values. + + Afterwards we load the settings again, this time including the env variables. + """ + # Set invalid env prefix to avoid loading env variables. + settings = MVTSettings(load_env=False) + settings.save_settings() + + # Load the settings again with any ENV variables. + settings = MVTSettings(load_env=True) + return settings + + +settings = MVTSettings.initialise() From 28c0c86c4efea090e642f36ed2a7d44be77adfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 25 Dec 2024 00:09:29 +0000 Subject: [PATCH 10/24] Update MVT code to use config file rather than raw env variables --- src/mvt/common/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mvt/common/config.py b/src/mvt/common/config.py index 4341d87..8a116e2 100644 --- a/src/mvt/common/config.py +++ b/src/mvt/common/config.py @@ -1,5 +1,4 @@ import os -import uuid import yaml import json From f4425865c0e51de5872afd08cf1ec737a80b57a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 25 Dec 2024 00:14:14 +0000 Subject: [PATCH 11/24] Add missed modules using updated settings module --- src/mvt/android/modules/backup/helpers.py | 17 +++++++-------- src/mvt/common/command.py | 5 +++-- src/mvt/common/config.py | 25 ++++++++++++++++++----- src/mvt/common/indicators.py | 7 ++++--- src/mvt/common/updates.py | 3 ++- src/mvt/common/utils.py | 2 +- tests/common/test_indicators.py | 4 ++++ tests/test_check_android_androidqf.py | 4 ++++ tests/test_check_android_backup.py | 4 ++++ 9 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/mvt/android/modules/backup/helpers.py b/src/mvt/android/modules/backup/helpers.py index 5ddb80d..3e48078 100644 --- a/src/mvt/android/modules/backup/helpers.py +++ b/src/mvt/android/modules/backup/helpers.py @@ -3,10 +3,11 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import os from rich.prompt import Prompt +from mvt.common.config import settings + MVT_ANDROID_BACKUP_PASSWORD = "MVT_ANDROID_BACKUP_PASSWORD" @@ -16,24 +17,24 @@ def cli_load_android_backup_password(log, backup_password): Used in MVT CLI command parsers. """ - password_from_env = os.environ.get(MVT_ANDROID_BACKUP_PASSWORD, None) + password_from_env_or_config = settings.ANDROID_BACKUP_PASSWORD if backup_password: log.info( "Your password may be visible in the process table because it " "was supplied on the command line!" ) - if password_from_env: + if password_from_env_or_config: log.info( "Ignoring %s environment variable, using --backup-password argument instead", - MVT_ANDROID_BACKUP_PASSWORD, + "MVT_ANDROID_BACKUP_PASSWORD", ) return backup_password - elif password_from_env: + elif password_from_env_or_config: log.info( - "Using backup password from %s environment variable", - MVT_ANDROID_BACKUP_PASSWORD, + "Using backup password from %s environment variable or config file", + "MVT_ANDROID_BACKUP_PASSWORD", ) - return password_from_env + return password_from_env_or_config def prompt_or_load_android_backup_password(log, module_options): diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index b44a56b..527c21a 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -17,6 +17,7 @@ from mvt.common.utils import ( generate_hashes_from_path, get_sha256_from_file_path, ) +from mvt.common.config import settings from mvt.common.version import MVT_VERSION @@ -132,7 +133,7 @@ class Command: if ioc_file_path and ioc_file_path not in info["ioc_files"]: info["ioc_files"].append(ioc_file_path) - if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes): + if self.target_path and (settings.HASH_FILES or self.hashes): self.generate_hashes() info["hashes"] = self.hash_values @@ -141,7 +142,7 @@ class Command: with open(info_path, "w+", encoding="utf-8") as handle: json.dump(info, handle, indent=4) - if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes): + if self.target_path and (settings.HASH_FILES or self.hashes): info_hash = get_sha256_from_file_path(info_path) self.log.info('Reference hash of the info.json file: "%s"', info_hash) diff --git a/src/mvt/common/config.py b/src/mvt/common/config.py index 8a116e2..b2fcf6d 100644 --- a/src/mvt/common/config.py +++ b/src/mvt/common/config.py @@ -19,7 +19,10 @@ MVT_CONFIG_PATH = os.path.join(MVT_CONFIG_FOLDER, "config.yaml") class MVTSettings(BaseSettings): model_config = SettingsConfigDict( - env_prefix="MVT_", env_nested_delimiter="_", extra="ignore" + env_prefix="MVT_", + env_nested_delimiter="_", + extra="ignore", + nested_model_default_partial_updates=True, ) # Allow to decided if want to load environment variables load_env: bool = Field(True, exclude=True) @@ -29,13 +32,25 @@ class MVTSettings(BaseSettings): "https://pypi.org/pypi/mvt/json", validate_default=False, ) - INDICATORS_UPDATE_URL: AnyHttpUrl = Field( - default="https://raw.githubusercontent.com/mvt-project/mvt-indicators/main/indicators.yaml", - validate_default=False, - ) NETWORK_ACCESS_ALLOWED: bool = True NETWORK_TIMEOUT: int = 15 + # Command default settings, all can be specified by MVT_ prefixed environment variables too. + IOS_BACKUP_PASSWORD: str | None = Field( + None, description="Default password to use to decrypt iOS backups" + ) + ANDROID_BACKUP_PASSWORD: str | None = Field( + None, description="Default password to use to decrypt Android backups" + ) + STIX2: str | None = Field( + None, description="List of directories where STIX2 files are stored" + ) + VT_API_KEY: str | None = Field( + None, description="API key to use for VirusTotal lookups" + ) + PROFILE: bool = Field(False, description="Profile the execution of MVT modules") + HASH_FILES: bool = Field(False, description="Should MVT hash output files") + @classmethod def settings_customise_sources( cls, diff --git a/src/mvt/common/indicators.py b/src/mvt/common/indicators.py index a73938d..6bcce2e 100644 --- a/src/mvt/common/indicators.py +++ b/src/mvt/common/indicators.py @@ -14,6 +14,7 @@ import ahocorasick from appdirs import user_data_dir from .url import URL +from .config import settings MVT_DATA_FOLDER = user_data_dir("mvt") MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators") @@ -41,12 +42,12 @@ class Indicators: def _check_stix2_env_variable(self) -> None: """ - Checks if a variable MVT_STIX2 contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2 + Checks if MVT_STIX2 setting or environment variable contains path to a STIX file. Also recursively searches through dirs in MVT_STIX2 """ - if "MVT_STIX2" not in os.environ: + if not settings.STIX2: return - paths = os.environ["MVT_STIX2"].split(":") + paths = settings.STIX2.split(":") for path in paths: if os.path.isfile(path) and path.lower().endswith(".stix2"): self.parse_stix2(path) diff --git a/src/mvt/common/updates.py b/src/mvt/common/updates.py index c82a3f8..e782b91 100644 --- a/src/mvt/common/updates.py +++ b/src/mvt/common/updates.py @@ -14,6 +14,7 @@ from packaging import version from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER from .version import MVT_VERSION +from .config import settings log = logging.getLogger(__name__) @@ -23,7 +24,7 @@ INDICATORS_CHECK_FREQUENCY = 12 class MVTUpdates: def check(self) -> str: - res = requests.get("https://pypi.org/pypi/mvt/json", timeout=15) + res = requests.get(settings.PYPI_UPDATE_URL, timeout=15) data = res.json() latest_version = data.get("info", {}).get("version", "") diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index 0a64521..b7f5f83 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -256,7 +256,7 @@ def set_verbose_logging(verbose: bool = False): def exec_or_profile(module, globals, locals): """Hook for profiling MVT modules""" - if int(os.environ.get("MVT_PROFILE", False)): + if settings.PROFILE: cProfile.runctx(module, globals, locals) else: exec(module, globals, locals) diff --git a/tests/common/test_indicators.py b/tests/common/test_indicators.py index 9a687c0..efc24f7 100644 --- a/tests/common/test_indicators.py +++ b/tests/common/test_indicators.py @@ -6,6 +6,8 @@ import logging import os + +from mvt.common.config import settings from mvt.common.indicators import Indicators from ..utils import get_artifact_folder @@ -100,6 +102,8 @@ class TestIndicators: def test_env_stix(self, indicator_file): os.environ["MVT_STIX2"] = indicator_file + settings.__init__() # Reset settings + ind = Indicators(log=logging) ind.load_indicators_files([], load_default=False) assert ind.total_ioc_count == 9 diff --git a/tests/test_check_android_androidqf.py b/tests/test_check_android_androidqf.py index 29b4b99..167b5b7 100644 --- a/tests/test_check_android_androidqf.py +++ b/tests/test_check_android_androidqf.py @@ -8,6 +8,7 @@ import os from click.testing import CliRunner from mvt.android.cli import check_androidqf +from mvt.common.config import settings from .utils import get_artifact_folder @@ -56,6 +57,8 @@ class TestCheckAndroidqfCommand: ) os.environ["MVT_ANDROID_BACKUP_PASSWORD"] = TEST_BACKUP_PASSWORD + settings.__init__() # Reset settings + runner = CliRunner() path = os.path.join(get_artifact_folder(), "androidqf_encrypted") result = runner.invoke(check_androidqf, [path]) @@ -63,3 +66,4 @@ class TestCheckAndroidqfCommand: assert prompt_mock.call_count == 0 assert result.exit_code == 0 del os.environ["MVT_ANDROID_BACKUP_PASSWORD"] + settings.__init__() # Reset settings diff --git a/tests/test_check_android_backup.py b/tests/test_check_android_backup.py index 7ccd8ec..71c0586 100644 --- a/tests/test_check_android_backup.py +++ b/tests/test_check_android_backup.py @@ -9,6 +9,7 @@ import os from click.testing import CliRunner from mvt.android.cli import check_backup +from mvt.common.config import settings from .utils import get_artifact_folder @@ -63,6 +64,8 @@ class TestCheckAndroidBackupCommand: ) os.environ["MVT_ANDROID_BACKUP_PASSWORD"] = TEST_BACKUP_PASSWORD + settings.__init__() # Reset settings + runner = CliRunner() path = os.path.join(get_artifact_folder(), "androidqf_encrypted/backup.ab") result = runner.invoke(check_backup, [path]) @@ -70,3 +73,4 @@ class TestCheckAndroidBackupCommand: assert prompt_mock.call_count == 0 assert result.exit_code == 0 del os.environ["MVT_ANDROID_BACKUP_PASSWORD"] + settings.__init__() # Reset settings From 0f1eec39719e796b37be8971cff4cf45a2405846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 25 Dec 2024 00:21:42 +0000 Subject: [PATCH 12/24] Add Pydantic dependencies --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b2faba7..63c3c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ "cryptography >=42.0.5", "pyyaml >=6.0", "pyahocorasick >= 2.0.0", + "pydantic >= 2.10.0", + "pydantic-settings >= 2.7.0", ] requires-python = ">= 3.8" From 52e854b8b7302ddc65c759b68344ea41ba39e45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 25 Dec 2024 00:23:36 +0000 Subject: [PATCH 13/24] Add missing import --- src/mvt/common/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index b7f5f83..3d054f5 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -13,6 +13,7 @@ import re from typing import Any, Iterator, Union from rich.logging import RichHandler +from mvt.common.config import settings class CustomJSONEncoder(json.JSONEncoder): From 458195a0abe9fcab78565076083c5dbef92d281f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 25 Dec 2024 00:28:02 +0000 Subject: [PATCH 14/24] Fix optional typing syntax for Python 3.8 --- src/mvt/common/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mvt/common/config.py b/src/mvt/common/config.py index b2fcf6d..d2e4e20 100644 --- a/src/mvt/common/config.py +++ b/src/mvt/common/config.py @@ -2,7 +2,7 @@ import os import yaml import json -from typing import Tuple, Type +from typing import Tuple, Type, Optional from appdirs import user_config_dir from pydantic import AnyHttpUrl, Field from pydantic_settings import ( @@ -36,16 +36,16 @@ class MVTSettings(BaseSettings): NETWORK_TIMEOUT: int = 15 # Command default settings, all can be specified by MVT_ prefixed environment variables too. - IOS_BACKUP_PASSWORD: str | None = Field( + IOS_BACKUP_PASSWORD: Optional[str] = Field( None, description="Default password to use to decrypt iOS backups" ) - ANDROID_BACKUP_PASSWORD: str | None = Field( + ANDROID_BACKUP_PASSWORD: Optional[str] = Field( None, description="Default password to use to decrypt Android backups" ) - STIX2: str | None = Field( + STIX2: Optional[str] = Field( None, description="List of directories where STIX2 files are stored" ) - VT_API_KEY: str | None = Field( + VT_API_KEY: Optional[str] = Field( None, description="API key to use for VirusTotal lookups" ) PROFILE: bool = Field(False, description="Profile the execution of MVT modules") From 7d6dc9e6dc40d8d34aa44ce78b047cdab139d0c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:07:57 +0100 Subject: [PATCH 15/24] Add new iOS versions and build numbers (#595) Co-authored-by: DonnchaC --- src/mvt/ios/data/ios_versions.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index a9d3f16..6f0c3d0 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1095,5 +1095,9 @@ { "version": "18.2", "build": "22C152" + }, + { + "version": "18.2.1", + "build": "22C161" } ] \ No newline at end of file From 2aa76c8a1ce83dfa25cdd6473c1729d2f8fde068 Mon Sep 17 00:00:00 2001 From: Tek Date: Tue, 7 Jan 2025 12:48:35 +0100 Subject: [PATCH 16/24] Fixes a bug on recent phones not having WIFI column in net usage (#580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Donncha Ó Cearbhaill Co-authored-by: Rory Flynn <75283103+roaree@users.noreply.github.com> --- src/mvt/ios/modules/net_base.py | 110 ++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 42 deletions(-) diff --git a/src/mvt/ios/modules/net_base.py b/src/mvt/ios/modules/net_base.py index 97fe263..1773e29 100644 --- a/src/mvt/ios/modules/net_base.py +++ b/src/mvt/ios/modules/net_base.py @@ -38,44 +38,70 @@ class NetBase(IOSExtraction): def _extract_net_data(self): conn = sqlite3.connect(self.file_path) + conn.row_factory = sqlite3.Row cur = conn.cursor() - cur.execute( + try: + cur.execute( + """ + SELECT + ZPROCESS.ZFIRSTTIMESTAMP, + ZPROCESS.ZTIMESTAMP, + ZPROCESS.ZPROCNAME, + ZPROCESS.ZBUNDLENAME, + ZPROCESS.Z_PK AS ZPROCESS_PK, + ZLIVEUSAGE.ZWIFIIN, + ZLIVEUSAGE.ZWIFIOUT, + ZLIVEUSAGE.ZWWANIN, + ZLIVEUSAGE.ZWWANOUT, + ZLIVEUSAGE.Z_PK AS ZLIVEUSAGE_PK, + ZLIVEUSAGE.ZHASPROCESS, + ZLIVEUSAGE.ZTIMESTAMP AS ZL_TIMESTAMP + FROM ZLIVEUSAGE + LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK + UNION + SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK, + NULL, NULL, NULL, NULL, NULL, NULL, NULL + FROM ZPROCESS WHERE Z_PK NOT IN + (SELECT ZHASPROCESS FROM ZLIVEUSAGE); """ - SELECT - ZPROCESS.ZFIRSTTIMESTAMP, - ZPROCESS.ZTIMESTAMP, - ZPROCESS.ZPROCNAME, - ZPROCESS.ZBUNDLENAME, - ZPROCESS.Z_PK, - ZLIVEUSAGE.ZWIFIIN, - ZLIVEUSAGE.ZWIFIOUT, - ZLIVEUSAGE.ZWWANIN, - ZLIVEUSAGE.ZWWANOUT, - ZLIVEUSAGE.Z_PK, - ZLIVEUSAGE.ZHASPROCESS, - ZLIVEUSAGE.ZTIMESTAMP - FROM ZLIVEUSAGE - LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK - UNION - SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK, - NULL, NULL, NULL, NULL, NULL, NULL, NULL - FROM ZPROCESS WHERE Z_PK NOT IN - (SELECT ZHASPROCESS FROM ZLIVEUSAGE); - """ - ) + ) + except sqlite3.OperationalError: + # Recent phones don't have ZWIFIIN and ZWIFIOUT columns + cur.execute( + """ + SELECT + ZPROCESS.ZFIRSTTIMESTAMP, + ZPROCESS.ZTIMESTAMP, + ZPROCESS.ZPROCNAME, + ZPROCESS.ZBUNDLENAME, + ZPROCESS.Z_PK AS ZPROCESS_PK, + ZLIVEUSAGE.ZWWANIN, + ZLIVEUSAGE.ZWWANOUT, + ZLIVEUSAGE.Z_PK AS ZLIVEUSAGE_PK, + ZLIVEUSAGE.ZHASPROCESS, + ZLIVEUSAGE.ZTIMESTAMP AS ZL_TIMESTAMP + FROM ZLIVEUSAGE + LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK + UNION + SELECT ZFIRSTTIMESTAMP, ZTIMESTAMP, ZPROCNAME, ZBUNDLENAME, Z_PK, + NULL, NULL, NULL, NULL, NULL + FROM ZPROCESS WHERE Z_PK NOT IN + (SELECT ZHASPROCESS FROM ZLIVEUSAGE); + """ + ) for row in cur: # ZPROCESS records can be missing after the JOIN. # Handle NULL timestamps. - if row[0] and row[1]: - first_isodate = convert_mactime_to_iso(row[0]) - isodate = convert_mactime_to_iso(row[1]) + if row["ZFIRSTTIMESTAMP"] and row["ZTIMESTAMP"]: + first_isodate = convert_mactime_to_iso(row["ZFIRSTTIMESTAMP"]) + isodate = convert_mactime_to_iso(row["ZTIMESTAMP"]) else: - first_isodate = row[0] - isodate = row[1] + first_isodate = row["ZFIRSTTIMESTAMP"] + isodate = row["ZTIMESTAMP"] - if row[11]: - live_timestamp = convert_mactime_to_iso(row[11]) + if row["ZL_TIMESTAMP"]: + live_timestamp = convert_mactime_to_iso(row["ZL_TIMESTAMP"]) else: live_timestamp = "" @@ -83,16 +109,18 @@ class NetBase(IOSExtraction): { "first_isodate": first_isodate, "isodate": isodate, - "proc_name": row[2], - "bundle_id": row[3], - "proc_id": row[4], - "wifi_in": row[5], - "wifi_out": row[6], - "wwan_in": row[7], - "wwan_out": row[8], - "live_id": row[9], - "live_proc_id": row[10], - "live_isodate": live_timestamp if row[11] else first_isodate, + "proc_name": row["ZPROCNAME"], + "bundle_id": row["ZBUNDLENAME"], + "proc_id": row["ZPROCESS_PK"], + "wifi_in": row["ZWIFIIN"] if "ZWIFIIN" in row.keys() else None, + "wifi_out": row["ZWIFIOUT"] if "ZWIFIOUT" in row.keys() else None, + "wwan_in": row["ZWWANIN"], + "wwan_out": row["ZWWANOUT"], + "live_id": row["ZLIVEUSAGE_PK"], + "live_proc_id": row["ZHASPROCESS"], + "live_isodate": live_timestamp + if row["ZL_TIMESTAMP"] + else first_isodate, } ) @@ -108,8 +136,6 @@ class NetBase(IOSExtraction): ) record_data_usage = ( record_data + " " - f"WIFI IN: {record['wifi_in']}, " - f"WIFI OUT: {record['wifi_out']} - " f"WWAN IN: {record['wwan_in']}, " f"WWAN OUT: {record['wwan_out']}" ) From 6e230bdb6a8501a3c9c062e0769614258e976564 Mon Sep 17 00:00:00 2001 From: Rory Flynn <75283103+roaree@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:02:10 +0100 Subject: [PATCH 17/24] Autofix for ruff (#598) --- src/mvt/android/artifacts/dumpsys_appops.py | 2 +- src/mvt/android/artifacts/dumpsys_packages.py | 3 +-- src/mvt/android/modules/adb/base.py | 3 +-- src/mvt/android/modules/adb/packages.py | 3 +-- src/mvt/android/modules/adb/sms.py | 2 +- src/mvt/android/modules/androidqf/packages.py | 3 +-- src/mvt/common/command.py | 2 +- src/mvt/common/indicators.py | 3 +-- src/mvt/ios/cli.py | 7 +++---- src/mvt/ios/modules/backup/backup_info.py | 2 +- src/mvt/ios/modules/backup/manifest.py | 3 +-- src/mvt/ios/modules/base.py | 2 +- src/mvt/ios/modules/mixed/sms.py | 2 +- src/mvt/ios/modules/mixed/webkit_session_resource_log.py | 2 +- 14 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_appops.py b/src/mvt/android/artifacts/dumpsys_appops.py index 6f066c8..12c8114 100644 --- a/src/mvt/android/artifacts/dumpsys_appops.py +++ b/src/mvt/android/artifacts/dumpsys_appops.py @@ -51,7 +51,7 @@ class DumpsysAppopsArtifact(AndroidArtifact): and perm["access"] == "allow" ): self.log.info( - "Package %s with REQUEST_INSTALL_PACKAGES " "permission", + "Package %s with REQUEST_INSTALL_PACKAGES permission", result["package_name"], ) diff --git a/src/mvt/android/artifacts/dumpsys_packages.py b/src/mvt/android/artifacts/dumpsys_packages.py index 2ca7e4c..2204180 100644 --- a/src/mvt/android/artifacts/dumpsys_packages.py +++ b/src/mvt/android/artifacts/dumpsys_packages.py @@ -16,8 +16,7 @@ class DumpsysPackagesArtifact(AndroidArtifact): for result in self.results: if result["package_name"] in ROOT_PACKAGES: self.log.warning( - "Found an installed package related to " - 'rooting/jailbreaking: "%s"', + 'Found an installed package related to rooting/jailbreaking: "%s"', result["package_name"], ) self.detected.append(result) diff --git a/src/mvt/android/modules/adb/base.py b/src/mvt/android/modules/adb/base.py index bdc2685..72df794 100644 --- a/src/mvt/android/modules/adb/base.py +++ b/src/mvt/android/modules/adb/base.py @@ -326,8 +326,7 @@ class AndroidExtraction(MVTModule): if not header["backup"]: self.log.error( - "Extracting SMS via Android backup failed. " - "No valid backup data found." + "Extracting SMS via Android backup failed. No valid backup data found." ) return None diff --git a/src/mvt/android/modules/adb/packages.py b/src/mvt/android/modules/adb/packages.py index 078d8dc..1d9c821 100644 --- a/src/mvt/android/modules/adb/packages.py +++ b/src/mvt/android/modules/adb/packages.py @@ -75,8 +75,7 @@ class Packages(AndroidExtraction): for result in self.results: if result["package_name"] in ROOT_PACKAGES: self.log.warning( - "Found an installed package related to " - 'rooting/jailbreaking: "%s"', + 'Found an installed package related to rooting/jailbreaking: "%s"', result["package_name"], ) self.detected.append(result) diff --git a/src/mvt/android/modules/adb/sms.py b/src/mvt/android/modules/adb/sms.py index 63c1de6..673e56a 100644 --- a/src/mvt/android/modules/adb/sms.py +++ b/src/mvt/android/modules/adb/sms.py @@ -70,7 +70,7 @@ class SMS(AndroidExtraction): "timestamp": record["isodate"], "module": self.__class__.__name__, "event": f"sms_{record['direction']}", - "data": f"{record.get('address', 'unknown source')}: \"{body}\"", + "data": f'{record.get("address", "unknown source")}: "{body}"', } def check_indicators(self) -> None: diff --git a/src/mvt/android/modules/androidqf/packages.py b/src/mvt/android/modules/androidqf/packages.py index e3de0f5..1d36777 100644 --- a/src/mvt/android/modules/androidqf/packages.py +++ b/src/mvt/android/modules/androidqf/packages.py @@ -44,8 +44,7 @@ class Packages(AndroidQFModule): for result in self.results: if result["name"] in ROOT_PACKAGES: self.log.warning( - "Found an installed package related to " - 'rooting/jailbreaking: "%s"', + 'Found an installed package related to rooting/jailbreaking: "%s"', result["name"], ) self.detected.append(result) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index b44a56b..d65555e 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -81,7 +81,7 @@ class Command: os.path.join(self.results_path, "command.log") ) formatter = logging.Formatter( - "%(asctime)s - %(name)s - " "%(levelname)s - %(message)s" + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) diff --git a/src/mvt/common/indicators.py b/src/mvt/common/indicators.py index a73938d..2529661 100644 --- a/src/mvt/common/indicators.py +++ b/src/mvt/common/indicators.py @@ -383,8 +383,7 @@ class Indicators: for ioc in self.get_iocs("urls"): if ioc["value"] == url: self.log.warning( - "Found a known suspicious URL %s " - 'matching indicator "%s" from "%s"', + 'Found a known suspicious URL %s matching indicator "%s" from "%s"', url, ioc["value"], ioc["name"], diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index a74eb32..1d06c96 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -100,7 +100,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): if key_file: if MVT_IOS_BACKUP_PASSWORD in os.environ: log.info( - "Ignoring %s environment variable, using --key-file" "'%s' instead", + "Ignoring %s environment variable, using --key-file'%s' instead", MVT_IOS_BACKUP_PASSWORD, key_file, ) @@ -114,7 +114,7 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): if MVT_IOS_BACKUP_PASSWORD in os.environ: log.info( - "Ignoring %s environment variable, using --password" "argument instead", + "Ignoring %s environment variable, using --passwordargument instead", MVT_IOS_BACKUP_PASSWORD, ) @@ -168,8 +168,7 @@ def extract_key(password, key_file, backup_path): if MVT_IOS_BACKUP_PASSWORD in os.environ: log.info( - "Ignoring %s environment variable, using --password " - "argument instead", + "Ignoring %s environment variable, using --password argument instead", MVT_IOS_BACKUP_PASSWORD, ) elif MVT_IOS_BACKUP_PASSWORD in os.environ: diff --git a/src/mvt/ios/modules/backup/backup_info.py b/src/mvt/ios/modules/backup/backup_info.py index 5fdc65f..c8f55f6 100644 --- a/src/mvt/ios/modules/backup/backup_info.py +++ b/src/mvt/ios/modules/backup/backup_info.py @@ -41,7 +41,7 @@ class BackupInfo(IOSExtraction): info_path = os.path.join(self.target_path, "Info.plist") if not os.path.exists(info_path): raise DatabaseNotFoundError( - "No Info.plist at backup path, unable to extract device " "information" + "No Info.plist at backup path, unable to extract device information" ) with open(info_path, "rb") as handle: diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index 107d645..ccbc459 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -110,8 +110,7 @@ class Manifest(IOSExtraction): ioc = self.indicators.check_url(part) if ioc: self.log.warning( - 'Found mention of domain "%s" in a backup file with ' - "path: %s", + 'Found mention of domain "%s" in a backup file with path: %s', ioc["value"], rel_path, ) diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index ef56a5d..f96d99a 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -74,7 +74,7 @@ class IOSExtraction(MVTModule): if not shutil.which("sqlite3"): raise DatabaseCorruptedError( - "failed to recover without sqlite3 binary: please install " "sqlite3!" + "failed to recover without sqlite3 binary: please install sqlite3!" ) if '"' in file_path: raise DatabaseCorruptedError( diff --git a/src/mvt/ios/modules/mixed/sms.py b/src/mvt/ios/modules/mixed/sms.py index 12eef0c..34c064b 100644 --- a/src/mvt/ios/modules/mixed/sms.py +++ b/src/mvt/ios/modules/mixed/sms.py @@ -43,7 +43,7 @@ class SMS(IOSExtraction): def serialize(self, record: dict) -> Union[dict, list]: text = record["text"].replace("\n", "\\n") - sms_data = f"{record['service']}: {record['guid']} \"{text}\" from {record['phone_number']} ({record['account']})" + sms_data = f'{record["service"]}: {record["guid"]} "{text}" from {record["phone_number"]} ({record["account"]})' records = [ { "timestamp": record["isodate"], diff --git a/src/mvt/ios/modules/mixed/webkit_session_resource_log.py b/src/mvt/ios/modules/mixed/webkit_session_resource_log.py index 19ba8a2..0ae2545 100644 --- a/src/mvt/ios/modules/mixed/webkit_session_resource_log.py +++ b/src/mvt/ios/modules/mixed/webkit_session_resource_log.py @@ -100,7 +100,7 @@ class WebkitSessionResourceLog(IOSExtraction): redirect_path += ", ".join(source_domains) redirect_path += " -> " - redirect_path += f"ORIGIN: \"{entry['origin']}\"" + redirect_path += f'ORIGIN: "{entry["origin"]}"' if len(destination_domains) > 0: redirect_path += " -> " From 0dc6228a59dc88f32140211dac730122dca1ffae Mon Sep 17 00:00:00 2001 From: Nim Date: Tue, 14 Jan 2025 12:04:07 +0100 Subject: [PATCH 18/24] Add command completion docs (#410) (#597) Co-authored-by: Rory Flynn <75283103+roaree@users.noreply.github.com> --- docs/command_completion.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/command_completion.md diff --git a/docs/command_completion.md b/docs/command_completion.md new file mode 100644 index 0000000..643dea3 --- /dev/null +++ b/docs/command_completion.md @@ -0,0 +1,37 @@ +# Command Completion + +MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface. + +Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fish. + +To enable it, you need to manually register a special function with your shell, which varies depending on the shell you are using. + +The following describes how to generate the command completion scripts and add them to your shell configuration. + +`You will need to start a new shell for the changes to take effect.` + +### For Bash + +```bash +# Generates bash completion scripts +echo "$(_MVT_IOS_COMPLETE=bash_source mvt-ios)" > ~/.mvt-ios-complete.bash && +echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complete.bash + +# Sources the scripts in ~/.bashrc. +. ~/.mvt-ios-complete.bash && . ~/.mvt-android-complete.bash +``` + +### For Zsh + +```bash +# Generates zsh completion scripts +echo "$(_MVT_IOS_COMPLETE=zsh_source mvt-ios)" > ~/.mvt-ios-complete.zsh && +echo "$(_MVT_ANDROID_COMPLETE=zsh_source mvt-android)" > ~/.mvt-android-complete.zsh + +# Sources the scripts in ~/.zshrc. +. ~/.mvt-ios-complete.zsh && . ~/.mvt-android-complete.zsh +``` + +For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion). + + From a2493baead14a1156a87ccf777a85c8b0512deb7 Mon Sep 17 00:00:00 2001 From: Rory Flynn <75283103+roaree@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:12:10 +0100 Subject: [PATCH 19/24] Documentation tweaks (#599) * Adds link in install instructions to the command completion docs added in #597 * Small visual tweaks --- docs/command_completion.md | 12 +++++++++--- docs/install.md | 4 ++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/command_completion.md b/docs/command_completion.md index 643dea3..1cd4eb7 100644 --- a/docs/command_completion.md +++ b/docs/command_completion.md @@ -8,7 +8,7 @@ To enable it, you need to manually register a special function with your shell, The following describes how to generate the command completion scripts and add them to your shell configuration. -`You will need to start a new shell for the changes to take effect.` +> **Note: You will need to start a new shell for the changes to take effect.** ### For Bash @@ -16,8 +16,11 @@ The following describes how to generate the command completion scripts and add t # Generates bash completion scripts echo "$(_MVT_IOS_COMPLETE=bash_source mvt-ios)" > ~/.mvt-ios-complete.bash && echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complete.bash +``` -# Sources the scripts in ~/.bashrc. +Add the following to `~/.bashrc`: +```bash +# source mvt completion scripts . ~/.mvt-ios-complete.bash && . ~/.mvt-android-complete.bash ``` @@ -27,8 +30,11 @@ echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complet # Generates zsh completion scripts echo "$(_MVT_IOS_COMPLETE=zsh_source mvt-ios)" > ~/.mvt-ios-complete.zsh && echo "$(_MVT_ANDROID_COMPLETE=zsh_source mvt-android)" > ~/.mvt-android-complete.zsh +``` -# Sources the scripts in ~/.zshrc. +Add the following to `~/.zshrc`: +```bash +# source mvt completion scripts . ~/.mvt-ios-complete.zsh && . ~/.mvt-android-complete.zsh ``` diff --git a/docs/install.md b/docs/install.md index cf54cb4..c08c75d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -98,3 +98,7 @@ You now should have the `mvt-ios` and `mvt-android` utilities installed. **Notes:** 1. The `--force` flag is necessary to force the reinstallation of the package. 2. To revert to using a PyPI version, it will be necessary to `pipx uninstall mvt` first. + +## Setting up command completions + +See ["Command completions"](command_completion.md) From dbb80d6320a4e99dc47fbcd90110a8800df23964 Mon Sep 17 00:00:00 2001 From: Rory Flynn <75283103+roaree@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:41:41 +0100 Subject: [PATCH 20/24] Mark release 2.6.0 (#601) --- src/mvt/common/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mvt/common/version.py b/src/mvt/common/version.py index ca21aff..dd19c1e 100644 --- a/src/mvt/common/version.py +++ b/src/mvt/common/version.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -MVT_VERSION = "2.5.4" +MVT_VERSION = "2.6.0" From 579b53f7eca21ddd73ab1b4df239f4a0034a1a00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jan 2025 01:27:17 +0100 Subject: [PATCH 21/24] Add new iOS versions and build numbers (#602) Co-authored-by: DonnchaC --- src/mvt/ios/data/ios_versions.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index 6f0c3d0..273ec55 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1099,5 +1099,9 @@ { "version": "18.2.1", "build": "22C161" + }, + { + "version": "18.3", + "build": "22D63" } ] \ No newline at end of file From 34cd08fd9ac7d8b5a7db91b62c54224f36500174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 30 Jan 2025 11:35:18 +0100 Subject: [PATCH 22/24] Add additional Android security setting to warn on --- src/mvt/android/artifacts/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mvt/android/artifacts/settings.py b/src/mvt/android/artifacts/settings.py index 1ce75e8..06a261e 100644 --- a/src/mvt/android/artifacts/settings.py +++ b/src/mvt/android/artifacts/settings.py @@ -16,6 +16,11 @@ ANDROID_DANGEROUS_SETTINGS = [ "key": "package_verifier_enable", "safe_value": "1", }, + { + "description": "disabled APK package verification", + "key": "package_verifier_state", + "safe_value": "1", + }, { "description": "disabled Google Play Protect", "key": "package_verifier_user_consent", From 0962383b460aaf41480a535f2864fd14d51ad727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 30 Jan 2025 11:48:19 +0100 Subject: [PATCH 23/24] Alert on potentially suspicious permissions from ADB --- src/mvt/android/artifacts/dumpsys_appops.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mvt/android/artifacts/dumpsys_appops.py b/src/mvt/android/artifacts/dumpsys_appops.py index 12c8114..98561b4 100644 --- a/src/mvt/android/artifacts/dumpsys_appops.py +++ b/src/mvt/android/artifacts/dumpsys_appops.py @@ -55,6 +55,11 @@ class DumpsysAppopsArtifact(AndroidArtifact): result["package_name"], ) + if result["package_name"] == "com.android.shell": + self.log.warning( + "Risky package com.android.shell requested a permission" + ) + def parse(self, output: str) -> None: self.results: List[Dict[str, Any]] = [] perm = {} From 43901c96a02fea5d6f9a5c7af0e632a483468165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 30 Jan 2025 13:02:26 +0100 Subject: [PATCH 24/24] Add improved heuristic detections to AppOps module --- src/mvt/android/artifacts/dumpsys_appops.py | 50 +++++++++++++++---- tests/android/test_artifact_dumpsys_appops.py | 18 ++++++- tests/android_androidqf/test_dumpsysappops.py | 7 ++- tests/android_bugreport/test_bugreport.py | 7 ++- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_appops.py b/src/mvt/android/artifacts/dumpsys_appops.py index 98561b4..8323c8a 100644 --- a/src/mvt/android/artifacts/dumpsys_appops.py +++ b/src/mvt/android/artifacts/dumpsys_appops.py @@ -11,6 +11,10 @@ from mvt.common.utils import convert_datetime_to_iso from .artifact import AndroidArtifact +RISKY_PERMISSIONS = ["REQUEST_INSTALL_PACKAGES"] +RISKY_PACKAGES = ["com.android.shell"] + + class DumpsysAppopsArtifact(AndroidArtifact): """ Parser for dumpsys app ops info @@ -45,20 +49,39 @@ class DumpsysAppopsArtifact(AndroidArtifact): self.detected.append(result) continue + detected_permissions = [] for perm in result["permissions"]: if ( - perm["name"] == "REQUEST_INSTALL_PACKAGES" - and perm["access"] == "allow" + perm["name"] in RISKY_PERMISSIONS + # and perm["access"] == "allow" ): - self.log.info( - "Package %s with REQUEST_INSTALL_PACKAGES permission", - result["package_name"], - ) + detected_permissions.append(perm) + for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]): + self.log.warning( + "Package '%s' had risky permission '%s' set to '%s' at %s", + result["package_name"], + perm["name"], + entry["access"], + entry["timestamp"], + ) - if result["package_name"] == "com.android.shell": - self.log.warning( - "Risky package com.android.shell requested a permission" - ) + elif result["package_name"] in RISKY_PACKAGES: + detected_permissions.append(perm) + for entry in sorted(perm["entries"], key=lambda x: x["timestamp"]): + self.log.warning( + "Risky package '%s' had '%s' permission set to '%s' at %s", + result["package_name"], + perm["name"], + entry["access"], + entry["timestamp"], + ) + + if detected_permissions: + # We clean the result to only include the risky permission, otherwise the timeline + # will be polluted with all the other irrelevant permissions + cleaned_result = result.copy() + cleaned_result["permissions"] = detected_permissions + self.detected.append(cleaned_result) def parse(self, output: str) -> None: self.results: List[Dict[str, Any]] = [] @@ -126,11 +149,16 @@ class DumpsysAppopsArtifact(AndroidArtifact): if line.startswith(" "): # Permission entry like: # Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms) + access_type = line.split(":")[0].strip() + if access_type not in ["Access", "Reject"]: + # Skipping invalid access type. Some entries are not in the format we expect + continue + if entry: perm["entries"].append(entry) entry = {} - entry["access"] = line.split(":")[0].strip() + entry["access"] = access_type entry["type"] = line[line.find("[") + 1 : line.find("]")] try: diff --git a/tests/android/test_artifact_dumpsys_appops.py b/tests/android/test_artifact_dumpsys_appops.py index e713e6b..7c2edc2 100644 --- a/tests/android/test_artifact_dumpsys_appops.py +++ b/tests/android/test_artifact_dumpsys_appops.py @@ -43,5 +43,21 @@ class TestDumpsysAppopsArtifact: ind.ioc_collections[0]["app_ids"].append("com.facebook.katana") da.indicators = ind assert len(da.detected) == 0 + da.check_indicators() - assert len(da.detected) == 1 + detected_by_ioc = [ + detected for detected in da.detected if detected.get("matched_indicator") + ] + detected_by_permission_heuristic = [ + detected + for detected in da.detected + if all( + [ + perm["name"] == "REQUEST_INSTALL_PACKAGES" + for perm in detected["permissions"] + ] + ) + ] + assert len(da.detected) == 3 + assert len(detected_by_ioc) == 1 + assert len(detected_by_permission_heuristic) == 2 diff --git a/tests/android_androidqf/test_dumpsysappops.py b/tests/android_androidqf/test_dumpsysappops.py index c28c240..b0649a1 100644 --- a/tests/android_androidqf/test_dumpsysappops.py +++ b/tests/android_androidqf/test_dumpsysappops.py @@ -21,4 +21,9 @@ class TestDumpsysAppOpsModule: run_module(m) assert len(m.results) == 12 assert len(m.timeline) == 16 - assert len(m.detected) == 0 + + detected_by_ioc = [ + detected for detected in m.detected if detected.get("matched_indicator") + ] + assert len(m.detected) == 1 + assert len(detected_by_ioc) == 0 diff --git a/tests/android_bugreport/test_bugreport.py b/tests/android_bugreport/test_bugreport.py index 52b4caf..cc9afec 100644 --- a/tests/android_bugreport/test_bugreport.py +++ b/tests/android_bugreport/test_bugreport.py @@ -33,7 +33,12 @@ class TestBugreportAnalysis: m = self.launch_bug_report_module(Appops) assert len(m.results) == 12 assert len(m.timeline) == 16 - assert len(m.detected) == 0 + + detected_by_ioc = [ + detected for detected in m.detected if detected.get("matched_indicator") + ] + assert len(m.detected) == 1 # Hueristic detection for suspicious permissions + assert len(detected_by_ioc) == 0 def test_packages_module(self): m = self.launch_bug_report_module(Packages)