From 0fdc2c34b00300392e60fa1e590f3f3238848cf5 Mon Sep 17 00:00:00 2001 From: besendorf Date: Mon, 17 Aug 2026 13:42:22 +0200 Subject: [PATCH 01/43] Document iOS device access from Docker (#875) * Document iOS device access from Docker * Simplify docker.md instructions Removed unnecessary note about mounting the entire '/var/run' directory. --- docs/docker.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/docker.md b/docs/docker.md index d8f4ff8..453c245 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -31,4 +31,18 @@ Test if the image was created successfully: docker run -it mvt ``` -If a prompt is spawned successfully, you can close it with `exit`. \ No newline at end of file +If a prompt is spawned successfully, you can close it with `exit`. + +## Access an iOS device from Docker + +On the Linux host, install and start [usbmuxd](https://github.com/libimobiledevice/usbmuxd), then connect and unlock the iOS device. The daemon exposes the device through the `/var/run/usbmuxd` socket. + +Bind that socket into the container to let MVT communicate with the device: + +```bash +docker run -it \ + --mount type=bind,source=/var/run/usbmuxd,target=/var/run/usbmuxd \ + ghcr.io/mvt-project/mvt +``` + +If you built the image from source, replace `ghcr.io/mvt-project/mvt` with `mvt`. From 3b21c9347c616ceed6dfce62a4a4767e052f7717 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 07:04:35 +0200 Subject: [PATCH 02/43] Add new iOS versions and build numbers (#879) Co-authored-by: DonnchaC <3081375+DonnchaC@users.noreply.github.com> --- src/mvt/ios/data/ios_versions.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index 16fbba5..9b4613b 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1220,6 +1220,10 @@ "version": "18.7.10", "build": "22H373" }, + { + "version": "18.7.10", + "build": "22H374" + }, { "version": "26", "build": "23A341" @@ -1263,5 +1267,9 @@ { "version": "26.6.1", "build": "23G82" + }, + { + "version": "26.6.1", + "build": "23G83" } ] \ No newline at end of file From 10a5bccbe08fa106bfa8b71e43fba0a1c3ceda9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 19 Aug 2026 11:43:40 +0200 Subject: [PATCH 03/43] Fix usability nits in the Docker images (#881) * Fix error in ADB key generation commmand * Add useful forensic analysis tools --- Dockerfile | 15 +++++++++++++-- Dockerfile.android | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index da06064..58bc216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,12 +119,23 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y \ adb \ + binutils \ default-jre-headless \ + file \ + jq \ + less \ libcurl4 \ + libimage-exiftool-perl \ libssl3 \ libusb-1.0-0 \ + moreutils \ + p7zip-full \ python3 \ - sqlite3 + ripgrep \ + sqlite3 \ + tree \ + unzip \ + xxd COPY --from=build-libplist /build / COPY --from=build-libimobiledevice-glue /build / COPY --from=build-libtatsu /build / @@ -144,7 +155,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android # Setup investigations environment diff --git a/Dockerfile.android b/Dockerfile.android index 2c060db..2b74611 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -31,7 +31,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android ENTRYPOINT [ "/usr/local/bin/mvt-android" ] From a92ec7f9638b54ce7bc617085fd2c1ce7a07986b Mon Sep 17 00:00:00 2001 From: besendorf Date: Wed, 19 Aug 2026 11:46:48 +0200 Subject: [PATCH 04/43] Build Docker images for ARM64 (#880) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Donncha Ó Cearbhaill --- .github/workflows/publish-release-docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish-release-docker.yml b/.github/workflows/publish-release-docker.yml index eddca22..b3fde3a 100644 --- a/.github/workflows/publish-release-docker.yml +++ b/.github/workflows/publish-release-docker.yml @@ -59,6 +59,9 @@ jobs: type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }} type=sha,suffix=${{ matrix.platform.tag-suffix }} type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }} + # Register emulators so Buildx can build the ARM64 images on the AMD64 runner. + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 # This step sets up some additional capabilities to generate the provenance and sbom attestations - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -71,6 +74,7 @@ jobs: with: file: ${{ matrix.platform.dockerfile }} context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 30c11f68c7a0cbfeca7dfa1b4c07d30d132a1037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 19 Aug 2026 14:07:27 +0200 Subject: [PATCH 05/43] Add WhatsApp contacts module and fix InteractionC contact resolution (#882) * Add WhatsappContacts module to extract WhatsApp disappearing messages state WhatsApp on iOS stores the disappearing messages timer for 1:1 chats on the contact records in ContactsV2.sqlite, not in ChatStorage.sqlite. Add a new WhatsappContacts module which extracts contact records from this database, including phone numbers, WhatsApp and LID identifiers, and the per-contact disappearing messages duration, and emits a timeline event when a disappearing messages timer was set. The database is often missing from incremental backups, so the module logs a clear warning and returns no results instead of failing. Columns are selected based on the actual table schema to tolerate changes across WhatsApp versions, and if the disappearing messages column is absent the state is reported as unknown rather than off. The test fixture is a synthetic ContactsV2.sqlite with fictional contacts, stored under the backup file ID derived from the WhatsApp shared app group domain. * Fix InteractionC contact resolution and resolve WhatsApp LIDs to contacts The two primary InteractionC queries contained a SQL syntax error in their direction CASE expression (a double column alias), so they always failed and the module silently fell back to a reduced query without the recipient join. As a result outgoing messages were serialized with no counterpart at all ("from None (None)"). Fix the syntax so recipient names and identifiers are extracted again, and normalize the raw 0/1 direction values from the fallback queries to INCOMING/OUTGOING. WhatsApp identifies chat peers in interactionC.db by LID and stores the peer LID in the domain identifier, which InteractionC could not map to a person. Declare a dependency on the WhatsappContacts module and resolve sender, recipient and domain identifiers (LID, JID or phone number) against the WhatsApp contacts database, adding resolved phone number and name fields to WhatsApp records. Rewrite the timeline serialization to use the resolved values, fall back to the chat peer from the domain identifier when no recipient was recorded, label the local user instead of printing None, and include the message direction and group name. * Add timeline events for all WhatsApp contact timestamps Extract ZABOUTEXPIRATIONTIMESTAMP and emit a timeline event for each timestamp stored on a WhatsApp contact record: disappearing messages timer changes, "about" text changes and scheduled expiry, and contact record updates. ContactsV2.sqlite stores no other date attributes in any released schema version. * Add first and last interaction timeline events for WhatsApp chats Extract one record per ZWACHATSESSION with the first and last stored message dates, the session's own last-message date, the group creation date and message counts. Each chat produces chat_first_message and chat_last_message timeline events, and groups a group_created event. The session last-message date is preferred over the newest stored message because it survives message deletion. * Resolve WhatsApp LID chat identifiers via the LID pair table Recent WhatsApp versions key 1:1 chat sessions by an opaque LID rather than the contact's phone number. Extract the ZWAPHONENUMBERLIDPAIR table from the dedicated LID.sqlite database (or from ChatStorage itself in versions that store it there) and use it to populate partner_resolved_phone_number on chat session records and in timeline events, without requiring the often-missing ContactsV2.sqlite. Each pair is also extracted as a record and produces a lid_pair_recorded timeline event marking when the association was learned. * Reduce duplicate InteractionC timeline events The interaction record's creation date normally trails its start date by milliseconds, so serializing both nearly doubled the timeline with duplicate entries. Only emit the creation date when it diverges from the start date by more than an hour, with explicit wording, since a record created long after its event indicates backfill by sync, restore or tampering. Per-contact aggregate dates from ZCONTACTS repeat on every interaction row of the same contact and carried that row's message text. Serialize them with contact-centric data strings instead, so timeline de-duplication collapses them into one first/last-seen event per contact. --- docs/ios/records.md | 16 +- src/mvt/ios/modules/mixed/__init__.py | 2 + src/mvt/ios/modules/mixed/interactionc.py | 240 +++++++++++++- src/mvt/ios/modules/mixed/whatsapp.py | 247 +++++++++++++- .../ios/modules/mixed/whatsapp_contacts.py | 308 ++++++++++++++++++ .../1f5a521220a3ad80ebfdc196978df8e7a2e49dee | Bin 0 -> 24576 bytes .../7c7fba66680ef796b916b067077cc246adacf01d | Bin 0 -> 24576 bytes .../b8548dc30aa1030df0ce18ef08b882cf7ab5212f | Bin 0 -> 8192 bytes .../e794f6ffcc3c222535f47684a63d5178da3c4500 | Bin 0 -> 8192 bytes tests/ios_backup/test_interactionc.py | 105 ++++++ tests/ios_backup/test_whatsapp.py | 71 ++++ tests/ios_backup/test_whatsapp_contacts.py | 83 +++++ tests/ios_fs/test_filesystem.py | 8 +- 13 files changed, 1063 insertions(+), 17 deletions(-) create mode 100644 src/mvt/ios/modules/mixed/whatsapp_contacts.py create mode 100644 tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee create mode 100644 tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d create mode 100644 tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f create mode 100644 tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 create mode 100644 tests/ios_backup/test_interactionc.py create mode 100644 tests/ios_backup/test_whatsapp_contacts.py diff --git a/docs/ios/records.md b/docs/ios/records.md index 016c861..b34183a 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -417,7 +417,21 @@ If indicators are provided through the command-line, they are checked against th Backup: :material-check: Full filesystem dump: :material-check: -This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*. +This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted. + +Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association. If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*. +--- + +### `whatsapp_contacts.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated). + +This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup. + diff --git a/src/mvt/ios/modules/mixed/__init__.py b/src/mvt/ios/modules/mixed/__init__.py index 0e50c26..a846bf2 100644 --- a/src/mvt/ios/modules/mixed/__init__.py +++ b/src/mvt/ios/modules/mixed/__init__.py @@ -26,6 +26,7 @@ from .tcc import TCC from .webkit_resource_load_statistics import WebkitResourceLoadStatistics from .webkit_session_resource_log import WebkitSessionResourceLog from .whatsapp import Whatsapp +from .whatsapp_contacts import WhatsappContacts MIXED_MODULES = [ Calls, @@ -47,6 +48,7 @@ MIXED_MODULES = [ WebkitResourceLoadStatistics, WebkitSessionResourceLog, Whatsapp, + WhatsappContacts, Shortcuts, Applications, Calendar, diff --git a/src/mvt/ios/modules/mixed/interactionc.py b/src/mvt/ios/modules/mixed/interactionc.py index 81a67e2..4d21382 100644 --- a/src/mvt/ios/modules/mixed/interactionc.py +++ b/src/mvt/ios/modules/mixed/interactionc.py @@ -3,9 +3,11 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import datetime import logging +import re import sqlite3 -from typing import Optional +from typing import Optional, Tuple from mvt.common.module_types import ( ModuleAtomicResult, @@ -15,6 +17,7 @@ from mvt.common.module_types import ( from mvt.common.utils import convert_mactime_to_iso from ..base import IOSExtraction +from .whatsapp_contacts import WhatsappContacts INTERACTIONC_BACKUP_IDS = [ "1f5a521220a3ad80ebfdc196978df8e7a2e49dee", @@ -22,6 +25,49 @@ INTERACTIONC_BACKUP_IDS = [ INTERACTIONC_ROOT_PATHS = [ "private/var/mobile/Library/CoreDuet/People/interactionC.db", ] + +# The interaction record's creation date normally trails its start date by +# milliseconds: emitting it as a timeline event only duplicates the start +# date event. A large divergence, however, indicates the record was +# backfilled (sync, restore, or tampering) and is worth surfacing. +CREATION_DATE_DIVERGENCE_THRESHOLD = 3600.0 + +# Per-contact aggregate dates from ZCONTACTS are repeated on every +# interaction row of the same contact. They are serialized with a +# contact-centric data string so that timeline de-duplication collapses +# them into one event per contact. +CONTACT_EVENT_TEMPLATES = { + "contacts_creation_date": "Contact {party} first recorded in interactionC", + "first_incoming_sender_date": "First incoming interaction from {party}", + "last_incoming_sender_date": "Last incoming interaction from {party}", + "first_incoming_recipient_date": ( + "First incoming interaction where {party} was a recipient" + ), + "last_incoming_recipient_date": ( + "Last incoming interaction where {party} was a recipient" + ), + "first_outgoing_recipient_date": ( + "First outgoing interaction to {party}" + ), + "last_outgoing_recipient_date": ( + "Last outgoing interaction to {party}" + ), +} + + +def _parse_iso(timestamp) -> Optional[datetime.datetime]: + try: + return datetime.datetime.strptime( + timestamp, "%Y-%m-%d %H:%M:%S.%f" + ) + except (TypeError, ValueError): + return None + + +def _describe_delta(seconds: float) -> str: + if seconds >= 86400: + return f"{seconds / 86400:.0f} days" + return f"{seconds / 3600:.0f} hours" # Taken from APOLLO # https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt QUERIES = [ @@ -34,7 +80,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -89,7 +135,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -117,7 +163,7 @@ QUERIES = [ CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE WHEN '0' THEN '0' ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE - END 'LAST INCOMING RECIPIENT DATE' AS "last_incoming_recipient_date", + END AS "last_incoming_recipient_date", ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date", ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id", ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url", @@ -218,9 +264,16 @@ QUERIES = [ ] +WHATSAPP_BUNDLE_ID = "net.whatsapp.WhatsApp" + + class InteractionC(IOSExtraction): """This module extracts data from InteractionC db.""" + # WhatsApp identifies chat peers by LID in interactionC.db, which only the + # WhatsApp contacts database can map back to a phone number and name. + dependencies = [WhatsappContacts] + def __init__( self, file_path: Optional[str] = None, @@ -252,10 +305,51 @@ class InteractionC(IOSExtraction): "last_outgoing_recipient_date", ] + @staticmethod + def _describe_party(record: ModuleAtomicResult, prefix: str) -> Optional[str]: + name = record.get(f"{prefix}_display_name") or record.get( + f"{prefix}_resolved_name" + ) + identifier = record.get(f"{prefix}_resolved_phone_number") or record.get( + f"{prefix}_identifier" + ) + if name and identifier: + # A display name that is just a formatted copy of the phone + # number adds no information. + name_digits = re.sub(r"\D", "", name) + if name_digits and name_digits == re.sub(r"\D", "", identifier): + return identifier + return f"{name} ({identifier})" + return name or identifier or None + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + sender = self._describe_party(record, "sender") + # The chat peer from the domain identifier stands in when the + # recipient was not recorded (or the recipient join is unavailable). + recipient = self._describe_party(record, "recipient") or self._describe_party( + record, "domain" + ) + direction = record.get("direction") + if not sender and direction == "OUTGOING": + sender = "local user" + if not recipient and direction == "INCOMING": + recipient = "local user" + + header = f"[{record['bundle_id']}]" + if record.get("account"): + header += f" {record['account']}" + if direction: + header += f" {direction}" + + data = f"{header} from {sender or 'unknown'} to {recipient or 'unknown'}" + if record.get("group_name"): + data += f" (group: {record['group_name']})" + if record.get("content"): + data += f": {record['content']}" + records = [] processed = [] - for timestamp in self.timestamps: + for timestamp in ("start_date", "end_date"): # Check if the record has the current timestamp. if timestamp not in record or not record[timestamp]: continue @@ -269,16 +363,142 @@ class InteractionC(IOSExtraction): "timestamp": record[timestamp], "module": self.__class__.__name__, "event": timestamp, - "data": f"[{record['bundle_id']}] {record['account']} - " - f"from {record['sender_display_name']} ({record['sender_identifier']}) " - f"to {record.get('recipient_display_name', '')} ({record.get('recipient_identifier', '')}):" - f" {record.get('content', '')}", + "data": data, } ) processed.append(record[timestamp]) + creation_event = self._serialize_creation_date(record, data) + if creation_event: + records.append(creation_event) + + # Contact-level aggregates describe the sender's contact record. + party = self._describe_party(record, "sender") + if party: + for field, template in CONTACT_EVENT_TEMPLATES.items(): + if not record.get(field): + continue + records.append( + { + "timestamp": record[field], + "module": self.__class__.__name__, + "event": field, + "data": template.format(party=party), + } + ) + return records + def _serialize_creation_date( + self, record: ModuleAtomicResult, data: str + ) -> Optional[dict]: + """Serialize the interaction record's creation date only when it + diverges from the start date enough to indicate the record was + backfilled.""" + creation = record.get("interactions_creation_date") + if not creation: + return None + + event = { + "timestamp": creation, + "module": self.__class__.__name__, + "event": "interactions_creation_date", + "data": data, + } + + start = _parse_iso(record.get("start_date")) + creation_parsed = _parse_iso(creation) + if not start or not creation_parsed: + # Without a start date the creation date is the only anchor. + return event + + delta = (creation_parsed - start).total_seconds() + if abs(delta) < CREATION_DATE_DIVERGENCE_THRESHOLD: + return None + + direction = "after" if delta > 0 else "before" + event["data"] = ( + f"Interaction record created {_describe_delta(abs(delta))} " + f"{direction} the event: {data}" + ) + return event + + def _whatsapp_contact_maps(self) -> Tuple[dict, dict]: + """Build LID and phone-digit lookup maps from the WhatsappContacts + module results, when available.""" + by_lid: dict = {} + by_phone: dict = {} + contacts_module = self.dependency_modules.get(WhatsappContacts) + if not contacts_module: + return by_lid, by_phone + + for contact in contacts_module.results: + name = contact.get("full_name") or contact.get("given_name") + phone = contact.get("phone_number") + entry = (phone, name) + if contact.get("lid"): + by_lid[contact["lid"]] = entry + if phone: + by_phone[re.sub(r"\D", "", phone)] = entry + whatsapp_id = contact.get("whatsapp_id") + if whatsapp_id and "@" in whatsapp_id: + by_phone.setdefault(whatsapp_id.split("@")[0], entry) + + return by_lid, by_phone + + @staticmethod + def _resolve_whatsapp_identifier( + value, by_lid: dict, by_phone: dict + ) -> Tuple[Optional[str], Optional[str]]: + """Resolve a WhatsApp identifier (LID, JID or phone number) to a + (phone_number, contact_name) tuple.""" + if not value: + return None, None + + value = str(value) + if value.endswith("@lid"): + return by_lid.get(value, (None, None)) + if value.endswith("@g.us"): + return None, None + if value.endswith("@s.whatsapp.net"): + digits = value.split("@")[0] + phone, name = by_phone.get(digits, (None, None)) + return phone or f"+{digits}", name + if value.startswith("+"): + _, name = by_phone.get(re.sub(r"\D", "", value), (None, None)) + return None, name + + return None, None + + def _postprocess_results(self) -> None: + by_lid, by_phone = self._whatsapp_contact_maps() + + for entry in self.results: + # The fallback queries return ZDIRECTION raw instead of labelled. + if entry.get("direction") in (0, "0"): + entry["direction"] = "INCOMING" + elif entry.get("direction") in (1, "1"): + entry["direction"] = "OUTGOING" + + if entry.get("bundle_id") != WHATSAPP_BUNDLE_ID: + continue + + candidates = { + "sender": [entry.get("sender_identifier"), entry.get("custom_id")], + "recipient": [entry.get("recipient_identifier")], + "domain": [entry.get("domain_identifier")], + } + for prefix, values in candidates.items(): + phone = name = None + for value in values: + phone, name = self._resolve_whatsapp_identifier( + value, by_lid, by_phone + ) + if phone or name: + break + entry[f"{prefix}_resolved_phone_number"] = phone + entry[f"{prefix}_resolved_name"] = name + def run(self) -> None: self._find_ios_database( backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS @@ -325,4 +545,6 @@ class InteractionC(IOSExtraction): cur.close() conn.close() + self._postprocess_results() + self.log.info("Extracted a total of %d InteractionC events", len(self.results)) diff --git a/src/mvt/ios/modules/mixed/whatsapp.py b/src/mvt/ios/modules/mixed/whatsapp.py index 0a80aad..89601df 100644 --- a/src/mvt/ios/modules/mixed/whatsapp.py +++ b/src/mvt/ios/modules/mixed/whatsapp.py @@ -4,7 +4,9 @@ # https://license.mvt.re/1.1/ import logging -from typing import Optional +import os +import sqlite3 +from typing import Dict, Optional from mvt.common.module_types import ( ModuleAtomicResult, @@ -22,9 +24,71 @@ WHATSAPP_ROOT_PATHS = [ "private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite", ] +WHATSAPP_LID_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-LID.sqlite" + "e794f6ffcc3c222535f47684a63d5178da3c4500", +] +WHATSAPP_LID_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/LID.sqlite", +] + +# WhatsApp records the mapping between a contact's LID and phone number +# identifiers in the ZWAPHONENUMBERLIDPAIR table. Depending on the WhatsApp +# version this lives in a dedicated LID.sqlite database or in +# ChatStorage.sqlite itself. +LID_PAIRS_QUERY = """ + SELECT + ZLID AS "lid", + ZPHONENUMBER AS "phone_number", + ZTIMESTAMP AS "pair_timestamp" + FROM ZWAPHONENUMBERLIDPAIR; +""" + +CHAT_SESSIONS_QUERY = """ + SELECT + ZWACHATSESSION.Z_PK AS "session_pk", + ZWACHATSESSION.ZCONTACTJID AS "contact_jid", + ZWACHATSESSION.ZPARTNERNAME AS "partner_name", + ZWACHATSESSION.ZSESSIONTYPE AS "session_type", + ZWACHATSESSION.ZARCHIVED AS "archived", + ZWACHATSESSION.ZREMOVED AS "removed", + ZWACHATSESSION.ZMESSAGECOUNTER AS "message_counter", + ZWACHATSESSION.ZLASTMESSAGEDATE AS "last_message_date", + ZWAGROUPINFO.ZCREATIONDATE AS "group_creation_date", + MIN(ZWAMESSAGE.ZMESSAGEDATE) AS "first_stored_message_date", + MAX(ZWAMESSAGE.ZMESSAGEDATE) AS "last_stored_message_date", + COUNT(ZWAMESSAGE.Z_PK) AS "stored_message_count" + FROM ZWACHATSESSION + LEFT JOIN ZWAGROUPINFO + ON ZWACHATSESSION.ZGROUPINFO = ZWAGROUPINFO.Z_PK + LEFT JOIN ZWAMESSAGE + ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK + GROUP BY ZWACHATSESSION.Z_PK; +""" + +CHAT_SESSION_DATE_FIELDS = [ + "last_message_date", + "group_creation_date", + "first_stored_message_date", + "last_stored_message_date", +] + + +def _describe_chat(record: ModuleAtomicResult) -> str: + jid = record.get("contact_jid") or "unknown" + name = record.get("partner_name") + identifier = record.get("partner_resolved_phone_number") or jid + label = f"'{name}' ({identifier})" if name else identifier + is_group = jid.endswith("@g.us") or record.get("group_creation_date") + if is_group: + return f"WhatsApp group chat {label}" + return f"WhatsApp chat with {label}" + class Whatsapp(IOSExtraction): - """This module extracts all WhatsApp messages containing links.""" + """This module extracts all WhatsApp messages containing links, as well + as per-chat records with the first and last interaction dates of each + conversation.""" def __init__( self, @@ -45,6 +109,21 @@ class Whatsapp(IOSExtraction): ) def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + if record.get("record_type") == "chat_session": + return self._serialize_chat_session(record) + if record.get("record_type") == "lid_phone_number_pair": + if not record.get("pair_timestamp"): + return [] + return { + "timestamp": record["pair_timestamp"], + "module": self.__class__.__name__, + "event": "lid_pair_recorded", + "data": ( + f"WhatsApp associated LID {record.get('lid')} with " + f"phone number {record.get('phone_number')}" + ), + } + text = record.get("ZTEXT", "").replace("\n", "\\n") links_text = "" if record.get("links"): @@ -57,6 +136,49 @@ class Whatsapp(IOSExtraction): "data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}", } + def _serialize_chat_session( + self, record: ModuleAtomicResult + ) -> ModuleSerializedResult: + records = [] + chat = _describe_chat(record) + + if record.get("group_creation_date"): + records.append( + { + "timestamp": record["group_creation_date"], + "module": self.__class__.__name__, + "event": "group_created", + "data": f"{chat} was created", + } + ) + + if record.get("first_stored_message_date"): + records.append( + { + "timestamp": record["first_stored_message_date"], + "module": self.__class__.__name__, + "event": "chat_first_message", + "data": f"First stored message in {chat}", + } + ) + + # The chat session's own last-message date is authoritative: it can + # postdate the newest stored message if that message was deleted. + last_message_date = record.get("last_message_date") or record.get( + "last_stored_message_date" + ) + if last_message_date: + records.append( + { + "timestamp": last_message_date, + "module": self.__class__.__name__, + "event": "chat_last_message", + "data": f"Last message in {chat}", + } + ) + + return records + def check_indicators(self) -> None: if not self.indicators: return @@ -146,7 +268,126 @@ class Whatsapp(IOSExtraction): message["links"] = list(set(filtered_links)) self.results.append(message) + total_messages = len(self.results) + lid_map = self._extract_lid_pairs(cur) + total_sessions = self._extract_chat_sessions(cur, lid_map) + cur.close() conn.close() - self.log.info("Extracted a total of %d WhatsApp messages", len(self.results)) + self.log.info( + "Extracted a total of %d WhatsApp messages, %d chat sessions " + "and %d LID-phone number pairs", + total_messages, + total_sessions, + len(lid_map), + ) + + def _find_lid_db_path(self) -> Optional[str]: + for backup_id in WHATSAPP_LID_BACKUP_IDS: + file_path = self._get_backup_file_from_id(backup_id) + if file_path and os.path.exists(file_path): + return file_path + for found_path in self._get_fs_files_from_patterns( + WHATSAPP_LID_ROOT_PATHS + ): + return found_path + return None + + def _extract_lid_pairs(self, chat_cur: sqlite3.Cursor) -> Dict[str, str]: + """Extract the LID to phone number mapping from the dedicated + LID.sqlite database, falling back to the same table in + ChatStorage.sqlite. Returns a map of LID digits to phone number + digits.""" + rows = [] + lid_db_path = self._find_lid_db_path() + if lid_db_path: + self.log.info( + "Found WhatsApp LID database at path: %s", lid_db_path + ) + lid_conn = self._open_sqlite_db(lid_db_path) + try: + lid_cur = lid_conn.cursor() + lid_cur.execute(LID_PAIRS_QUERY) + rows = lid_cur.fetchall() + lid_cur.close() + except sqlite3.DatabaseError as exc: + self.log.warning( + "Unable to extract WhatsApp LID-phone number pairs: %s", + exc, + ) + finally: + lid_conn.close() + else: + try: + chat_cur.execute(LID_PAIRS_QUERY) + rows = chat_cur.fetchall() + except sqlite3.OperationalError: + self.log.info( + "No WhatsApp LID database found in this backup or " + "filesystem dump: LID chat identifiers cannot be " + "resolved to phone numbers" + ) + + lid_map: Dict[str, str] = {} + for lid, phone_number, pair_timestamp in rows: + record = { + "record_type": "lid_phone_number_pair", + "lid": lid, + "phone_number": phone_number, + "pair_timestamp": ( + convert_mactime_to_iso(pair_timestamp) or None + ) + if pair_timestamp + else None, + } + self.results.append(record) + + if lid and phone_number: + lid_digits = str(lid).split("@")[0] + phone_digits = str(phone_number).split("@")[0].lstrip("+") + lid_map[lid_digits] = phone_digits + + return lid_map + + def _extract_chat_sessions( + self, cur: sqlite3.Cursor, lid_map: Dict[str, str] + ) -> int: + """Extract one record per chat session with the first and last + interaction dates of each conversation.""" + try: + cur.execute(CHAT_SESSIONS_QUERY) + except sqlite3.OperationalError as exc: + self.log.warning( + "Unable to extract WhatsApp chat sessions: %s", exc + ) + return 0 + + names = [description[0] for description in cur.description] + + total_sessions = 0 + for row in cur.fetchall(): + session = dict(zip(names, row)) + session["record_type"] = "chat_session" + + session["partner_resolved_phone_number"] = None + jid = session.get("contact_jid") or "" + if jid.endswith("@lid"): + phone_digits = lid_map.get(jid.split("@")[0]) + if phone_digits: + session["partner_resolved_phone_number"] = ( + f"+{phone_digits}" + ) + + for field in CHAT_SESSION_DATE_FIELDS: + if session.get(field): + session[field] = ( + convert_mactime_to_iso(session[field]) or None + ) + else: + session[field] = None + + self.results.append(session) + total_sessions += 1 + + return total_sessions diff --git a/src/mvt/ios/modules/mixed/whatsapp_contacts.py b/src/mvt/ios/modules/mixed/whatsapp_contacts.py new file mode 100644 index 0000000..309da22 --- /dev/null +++ b/src/mvt/ios/modules/mixed/whatsapp_contacts.py @@ -0,0 +1,308 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +import sqlite3 +from typing import Optional + +from mvt.common.module import DatabaseNotFoundError +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.utils import convert_mactime_to_iso + +from ..base import IOSExtraction + +WHATSAPP_CONTACTS_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-ContactsV2.sqlite" + "b8548dc30aa1030df0ce18ef08b882cf7ab5212f", +] +WHATSAPP_CONTACTS_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/ContactsV2.sqlite", +] + +# WhatsApp's standard disappearing-messages timer values, in seconds. +DISAPPEARING_DURATION_LABELS = { + 86400: "24 hours", + 604800: "7 days", + 1209600: "14 days", + 2592000: "30 days", + 7776000: "90 days", +} + +# Output field -> candidate columns in ZWAADDRESSBOOKCONTACT, in order of +# preference. WhatsApp renames columns across versions, so the query is built +# from the columns actually present in the database. +COLUMN_CANDIDATES = { + "whatsapp_id": ["ZWHATSAPPID"], + "lid": ["ZLID"], + "phone_number": ["ZPHONENUMBER"], + "localized_phone_number": ["ZLOCALIZEDPHONENUMBER"], + "full_name": ["ZFULLNAME"], + "given_name": ["ZGIVENNAME"], + "last_name": ["ZLASTNAME"], + "user_name": ["ZUSERNAME"], + "business_name": ["ZBUSINESSNAME"], + "about_text": ["ZABOUTTEXT"], + "about_emoji": ["ZABOUTEMOJI"], + "notes": ["ZNOTES"], + "disappearing_mode_duration": ["ZDISAPPEARINGMODEDURATION"], + "disappearing_mode_timestamp": ["ZDISAPPEARINGMODETIMESTAMP"], + "about_timestamp": ["ZABOUTTIMESTAMP"], + "about_expiration_timestamp": ["ZABOUTEXPIRATIONTIMESTAMP"], + "last_updated": ["ZLASTUPDATED"], + "phone_status": ["ZPHONESTATUS", "ZPHONENUMBERSTATUS"], + "sync_policy": ["ZSYNCPOLICY"], +} + +STRING_FIELDS = [ + "whatsapp_id", + "lid", + "phone_number", + "localized_phone_number", + "full_name", + "given_name", + "last_name", + "user_name", + "business_name", + "about_text", + "about_emoji", + "notes", +] + +DATE_FIELDS = [ + "disappearing_mode_timestamp", + "about_timestamp", + "about_expiration_timestamp", + "last_updated", +] + + +def _decode_string(value) -> Optional[str]: + # CoreData stores string attributes as UTF-8 blobs in some WhatsApp + # versions, so values can arrive as either bytes or str. + if value is None: + return None + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def _label_duration(duration) -> str: + if not duration: + return "off" + return DISAPPEARING_DURATION_LABELS.get( + int(duration), f"{int(duration)} seconds" + ) + + +def _describe_contact(record: ModuleAtomicResult) -> str: + contact = ( + record.get("whatsapp_id") + or record.get("lid") + or record.get("phone_number") + or "unknown" + ) + full_name = record.get("full_name") + if full_name: + contact = f"{contact} ({full_name})" + return contact + + +class WhatsappContacts(IOSExtraction): + """This module extracts WhatsApp contact records and per-contact + disappearing-messages settings from ContactsV2.sqlite. + + ChatStorage.sqlite does not record the disappearing-messages state of 1:1 + chats: the authoritative timer is stored on each contact record in this + database, alongside the mapping between a contact's LID and phone number + identifiers. + """ + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[ModuleResults] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + records = [] + contact = _describe_contact(record) + + if record.get("disappearing_mode_timestamp"): + records.append( + { + "timestamp": record["disappearing_mode_timestamp"], + "module": self.__class__.__name__, + "event": "disappearing_mode_set", + "data": ( + f"WhatsApp disappearing messages timer set to " + f"'{record.get('disappearing_mode_label')}' " + f"for {contact}" + ), + } + ) + + if record.get("about_timestamp"): + data = f"WhatsApp about text of {contact} changed" + about_text = record.get("about_text") + if about_text: + data += f' to "{about_text}"' + records.append( + { + "timestamp": record["about_timestamp"], + "module": self.__class__.__name__, + "event": "about_changed", + "data": data, + } + ) + + if record.get("about_expiration_timestamp"): + records.append( + { + "timestamp": record["about_expiration_timestamp"], + "module": self.__class__.__name__, + "event": "about_expiration", + "data": ( + f"WhatsApp about text of {contact} scheduled " + f"to expire" + ), + } + ) + + if record.get("last_updated"): + records.append( + { + "timestamp": record["last_updated"], + "module": self.__class__.__name__, + "event": "contact_last_updated", + "data": f"WhatsApp contact record for {contact} updated", + } + ) + + return records + + def run(self) -> None: + try: + self._find_ios_database( + backup_ids=WHATSAPP_CONTACTS_BACKUP_IDS, + root_paths=WHATSAPP_CONTACTS_ROOT_PATHS, + ) + except DatabaseNotFoundError: + self.log.warning( + "Unable to find the WhatsApp ContactsV2.sqlite database in " + "this backup or filesystem dump. WhatsApp disappearing " + "messages settings and contact records cannot be extracted. " + "This database is often missing from incremental backups." + ) + return + + self.log.info( + "Found WhatsApp contacts database at path: %s", self.file_path + ) + + assert self.file_path is not None + conn = self._open_sqlite_db(self.file_path) + cur = conn.cursor() + try: + try: + cur.execute("PRAGMA table_info(ZWAADDRESSBOOKCONTACT)") + available_columns = {row[1] for row in cur.fetchall()} + except sqlite3.DatabaseError as exc: + self.log.error( + "Unable to read the ZWAADDRESSBOOKCONTACT table schema: %s", + exc, + ) + return + + if not available_columns: + self.log.warning( + "The WhatsApp contacts database does not contain a " + "ZWAADDRESSBOOKCONTACT table" + ) + return + + selected = {} + for field, candidates in COLUMN_CANDIDATES.items(): + for candidate in candidates: + if candidate in available_columns: + selected[field] = candidate + break + + # A record with no duration column is "unknown", not "off": the + # timer state cannot be determined from this database version. + has_duration = "disappearing_mode_duration" in selected + if not has_duration: + self.log.warning( + "The ZDISAPPEARINGMODEDURATION column is not present in " + "this WhatsApp contacts database: disappearing messages " + "state is unknown" + ) + + columns = ["Z_PK"] + list(selected.values()) + cur.execute( + f"SELECT {', '.join(columns)} FROM ZWAADDRESSBOOKCONTACT;" + ) + fields = ["row_pk"] + list(selected.keys()) + + for row in cur: + record = dict(zip(fields, row)) + + for field in STRING_FIELDS: + if field in record: + record[field] = _decode_string(record[field]) + else: + record[field] = None + + for field in DATE_FIELDS: + if record.get(field) is not None: + record[field] = ( + convert_mactime_to_iso(record[field]) or None + ) + else: + record[field] = None + + duration = record.get("disappearing_mode_duration") + if has_duration: + record["disappearing_mode_is_on"] = bool(duration) + record["disappearing_mode_label"] = _label_duration( + duration + ) + else: + record["disappearing_mode_duration"] = None + record["disappearing_mode_is_on"] = None + record["disappearing_mode_label"] = None + + record.setdefault("phone_status", None) + record.setdefault("sync_policy", None) + + self.results.append(record) + finally: + cur.close() + conn.close() + + total_ephemeral = sum( + 1 for record in self.results if record["disappearing_mode_is_on"] + ) + self.log.info( + "Extracted a total of %d WhatsApp contacts (%d with disappearing " + "messages enabled)", + len(self.results), + total_ephemeral, + ) diff --git a/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee b/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee new file mode 100644 index 0000000000000000000000000000000000000000..1d278325152b6c4a2d7bcc67d721b5a364b26f24 GIT binary patch literal 24576 zcmeI2&u`;I6vxMj(`**4Tos{QkZP$yT~uV6%>swL)Ul`OX#FF2#tY45N!g&S)NTVC z3HE|0>|y@_B>n;p{1qHHAnlp|fq#J;GqIhHQ+rEqsQOOgcrx$3d2c@RW~6b#KMjuO zQ~Rs4Z%!w3yHD1MVURuBCWNfzm%J@XOMaSpUEK_=URPZs&d;y!NNKC{Ho5cHTR+@> z)cHeJQ9u9$KmY_l00cnb|3u*2X|=l#9vJ7(=aZw8>G<(I773cTo?vm5P>+omjYLiJ znwL;lP+PczfZ8=m_TAfdUZlpdOKvMiLz;Y`iThQ~3&n-r`{$O~?(Xau=V=~rPqX3U z$EB~Wl>ke^QeZt-2-iCZbkWoJqH2~Sxe%117(FZo zYmX8Zy2+vakRED;yjVhugrJ`bTYYGvRIt^iTsuvIRS9Qf%A&!cpv%E%U5yT65eKp7 z2F#b<*gm7lY7d%Z5X1w`+6&^;*TgE$jMU4d(e7?;8n5zXQ}M}6zEK@3)ugAz| zd=$8cksHd)EHhe(N(*{K6COv*U$yX3F5jzyjnN5f@advO|xViq+Ag(9bsPm?;W zI2FVAzjU_%C6}nL)}LI3f#zzTC0rE#mVsTVnMGEsb}6(f&A`3zT;HFr^~U&=V!huy zsqPx>?)J9v%F32sX*{Yo*}y91mg#@|nb}cJzv%mlIHzlYSh!OfHSuoq#8lkK%~air(wDAho}!kka(>Acsku zj;a>0TsrI#_0`VePSg)<0)>em6Ol4+nO7_j@17vfJD3IZwaeQ%?`xZLAYg7%c2Gt=*3eDecSGvQ#`F zsAy8>fBC0>6c7Lb5C8!X009sH0T2KI5C8!X0D&7!0O$W3Jh>Pe2!H?xfB*=900@8p z2!H?xfB*>O1l0N8u>K+PjRFE700JNY0w4eaAOHd&00JNY0w8eH2(*l*VKf^50&X<` A5&!@I literal 0 HcmV?d00001 diff --git a/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d b/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d new file mode 100644 index 0000000000000000000000000000000000000000..b8ae144dd2db59a308c6f17793c47e861eeaee6d GIT binary patch literal 24576 zcmeI(!B5jr90%~%Zetq@Ok?5>Udn;s64R}K7d^P=_Sh=xI=eR!5(8sam<~4C2p$FE z)tia`h!;=ZO}u&ZAK>oY*R5lvl*1BZ4Eesm`gp(hUVr_(K9a&#ds=SwZRJ(xV6V|v zmdFhvNhGHzgpeusTO0-@B3{F8-8o2}`c>5_qJH{5BchSSZ8Cc{yF9ay_$sO>5P$## zAOHafKmY;|_>TmR=OWS6{JeA$={L69_Qo15T4uefS5{1KeJXD;%9+CH<1$mc5#{c* z;+Bog>Z+nwI4d$ssam>0t#xIUt=}8>$d@Ruv06X z?saX^6p1Rwwb2tWV=5P$##AOHafKwwgVOejtyO74VX zFzy+*gI?%Vw}yWQ;U+3I%l8!=0uX=z z1Rwwb2tWV=5P$##AOL}DEP&_#YrMF)GzdTd0uX=z1Rwwb2tWV=5P*Oy;5`4!$QNgyu%Iv|B*b?wX6AGH~S5dZ)H literal 0 HcmV?d00001 diff --git a/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f b/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f new file mode 100644 index 0000000000000000000000000000000000000000..132059c9e044a30c9e8cf17697a344a73ccd596c GIT binary patch literal 8192 zcmeH~%Wl&^6o$tk5^|*iLWoLT4Hc?V0md$rF4%Z%56Lw4jBAfclMQmIp^=g{YK(wg zS+HTliuYm9jyK><80R94Ys$LPIr3%ZoZtSBznP8WeSer|_|56rIL+}BbPH)3vN1*o zH5RA3%|cVx=3-pjiuwPdfsAiIR+UOyeT>ur0R(^m5C8%|00;m9AOHk_01&t;1U_@^ z`r7WU_T_n=9*(j>j}Xs`XdF92bX~zEab<0?;YO55iY0M;ifbjfwSpHhcolXr*}Q|2 zulbj+`N!9+_*!N;EPFIO>SP~rew>|U_b|ih7{8kiPu}8Q(JWETYWwGg_4T{45u6+0 I9lL1!H%o*46#xJL literal 0 HcmV?d00001 diff --git a/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 b/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 new file mode 100644 index 0000000000000000000000000000000000000000..d54a9c5554d55ba745082b0f28c1b9ab57705d40 GIT binary patch literal 8192 zcmeI#K}*9h6bJC64u`^+H?I!_aZJXF%C5`wp00Izz00bZa0SG_<0ub1sz~_go@ArE0tC7zhU*^+WWv)gU zXE*79ZK5xX(zdb~om*Cp$*DoYRM|pDyHsZSV1}eK$Cqr$Se+_6p&?I>gAUHzUrSgX znX0nWDUY2>4f$gvLBBsUIlgm2Dfpt|Rk3Gl^>4lq&BX@!u{8uB009U<00Izz00bZa i0SG_<0{;>?lyxC@YVCuq&b{u(-7s-{8lU-eefbxJO-!r+ literal 0 HcmV?d00001 diff --git a/tests/ios_backup/test_interactionc.py b/tests/ios_backup/test_interactionc.py new file mode 100644 index 0000000..eec2656 --- /dev/null +++ b/tests/ios_backup/test_interactionc.py @@ -0,0 +1,105 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.interactionc import InteractionC +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestInteractionCModule: + def test_extraction_with_whatsapp_contacts(self): + contacts = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(contacts) + + m = InteractionC(target_path=get_ios_backup_folder()) + m.dependency_modules = {WhatsappContacts: contacts} + run_module(m) + + assert len(m.results) == 3 + + incoming = next( + r for r in m.results if r["sender_identifier"] == "100000000000001@lid" + ) + assert incoming["direction"] == "INCOMING" + assert incoming["sender_resolved_phone_number"] == "+14155550100" + assert incoming["sender_resolved_name"] == "Alice Example" + + outgoing = next( + r for r in m.results if r["direction"] == "OUTGOING" + ) + assert outgoing["recipient_identifier"] == "+14155550100" + assert outgoing["recipient_resolved_name"] == "Alice Example" + assert outgoing["domain_resolved_phone_number"] == "+14155550100" + assert outgoing["domain_resolved_name"] == "Alice Example" + + sms = next( + r for r in m.results if r["bundle_id"] == "com.apple.MobileSMS" + ) + assert sms.get("sender_resolved_name") is None + assert sms["sender_display_name"] == "Bob Example" + + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "Alice Example (+14155550100) to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "Alice Example (+14155550100)" in events + ) + assert ( + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" in events + ) + + # The creation date is only serialized when it diverges from the + # start date; the SMS record was created 90 days after the event. + creation_events = [ + entry + for entry in m.timeline + if entry["event"] == "interactions_creation_date" + ] + assert len(creation_events) == 1 + assert creation_events[0]["timestamp"] == "2025-12-09 12:26:40.000000" + assert creation_events[0]["data"] == ( + "Interaction record created 90 days after the event: " + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" + ) + + # Per-contact aggregate dates use contact-centric data strings. + first_seen = [ + entry + for entry in m.timeline + if entry["event"] == "first_incoming_sender_date" + ] + assert len(first_seen) == 1 + assert first_seen[0]["timestamp"] == "2025-09-03 13:46:40.000000" + assert first_seen[0]["data"] == ( + "First incoming interaction from Bob Example (+14155550101)" + ) + assert ( + "Last incoming interaction from Bob Example (+14155550101)" + in events + ) + + def test_extraction_without_whatsapp_contacts(self): + # Without the WhatsappContacts dependency the module still runs, and + # unresolvable LIDs are shown as-is. + m = InteractionC(target_path=get_ios_backup_folder()) + run_module(m) + + assert len(m.results) == 3 + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "100000000000001@lid to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "+14155550100" in events + ) diff --git a/tests/ios_backup/test_whatsapp.py b/tests/ios_backup/test_whatsapp.py index 864fb84..3bc920d 100644 --- a/tests/ios_backup/test_whatsapp.py +++ b/tests/ios_backup/test_whatsapp.py @@ -6,8 +6,79 @@ import logging from mvt.common.indicators import Indicators +from mvt.common.module import run_module from mvt.ios.modules.mixed.whatsapp import Whatsapp +from ..utils import get_ios_backup_folder + + +def test_extraction(): + m = Whatsapp(target_path=get_ios_backup_folder()) + run_module(m) + + messages = [r for r in m.results if "ZTEXT" in r] + sessions = [r for r in m.results if r.get("record_type") == "chat_session"] + pairs = [ + r for r in m.results + if r.get("record_type") == "lid_phone_number_pair" + ] + assert len(messages) == 3 + assert len(sessions) == 2 + assert len(pairs) == 1 + + assert pairs[0]["lid"] == "100000000000001" + assert pairs[0]["phone_number"] == "14155550100" + assert pairs[0]["pair_timestamp"] == "2025-08-25 07:33:20.000000" + + linked = next(r for r in messages if r.get("links")) + assert linked["links"] == ["https://example.org/news"] + + alice = next(s for s in sessions if s["partner_name"] == "Alice Example") + assert alice["contact_jid"] == "100000000000001@lid" + assert alice["partner_resolved_phone_number"] == "+14155550100" + assert alice["first_stored_message_date"] == "2025-08-27 15:06:40.000000" + assert alice["last_message_date"] == "2025-08-28 18:53:20.000000" + assert alice["group_creation_date"] is None + assert alice["stored_message_count"] == 2 + + group = next(s for s in sessions if s["partner_name"] == "Example Group") + assert group["group_creation_date"] == "2025-08-21 20:13:20.000000" + assert group["first_stored_message_date"] == "2025-08-29 22:40:00.000000" + # The last stored message predates the session's own last-message date: + # the newest message in this chat was deleted. + assert group["last_stored_message_date"] == "2025-08-29 22:40:00.000000" + assert group["last_message_date"] == "2025-08-31 02:26:40.000000" + + # 3 message events, first/last per chat, the group creation and the + # LID-phone number pair. + assert len(m.timeline) == 9 + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + # Alice's session is keyed by LID but labelled with the phone number + # resolved through LID.sqlite. + assert events[("chat_first_message", "2025-08-27 15:06:40.000000")] == ( + "First stored message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("chat_last_message", "2025-08-28 18:53:20.000000")] == ( + "Last message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("lid_pair_recorded", "2025-08-25 07:33:20.000000")] == ( + "WhatsApp associated LID 100000000000001 with " + "phone number 14155550100" + ) + assert events[("group_created", "2025-08-21 20:13:20.000000")] == ( + "WhatsApp group chat 'Example Group' " + "(120000000000000001@g.us) was created" + ) + assert ("chat_first_message", "2025-08-29 22:40:00.000000") in events + assert ("chat_last_message", "2025-08-31 02:26:40.000000") in events + + assert len(m.alertstore.alerts) == 0 + def test_collect_url_results_includes_expansion(): module = Whatsapp( diff --git a/tests/ios_backup/test_whatsapp_contacts.py b/tests/ios_backup/test_whatsapp_contacts.py new file mode 100644 index 0000000..a91b8f1 --- /dev/null +++ b/tests/ios_backup/test_whatsapp_contacts.py @@ -0,0 +1,83 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestWhatsappContactsModule: + def test_extraction(self): + m = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(m) + assert len(m.results) == 2 + + alice = next(r for r in m.results if r["given_name"] == "Alice") + assert alice["full_name"] == "Alice Example" + assert alice["phone_number"] == "+14155550100" + assert alice["whatsapp_id"] == "14155550100@s.whatsapp.net" + assert alice["lid"] == "100000000000001@lid" + assert alice["user_name"] == "alice.example" + assert alice["disappearing_mode_duration"] == 86400.0 + assert alice["disappearing_mode_is_on"] is True + assert alice["disappearing_mode_label"] == "24 hours" + assert alice["disappearing_mode_timestamp"] == "2025-07-23 21:46:40.000000" + assert alice["about_timestamp"] == "2025-07-12 08:00:00.000000" + assert alice["about_expiration_timestamp"] == "2025-08-16 01:20:00.000000" + assert alice["last_updated"] == "2025-08-04 11:33:20.000000" + + bob = next(r for r in m.results if r["given_name"] == "Bob") + assert bob["lid"] is None + assert bob["disappearing_mode_duration"] is None + assert bob["disappearing_mode_is_on"] is False + assert bob["disappearing_mode_label"] == "off" + assert bob["disappearing_mode_timestamp"] is None + + # Alice: disappearing_mode_set, about_changed, about_expiration and + # contact_last_updated. Bob: contact_last_updated only. + assert len(m.timeline) == 5 + + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + assert ( + "24 hours" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + "14155550100@s.whatsapp.net (Alice Example)" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + 'changed to "Hey there! I am using WhatsApp."' + in events[("about_changed", "2025-07-12 08:00:00.000000")] + ) + assert ( + "scheduled to expire" + in events[("about_expiration", "2025-08-16 01:20:00.000000")] + ) + + updated = [ + entry["data"] + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ] + assert len(updated) == 2 + assert all( + entry["timestamp"] == "2025-08-04 11:33:20.000000" + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ) + assert any("14155550101@s.whatsapp.net (Bob Example)" in d for d in updated) + + assert len(m.alertstore.alerts) == 0 + + def test_missing_database(self, tmp_path): + m = WhatsappContacts(target_path=str(tmp_path)) + run_module(m) + assert m.results == [] + assert len(m.alertstore.alerts) == 0 diff --git a/tests/ios_fs/test_filesystem.py b/tests/ios_fs/test_filesystem.py index 9fa664f..636c004 100644 --- a/tests/ios_fs/test_filesystem.py +++ b/tests/ios_fs/test_filesystem.py @@ -15,8 +15,8 @@ class TestFilesystem: def test_filesystem(self): m = Filesystem(target_path=get_ios_backup_folder()) run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 0 def test_detection(self, indicator_file): @@ -29,6 +29,6 @@ class TestFilesystem: ) m.indicators = ind run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 1 From dd8bd2cb01bb5275affebd93e71f311166014972 Mon Sep 17 00:00:00 2001 From: besendorf Date: Wed, 19 Aug 2026 16:46:31 +0200 Subject: [PATCH 06/43] Group custom docs under development (#878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Donncha Ó Cearbhaill --- mkdocs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 66f3471..a7b22ac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,7 +31,6 @@ nav: - Introduction: "introduction.md" - Installation: "install.md" - Command Completion: "command_completion.md" - - Custom CLI Commands: "custom_commands.md" - Using Docker: "docker.md" - MVT for iOS: - iOS Forensic Methodology: "ios/methodology.md" @@ -50,5 +49,7 @@ nav: - Check an Android Backup (SMS messages): "android/backup.md" - Check Android Intrusion Logs: "android/intrusion_logs.md" - Indicators of Compromise: "iocs.md" - - Development: "development.md" + - Development: + - Development Instructions: "development.md" + - Custom CLI Commands: "custom_commands.md" - License: "license.md" From dac4acb180457e2e44673d03d3a29dc845102b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Wed, 19 Aug 2026 23:15:48 +0200 Subject: [PATCH 07/43] Load installed module packages via entry points (#883) * Load installed module packages via entry points Python packages can already register custom CLI commands which load automatically, but custom modules still require --load-module or the MVT_CUSTOM_MODULES environment variable on every invocation. Add an mvt.modules entry-point group so installed packages can register forensic modules which load automatically into every module-running check-* command. An entry point resolves to an iterable of MVTModule subclasses, or a callable returning one. Broken entry points are skipped with a warning so a faulty package cannot break MVT. * Record the source of loaded modules for auditability Now that installed module packages load automatically, record where every module came from: - --list-modules groups the available modules by source, one line per source with the modules comma-separated: MVT itself with its version, each installed package with its version and VCS commit when recorded (PEP 610 direct_url.json), and each --load-module/MVT_CUSTOM_MODULES file with its SHA-256 hash. - Commands log one line per module source with its version or hash and the modules loaded from it, so command.log records exactly which modules ran and where they came from. - Make init_logging() idempotent: a loaded module package importing an MVT CLI module would previously add a second console handler and duplicate every console log line. * Route loaded module logging under the mvt.ext namespace Modules loaded from installed packages or file paths live outside the mvt logger hierarchy, so their log records never reach MVT's console and file handlers and instead fall through to logging.lastResort: alerts print as bare unformatted lines and INFO messages are dropped entirely. Add get_module_logger() and use it everywhere module loggers are created. Built-in mvt.* modules keep their existing logger names, and everything external is parented under a dedicated mvt.ext namespace so records reach the handlers and external names can never collide with MVT's internal logger tree. File-path modules are named after their file (mvt.ext.) instead of the mangled internal import name. Document a naming convention for community module packages: distribute as mvt-plugin- with import package mvt_plugin_, including the publishing organization in the name. The prefix is advisory (loading is by entry point, and it is no mark of authenticity), but conforming packages get a cleaner logger namespace: the mvt_plugin_ prefix is stripped, so mvt_plugin_amnesty_custom logs as mvt.ext.amnesty_custom. --- docs/development.md | 87 +++++++++++ src/mvt/common/cmd_check_iocs.py | 3 +- src/mvt/common/command.py | 39 ++++- src/mvt/common/help.py | 2 +- src/mvt/common/module_loader.py | 227 ++++++++++++++++++++++++++++- src/mvt/common/utils.py | 7 + tests/common/test_module_loader.py | 43 ++++++ tests/common/test_utils.py | 19 +++ tests/test_custom_modules.py | 150 +++++++++++++++++++ 9 files changed, 566 insertions(+), 11 deletions(-) diff --git a/docs/development.md b/docs/development.md index ffe1265..53a8f61 100644 --- a/docs/development.md +++ b/docs/development.md @@ -117,6 +117,93 @@ class DependentCustomModule(MVTModule): self.results = [{"manifest_entries": len(manifest_results)}] ``` +## Installed module packages + +Python packages can register modules so they load automatically in every +module-running `check-*` command, without `--load-module` or +`MVT_CUSTOM_MODULES`. Register an entry point in the `mvt.modules` group in +the package's `pyproject.toml`: + +```toml +[project.entry-points."mvt.modules"] +mvt-plugin-amnesty-custom = "mvt_plugin_amnesty_custom:get_modules" +``` + +The entry point must resolve to an iterable of `MVTModule` subclasses, or to +a callable returning one: + +```python +from mvt.common.module import MVTModule + + +class PackagedModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + def run(self): + self.results = [{"message": "packaged module ran"}] + + +def get_modules() -> list[type[MVTModule]]: + return [PackagedModule] +``` + +Installed modules follow the same rules as other custom modules: each module +must declare `supported_commands`, and dependencies are resolved with the +standard ordering logic. A broken entry point is skipped with a warning and +does not prevent MVT from running. As with custom commands, installed module +packages run as trusted code inside the MVT process, so install only packages +from sources you trust. + +For a `pipx` installation of MVT, inject the package into MVT's environment: + +```bash +pipx inject mvt mvt-plugin-amnesty-custom +``` + +### Naming module packages + +Name module packages `mvt-plugin-` (import package `mvt_plugin_`), +and include the name of the publishing organization or author so packages from +different groups do not collide: for example, Amnesty International's custom +modules would be distributed as `mvt-plugin-amnesty-custom` with the import +package `mvt_plugin_amnesty_custom`. + +The prefix makes module packages easy to find on PyPI and keeps their import +names from clashing with unrelated Python packages. It is a convention, not a +technical requirement: modules load through the `mvt.modules` entry point +regardless of what the package is called, and existing packages with other +names keep working. Note that the prefix is also not a mark of authenticity — +anyone can publish a package with any available name, so vet a module package +and its publisher before installing it, whatever it is called. + +### Module logging + +Modules log through `self.log`, and MVT names the logger for where the module +came from. MVT's own modules log under their dotted path (for example +`mvt.ios.modules.mixed.whatsapp`). Everything external is namespaced under +`mvt.ext` to keep it visually distinct from built-in modules and isolated from +MVT's internal logger tree: + +- Installed packages log under `mvt.ext.`, with the `mvt_plugin_` + prefix stripped: modules in `mvt_plugin_amnesty_custom` log as + `mvt.ext.amnesty_custom.*`. +- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as + `mvt.ext.`. + +## Auditing loaded modules + +Because installed module packages load automatically, MVT records where every +module came from: + +- `--list-modules` groups the available modules by source: MVT itself + (with its version), each installed package (with its version and, when + installed directly from a repository, the commit), and each file loaded + with `--load-module` or `MVT_CUSTOM_MODULES` (with the SHA-256 hash of the + file). +- When a command runs with an `--output` folder, the `command.log` file + records one line per module source with the source's version or hash and + the list of modules loaded from it. + ## Profiling Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be diff --git a/src/mvt/common/cmd_check_iocs.py b/src/mvt/common/cmd_check_iocs.py index c227659..c2dbdcd 100644 --- a/src/mvt/common/cmd_check_iocs.py +++ b/src/mvt/common/cmd_check_iocs.py @@ -9,6 +9,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.module import MVTModule +from mvt.common.module_loader import get_module_logger from mvt.common.utils import exec_or_profile log = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class CmdCheckIOCS(Command): ) m = iocs_module.from_json( - file_path, log=logging.getLogger(iocs_module.__module__) + file_path, log=get_module_logger(iocs_module) ) if not m: log.warning("No result from this module, skipping it") diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index aab53e6..6111dc7 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -19,7 +19,12 @@ from .alerts import AlertLevel, AlertStore from .config import settings from .indicators import Indicators from .module import EncryptedBackupError, MVTModule, run_module, save_timeline -from .module_loader import module_supports_command +from .module_loader import ( + ModuleOrigin, + get_module_logger, + get_module_origin, + module_supports_command, +) from .module_types import ModuleTimeline, URLResult from .utils import ( CustomJSONEncoder, @@ -210,10 +215,34 @@ class Command: for file in generate_hashes_from_path(self.target_path, self.log): self.hash_values.append(file) + @staticmethod + def _modules_by_origin( + modules: list[type[MVTModule]], + ) -> dict[ModuleOrigin, list[str]]: + origins: dict[ModuleOrigin, list[str]] = {} + for module in modules: + origins.setdefault(get_module_origin(module), []).append(module.__name__) + return origins + def list_modules(self) -> None: self.log.info("Following is the list of available %s modules:", self.name) - for module in self._available_modules(): - self.log.info(" - %s", module.__name__) + for origin, module_names in self._modules_by_origin( + self._available_modules() + ).items(): + self.log.info( + " - Modules from %s: %s", origin.label, ", ".join(module_names) + ) + + def _log_loaded_modules(self, modules: list[type[MVTModule]]) -> None: + """Record the loaded modules and their origins for auditability.""" + for origin, module_names in self._modules_by_origin(modules).items(): + self.log.info( + "Loaded %d %s modules from %s: %s", + len(module_names), + self.name, + origin.label, + ", ".join(module_names), + ) def _available_modules(self) -> list[type[MVTModule]]: modules = list(self.modules) @@ -360,6 +389,8 @@ class Command: if ordered_modules is None: return + self._log_loaded_modules(ordered_modules) + try: self.init() except NotImplementedError: @@ -368,7 +399,7 @@ class Command: executed_by_type: dict[type[MVTModule], MVTModule] = {} for module in ordered_modules: - module_logger = logging.getLogger(module.__module__) + module_logger = get_module_logger(module) m = module( target_path=self.target_path, diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 5101f93..7514113 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -8,7 +8,7 @@ HELP_MSG_VERSION = "Show the currently installed version of MVT" HELP_MSG_OUTPUT = "Specify a path to a folder where you want to store JSON results" HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)" HELP_MSG_FAST = "Avoid running time/resource consuming features" -HELP_MSG_LIST_MODULES = "Print list of available modules and exit" +HELP_MSG_LIST_MODULES = "Print list of available modules and their source, then exit" HELP_MSG_MODULE = "Name of a single module you would like to run instead of all" HELP_MSG_LOAD_MODULE = ( "Load custom MVT module(s) from a Python file or folder " diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index da94826..956da4c 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -4,18 +4,28 @@ # https://license.mvt.re/1.1/ import hashlib +import importlib.metadata import importlib.util import inspect +import json import logging import os import sys +from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from types import ModuleType from typing import Iterable, Optional from .module import MVTModule +from .version import MVT_VERSION MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES" +MODULES_ENTRY_POINT_GROUP = "mvt.modules" +EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" +PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" +_ORIGIN_ATTRIBUTE = "_mvt_module_origin" +_PATH_MODULE_PREFIX = "_mvt_custom_module_" log = logging.getLogger(__name__) @@ -23,15 +33,74 @@ class CustomModuleLoadError(Exception): pass +@dataclass(frozen=True) +class ModuleOrigin: + """Describes where a loaded module came from, for auditability. + + ``kind`` is one of ``builtin`` (shipped with MVT), ``package`` (loaded + from an installed package) or ``path`` (loaded from a file passed with + ``--load-module`` or the environment variable). + """ + + kind: str + name: str + version: Optional[str] = None + commit: Optional[str] = None + file_sha256: Optional[str] = None + + @property + def label(self) -> str: + label = self.name + if self.version: + label += f"@{self.version}" + label = f"'{label}'" + if self.commit: + label += f" (commit {self.commit})" + if self.file_sha256: + label += f" (sha256: {self.file_sha256})" + return label + + def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_module_{path.stem}_{digest}" + return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}" + + +def get_module_logger(module_class: type[MVTModule]) -> logging.Logger: + """Return the logger a module's records should be emitted through. + + Modules loaded from installed packages or file paths live outside the + "mvt" logger hierarchy, so their records would never reach the handlers + attached to the "mvt" logger and instead fall through to + logging.lastResort (which prints bare messages and drops anything below + WARNING). Their loggers are parented under the "mvt.ext" namespace, + keeping external module names from colliding with MVT's own logger + tree. File-path modules are named after their file instead of the + mangled internal import name, and packages following the recommended + "mvt_plugin_" naming convention log under "mvt.ext.". + """ + name = module_class.__module__ + if name == "mvt" or name.startswith("mvt."): + return logging.getLogger(name) + + if name.startswith(_PATH_MODULE_PREFIX): + name = Path(get_module_origin(module_class).name).stem + else: + top_level, separator, rest = name.partition(".") + if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len( + PLUGIN_PACKAGE_PREFIX + ): + name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest + + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{name}") def _iter_module_files(path: Path) -> Iterable[Path]: if path.is_file(): if path.suffix != ".py": - raise CustomModuleLoadError(f"Custom module file is not a Python file: {path}") + raise CustomModuleLoadError( + f"Custom module file is not a Python file: {path}" + ) yield path return @@ -59,7 +128,9 @@ def _load_python_file(path: Path) -> ModuleType: try: spec.loader.exec_module(module) except Exception as exc: - raise CustomModuleLoadError(f"Unable to import custom module {path}: {exc}") from exc + raise CustomModuleLoadError( + f"Unable to import custom module {path}: {exc}" + ) from exc return module @@ -84,17 +155,156 @@ def load_custom_modules_from_path(path: str) -> list[type[MVTModule]]: resolved_path = Path(path).expanduser().resolve() for module_file in _iter_module_files(resolved_path): + file_sha256 = hashlib.sha256(module_file.read_bytes()).hexdigest() loaded_module = _load_python_file(module_file) + origin = ModuleOrigin( + kind="path", name=str(module_file), file_sha256=file_sha256 + ) for module_class in discover_mvt_modules(loaded_module): key = (str(module_file), module_class.__qualname__) if key in seen: continue seen.add(key) + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) custom_modules.append(module_class) return custom_modules +def _module_key(module_class: type[MVTModule]) -> tuple[str, str]: + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return (source, module_class.__qualname__) + + +def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]: + """Return the VCS commit a distribution was installed from, if recorded. + + Packages installed directly from a repository (``pip install git+...``) + record the commit in ``direct_url.json`` (PEP 610). + """ + try: + direct_url_text = dist.read_text("direct_url.json") + if not direct_url_text: + return None + commit = json.loads(direct_url_text).get("vcs_info", {}).get("commit_id") + return commit if isinstance(commit, str) else None + except Exception: + return None + + +def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin: + name = entry_point.name + version = None + commit = None + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is not None: + try: + name = dist.name or name + version = dist.version + except Exception: + pass + commit = _distribution_commit(dist) + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + +@lru_cache(maxsize=1) +def _packages_distributions() -> dict[str, list[str]]: + try: + return dict(importlib.metadata.packages_distributions()) + except Exception: + return {} + + +def get_module_origin(module_class: type[MVTModule]) -> ModuleOrigin: + """Return the origin of a module class for auditing purposes.""" + origin = module_class.__dict__.get(_ORIGIN_ATTRIBUTE) + if isinstance(origin, ModuleOrigin): + return origin + + top_level = module_class.__module__.partition(".")[0] + if top_level == "mvt": + return ModuleOrigin(kind="builtin", name="mvt", version=MVT_VERSION) + + distributions = _packages_distributions().get(top_level) + if distributions: + name = distributions[0] + version = None + commit = None + try: + dist = importlib.metadata.distribution(name) + version = dist.version + commit = _distribution_commit(dist) + except Exception: + pass + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return ModuleOrigin(kind="path", name=source) + + +def load_installed_modules() -> list[type[MVTModule]]: + """Load MVT modules registered by installed packages. + + Packages register modules in the ``mvt.modules`` entry-point group. Each + entry point must resolve to an iterable of MVTModule subclasses, or to a + callable which returns one. A broken entry point is skipped with a + warning so that a faulty plugin package cannot break MVT. + """ + try: + entry_points = importlib.metadata.entry_points(group=MODULES_ENTRY_POINT_GROUP) + except Exception as exc: + log.warning( + "Unable to discover installed module packages in entry-point group %s: %s", + MODULES_ENTRY_POINT_GROUP, + exc, + ) + return [] + + installed_modules: list[type[MVTModule]] = [] + ordered_entry_points = sorted( + entry_points, key=lambda entry_point: (entry_point.name, entry_point.value) + ) + for entry_point in ordered_entry_points: + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + module_classes = list(loaded) + except (Exception, SystemExit) as exc: + log.warning( + "Unable to load modules from entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + continue + + origin = _entry_point_origin(entry_point) + for module_class in module_classes: + if not ( + isinstance(module_class, type) and issubclass(module_class, MVTModule) + ): + log.warning( + "Entry point %s (%s) provided %r which is not an " + "MVTModule subclass", + entry_point.name, + entry_point.value, + module_class, + ) + continue + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) + installed_modules.append(module_class) + + return installed_modules + + def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTModule]]: search_paths: list[str] = [] env_path = os.environ.get(MVT_CUSTOM_MODULES_ENV) @@ -105,10 +315,17 @@ def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTM custom_modules: list[type[MVTModule]] = [] seen: set[tuple[str, str]] = set() + + for module_class in load_installed_modules(): + key = _module_key(module_class) + if key in seen: + continue + seen.add(key) + custom_modules.append(module_class) + for path in search_paths: for module_class in load_custom_modules_from_path(path): - source = Path(inspect.getfile(module_class)).resolve() - key = (str(source), module_class.__qualname__) + key = _module_key(module_class) if key in seen: continue seen.add(key) diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index 30de159..ad3b394 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -239,6 +239,13 @@ def init_logging(verbose: bool = False): """ log = logging.getLogger("mvt") log.setLevel(logging.DEBUG) + + # Importing an MVT CLI module calls init_logging() at import time, and + # loaded module packages may import one indirectly. Keep this idempotent + # so console log lines are not duplicated by a second handler. + if any(isinstance(handler, MVTLogHandler) for handler in log.handlers): + return + consoleHandler = MVTLogHandler() consoleHandler.setFormatter(logging.Formatter("%(message)s")) if verbose: diff --git a/tests/common/test_module_loader.py b/tests/common/test_module_loader.py index 15b6052..dec636e 100644 --- a/tests/common/test_module_loader.py +++ b/tests/common/test_module_loader.py @@ -3,10 +3,12 @@ import pytest from mvt.common.module import MVTModule from mvt.common.module_loader import ( CustomModuleLoadError, + get_module_logger, load_custom_modules, load_custom_modules_from_path, module_supports_command, ) +from mvt.ios.modules.mixed.whatsapp import Whatsapp MODULE_TEMPLATE = """ @@ -144,3 +146,44 @@ def test_module_supports_command_honors_supported_commands(tmp_path): assert module_supports_command(module, "ios", "check-backup") assert not module_supports_command(module, "ios", "check-fs") + + +def test_get_module_logger_keeps_builtin_names(): + assert get_module_logger(Whatsapp).name == "mvt.ios.modules.mixed.whatsapp" + + +def test_get_module_logger_parents_package_modules_under_mvt_ext(): + class PackageModule(MVTModule): + pass + + PackageModule.__module__ = "some_plugin_package.ios.custom" + + assert ( + get_module_logger(PackageModule).name + == "mvt.ext.some_plugin_package.ios.custom" + ) + + +def test_get_module_logger_strips_the_plugin_package_prefix(): + class PluginModule(MVTModule): + pass + + PluginModule.__module__ = "mvt_plugin_amnesty_custom.ios.custom" + + assert get_module_logger(PluginModule).name == "mvt.ext.amnesty_custom.ios.custom" + + +def test_get_module_logger_only_strips_the_prefix_from_the_top_level(): + class NestedModule(MVTModule): + pass + + NestedModule.__module__ = "other_package.mvt_plugin_sub" + + assert get_module_logger(NestedModule).name == "mvt.ext.other_package.mvt_plugin_sub" + + +def test_get_module_logger_names_path_modules_after_their_file(tmp_path): + module_path = _write_module(tmp_path / "my_custom_module.py", "PathModule") + module = load_custom_modules_from_path(str(module_path))[0] + + assert get_module_logger(module).name == "mvt.ext.my_custom_module" diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index 4dbe5c0..b8791e1 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -8,6 +8,7 @@ import logging import os from datetime import datetime +from mvt.common.log import MVTLogHandler from mvt.common.utils import ( CustomJSONEncoder, convert_datetime_to_iso, @@ -16,6 +17,7 @@ from mvt.common.utils import ( convert_unix_to_utc_datetime, generate_hashes_from_path, get_sha256_from_file_path, + init_logging, ) from ..utils import get_artifact_folder @@ -103,3 +105,20 @@ class TestCustomJSONEncoder: json.dumps({"name": "家".encode()}, cls=CustomJSONEncoder) == '{"name": "\\u5bb6"}' ) + + +class TestInitLogging: + def test__init_logging_is_idempotent(self): + # Loaded module packages may import an MVT CLI module, which calls + # init_logging() again at import time. A second call must not add + # a duplicate console handler. + log = logging.getLogger("mvt") + init_logging() + handler_count = sum( + isinstance(handler, MVTLogHandler) for handler in log.handlers + ) + init_logging() + assert ( + sum(isinstance(handler, MVTLogHandler) for handler in log.handlers) + == handler_count + ) diff --git a/tests/test_custom_modules.py b/tests/test_custom_modules.py index a00faea..43ab621 100644 --- a/tests/test_custom_modules.py +++ b/tests/test_custom_modules.py @@ -1,3 +1,7 @@ +import hashlib +import importlib.metadata +import json + from click.testing import CliRunner from mvt.android.cli import check_bugreport @@ -5,7 +9,9 @@ from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF from mvt.android.cmd_check_backup import CmdAndroidCheckBackup from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs +from mvt.common import module_loader from mvt.common.module import MVTModule +from mvt.common.version import MVT_VERSION from mvt.ios.cli import check_backup, check_fs @@ -106,6 +112,150 @@ def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeyp assert "EnvBugreportModule" in result.output +class InstalledPackageModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + +def get_installed_package_modules(): + return [InstalledPackageModule] + + +def _fake_entry_points(monkeypatch, value, name="test-modules"): + entry_point = importlib.metadata.EntryPoint( + name=name, value=value, group=module_loader.MODULES_ENTRY_POINT_GROUP + ) + + def fake_entry_points(*, group): + assert group == module_loader.MODULES_ENTRY_POINT_GROUP + return [entry_point] + + monkeypatch.setattr( + module_loader.importlib.metadata, "entry_points", fake_entry_points + ) + + +def test_installed_module_package_loads_from_entry_point(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + assert modules == [InstalledPackageModule] + + +def test_broken_module_entry_point_is_skipped(monkeypatch, caplog): + _fake_entry_points(monkeypatch, "nonexistent_module_xyz:get_modules") + + with caplog.at_level("WARNING"): + modules = module_loader.load_custom_modules() + + assert modules == [] + assert "Unable to load modules from entry point" in caplog.text + + +def test_entry_point_module_deduplicated_against_paths(monkeypatch, tmp_path): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + module_path = _write_custom_module( + tmp_path / "custom.py", + "PathLoadedModule", + (("ios", "check-backup"),), + ) + + modules = module_loader.load_custom_modules([str(module_path)]) + + assert [module.__name__ for module in modules] == [ + "InstalledPackageModule", + "PathLoadedModule", + ] + + +def test_list_modules_shows_module_sources(tmp_path, caplog): + module_path = _write_custom_module( + tmp_path / "custom.py", + "SourcedBackupModule", + (("ios", "check-backup"),), + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + custom_modules = module_loader.load_custom_modules([str(module_path)]) + + from mvt.ios.cmd_check_backup import CmdIOSCheckBackup + + cmd = CmdIOSCheckBackup(target_path=str(tmp_path), custom_modules=custom_modules) + cmd.list_modules() + + assert f" - Modules from 'mvt@{MVT_VERSION}':" in caplog.text + assert ( + f" - Modules from '{module_path}' (sha256: {file_sha256}): SourcedBackupModule" + in caplog.text + ) + + +def test_builtin_module_origin(): + from mvt.ios.modules.backup import BACKUP_MODULES + + origin = module_loader.get_module_origin(BACKUP_MODULES[0]) + + assert origin.kind == "builtin" + assert origin.name == "mvt" + assert origin.version == MVT_VERSION + + +def test_installed_module_origin(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + origin = module_loader.get_module_origin(modules[0]) + assert origin.kind == "package" + assert origin.name == "test-modules" + + +def test_distribution_commit_read_from_direct_url(): + class FakeDistribution: + def read_text(self, filename): + assert filename == "direct_url.json" + return json.dumps( + { + "url": "https://github.com/example/example-modules", + "vcs_info": {"commit_id": "abc1234", "vcs": "git"}, + } + ) + + assert module_loader._distribution_commit(FakeDistribution()) == "abc1234" + + +def test_command_log_records_loaded_modules(tmp_path): + (tmp_path / "Manifest.db").touch() + (tmp_path / "Info.plist").touch() + module_path = _write_custom_module( + tmp_path / "custom.py", + "AuditedRunModule", + (("ios", "check-backup"),), + slug="audited_run_module", + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + output_path = tmp_path / "out" + + result = CliRunner().invoke( + check_backup, + [ + "--module", + "AuditedRunModule", + "--load-module", + str(module_path), + "--output", + str(output_path), + str(tmp_path), + ], + ) + + assert result.exit_code == 0 + command_log = (output_path / "command.log").read_text(encoding="utf-8") + assert ( + f"Loaded 1 check-backup modules from '{module_path}' " + f"(sha256: {file_sha256}): AuditedRunModule" in command_log + ) + + class NestedBugreportModule(MVTModule): supported_commands = (("android", "check-bugreport"),) From dc0650bf76d75fc889b11fc5f1c0a83c643c77f8 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 08/43] Fix bugreport command section extraction --- src/mvt/android/artifacts/getprop.py | 8 +++--- src/mvt/android/modules/bugreport/base.py | 27 ++++++++++++++++++- .../modules/bugreport/dumpsys_getprop.py | 21 +++------------ tests/android/test_artifact_getprop.py | 9 +++++++ 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index debcfc4..a03491b 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -28,18 +28,18 @@ INTERESTING_PROPERTIES = [ class GetProp(AndroidArtifact): def parse(self, entry: str) -> None: self.results: List[Dict[str, str]] = [] - rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]") + rxp = re.compile(r"^\[([^]]+)\]: \[(.*)\]$") for line in entry.splitlines(): line = line.strip() if line == "": continue - matches = re.findall(rxp, line) - if not matches or len(matches[0]) != 2: + match = rxp.match(line) + if not match: continue - prop_entry = {"name": matches[0][0], "value": matches[0][1]} + prop_entry = {"name": match.group(1), "value": match.group(2)} self.results.append(prop_entry) def get_device_timezone(self) -> str | None: diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index 156e01c..01af8c9 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -72,7 +72,11 @@ class BugReportModule(MVTModule): if not self.extract_path: raise ValueError("extract_path is not set") joined = os.path.join(self.extract_path, file_path) - if not Path(joined).resolve().is_relative_to(Path(self.extract_path).resolve()): + if ( + not Path(joined) + .resolve() + .is_relative_to(Path(self.extract_path).resolve()) + ): raise ValueError("unsafe file_path") handle = open(joined, "rb") @@ -100,6 +104,27 @@ class BugReportModule(MVTModule): return None + @staticmethod + def extract_command_section(content: str, heading: str) -> str: + """Return a bugreport command section without consuming the next one. + + Bugreport separators include timing text, so looking for a line equal to + ``------`` is not sufficient and can accidentally feed the remainder of + dumpstate to a parser. + """ + lines: list[str] = [] + in_section = False + for line in content.splitlines(): + stripped = line.strip() + if not in_section: + if stripped.startswith(heading): + in_section = True + continue + if stripped.startswith("------"): + break + lines.append(line) + return "\n".join(lines) + def _get_file_modification_time(self, file_path: str) -> datetime.datetime: if self.zip_archive: file_timetuple = self.zip_archive.getinfo(file_path).date_time diff --git a/src/mvt/android/modules/bugreport/dumpsys_getprop.py b/src/mvt/android/modules/bugreport/dumpsys_getprop.py index 198a1d2..b9c20ea 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_getprop.py +++ b/src/mvt/android/modules/bugreport/dumpsys_getprop.py @@ -44,21 +44,8 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule): ) return - lines = [] - in_getprop = False - - for line in content.decode(errors="ignore").splitlines(): - if line.strip().startswith("------ SYSTEM PROPERTIES"): - in_getprop = True - continue - - if not in_getprop: - continue - - if line.strip() == "------": - break - - lines.append(line) - - self.parse("\n".join(lines)) + section = self.extract_command_section( + content.decode(errors="ignore"), "------ SYSTEM PROPERTIES" + ) + self.parse(section) self.log.info("Extracted %d Android system properties", len(self.results)) diff --git a/tests/android/test_artifact_getprop.py b/tests/android/test_artifact_getprop.py index d9fec6b..7815fbd 100644 --- a/tests/android/test_artifact_getprop.py +++ b/tests/android/test_artifact_getprop.py @@ -39,3 +39,12 @@ class TestGetPropArtifact: assert len(gp.alertstore.alerts) == 0 gp.check_indicators() assert len(gp.alertstore.alerts) == 1 + + def test_empty_values_and_invalid_lines(self): + gp = GetProp() + gp.parse("[empty]: []\n[valid]: [value]\n0\n[broken]: [value") + + assert gp.results == [ + {"name": "empty", "value": ""}, + {"name": "valid", "value": "value"}, + ] From ab879bb23b8c38ff436dc6b25e2ede54511176b3 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 09/43] Parse all package resolver categories --- .../artifacts/dumpsys_package_activities.py | 61 +------- .../android/artifacts/dumpsys_receivers.py | 147 +++++------------- .../android/artifacts/package_resolvers.py | 70 +++++++++ .../modules/bugreport/dumpsys_receivers.py | 4 +- ...est_artifact_dumpsys_package_activities.py | 14 +- .../test_artifact_dumpsys_receivers.py | 33 ++-- 6 files changed, 134 insertions(+), 195 deletions(-) create mode 100644 src/mvt/android/artifacts/package_resolvers.py diff --git a/src/mvt/android/artifacts/dumpsys_package_activities.py b/src/mvt/android/artifacts/dumpsys_package_activities.py index 31c6bbd..443d18f 100644 --- a/src/mvt/android/artifacts/dumpsys_package_activities.py +++ b/src/mvt/android/artifacts/dumpsys_package_activities.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +from .package_resolvers import parse_resolver_table class DumpsysPackageActivitiesArtifact(AndroidArtifact): @@ -19,67 +20,11 @@ class DumpsysPackageActivitiesArtifact(AndroidArtifact): ) continue - def parse(self, content: str): + def parse(self, content: str) -> None: """ Parse the Dumpsys Package section for activities Adds results to self.results :param content: content of the package section (string) """ - self.results = [] - - in_activity_resolver_table = False - in_non_data_actions = False - intent = None - for line in content.splitlines(): - if line.startswith("Activity Resolver Table:"): - in_activity_resolver_table = True - continue - - if not in_activity_resolver_table: - continue - - if line.startswith(" Non-Data Actions:"): - in_non_data_actions = True - continue - - if not in_non_data_actions: - continue - - # If we hit an empty line, the Non-Data Actions section should be - # finished. - if line.strip() == "": - break - - # We detect the action name. - if ( - line.startswith(" " * 6) - and not line.startswith(" " * 8) - and ":" in line - ): - intent = line.strip().replace(":", "") - continue - - # If we are not in an intent block yet, skip. - if not intent: - continue - - # If we are in a block but the line does not start with 8 spaces - # it means the block ended a new one started, so we reset and - # continue. - if not line.startswith(" " * 8): - intent = None - continue - - # If we got this far, we are processing receivers for the - # activities we are interested in. - activity = line.strip().split(" ")[1] - package_name = activity.split("/")[0] - - self.results.append( - { - "intent": intent, - "package_name": package_name, - "activity": activity, - } - ) + self.results = parse_resolver_table(content, "Activity") diff --git a/src/mvt/android/artifacts/dumpsys_receivers.py b/src/mvt/android/artifacts/dumpsys_receivers.py index b437930..7594335 100644 --- a/src/mvt/android/artifacts/dumpsys_receivers.py +++ b/src/mvt/android/artifacts/dumpsys_receivers.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +from .package_resolvers import parse_resolver_table INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS" INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED" @@ -18,115 +19,45 @@ class DumpsysReceiversArtifact(AndroidArtifact): """ def check_indicators(self) -> None: - for intent, receivers in self.results.items(): - for receiver in receivers: - if intent == INTENT_NEW_OUTGOING_SMS: - self.log.info( - 'Found a receiver to intercept outgoing SMS messages: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_SMS_RECEIVED: - self.log.info( - 'Found a receiver to intercept incoming SMS messages: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_DATA_SMS_RECEIVED: - self.log.info( - 'Found a receiver to intercept incoming data SMS message: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_PHONE_STATE: - self.log.info( - "Found a receiver monitoring " - 'telephony state/incoming calls: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_NEW_OUTGOING_CALL: - self.log.info( - 'Found a receiver monitoring outgoing calls: "%s"', - receiver["receiver"], - ) + for receiver in self.results: + intent = receiver["key"] + if intent == INTENT_NEW_OUTGOING_SMS: + self.log.info( + 'Found a receiver to intercept outgoing SMS messages: "%s"', + receiver["component"], + ) + elif intent == INTENT_SMS_RECEIVED: + self.log.info( + 'Found a receiver to intercept incoming SMS messages: "%s"', + receiver["component"], + ) + elif intent == INTENT_DATA_SMS_RECEIVED: + self.log.info( + 'Found a receiver to intercept incoming data SMS message: "%s"', + receiver["component"], + ) + elif intent == INTENT_PHONE_STATE: + self.log.info( + 'Found a receiver monitoring telephony state/incoming calls: "%s"', + receiver["component"], + ) + elif intent == INTENT_NEW_OUTGOING_CALL: + self.log.info( + 'Found a receiver monitoring outgoing calls: "%s"', + receiver["component"], + ) - if not self.indicators: - continue + if not self.indicators: + continue - ioc_match = self.indicators.check_app_id(receiver["package_name"]) - if ioc_match: - self.alertstore.critical( - ioc_match.message, - "", - {intent: receiver}, - matched_indicator=ioc_match.ioc, - ) - continue + ioc_match = self.indicators.check_app_id(receiver["package_name"]) + if ioc_match: + self.alertstore.critical( + ioc_match.message, + "", + receiver, + matched_indicator=ioc_match.ioc, + ) def parse(self, output: str) -> None: - self.results: dict[str, list[dict[str, str]]] = {} - - in_receiver_resolver_table = False - in_non_data_actions = False - intent = None - for line in output.splitlines(): - if line.startswith("Receiver Resolver Table:"): - in_receiver_resolver_table = True - continue - - if not in_receiver_resolver_table: - continue - - if line.startswith(" Non-Data Actions:"): - in_non_data_actions = True - continue - - if not in_non_data_actions: - continue - - # If we hit an empty line, the Non-Data Actions section should be - # finished. - if line.strip() == "": - break - - # We detect the action name. - if ( - line.startswith(" " * 6) - and not line.startswith(" " * 8) - and ":" in line - ): - intent = line.strip().replace(":", "") - self.results[intent] = [] - continue - - parts = line.strip().split(" ") - if len(parts) < 2: - # A single-token line here is not a receiver. Real dumpstate - # output can print an action header mis-indented (observed with - # 15 leading spaces instead of 6), which used to raise - # IndexError and abort the whole module. Treat a trailing-colon - # token as the next action, skip anything else. - if parts[0].endswith(":"): - intent = parts[0][:-1] - self.results.setdefault(intent, []) - continue - - # If we are not in an intent block yet, skip. - if not intent: - continue - - # If we are in a block but the line does not start with 8 spaces - # it means the block ended a new one started, so we reset and - # continue. - if not line.startswith(" " * 8): - intent = None - continue - - # If we got this far, we are processing receivers for the - # activities we are interested in. - receiver = parts[1] - package_name = receiver.split("/")[0] - - self.results[intent].append( - { - "package_name": package_name, - "receiver": receiver, - } - ) + self.results = parse_resolver_table(output, "Receiver") diff --git a/src/mvt/android/artifacts/package_resolvers.py b/src/mvt/android/artifacts/package_resolvers.py new file mode 100644 index 0000000..60c2212 --- /dev/null +++ b/src/mvt/android/artifacts/package_resolvers.py @@ -0,0 +1,70 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import re + + +RESOLVER_TYPES = { + "Full MIME Types": "full_mime_type", + "Base MIME Types": "base_mime_type", + "Wild MIME Types": "wild_mime_type", + "Schemes": "scheme", + "Non-Data Actions": "non_data_action", + "MIME Typed Actions": "mime_typed_action", +} + +ENTRY_RE = re.compile( + r"^\s{8,}[0-9a-fA-F]+\s+(?P\S+)" + r"(?:\s+\((?P\d+)\s+filters?\))?\s*$" +) + + +def parse_resolver_table(content: str, table_name: str) -> list[dict]: + """Parse every resolver category from a dumpsys package table.""" + results: list[dict] = [] + in_table = False + resolver_type: str | None = None + key: str | None = None + + for line in content.splitlines(): + if line.startswith(f"{table_name} Resolver Table:"): + in_table = True + continue + if not in_table: + continue + + if line and not line.startswith(" "): + break + + section_match = re.match(r"^ {2}([^ ].*):\s*$", line) + if section_match: + resolver_type = RESOLVER_TYPES.get(section_match.group(1)) + key = None + continue + if resolver_type is None: + continue + + key_match = re.match(r"^ {6,}([^ ].*):\s*$", line) + if key_match: + key = key_match.group(1) + continue + if key is None: + continue + + entry_match = ENTRY_RE.match(line) + if not entry_match: + continue + component = entry_match.group("component") + results.append( + { + "resolver_type": resolver_type, + "key": key, + "package_name": component.split("/", 1)[0], + "component": component, + "filter_count": int(entry_match.group("filters") or 1), + } + ) + + return results diff --git a/src/mvt/android/modules/bugreport/dumpsys_receivers.py b/src/mvt/android/modules/bugreport/dumpsys_receivers.py index 8907d5e..a82f021 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_receivers.py +++ b/src/mvt/android/modules/bugreport/dumpsys_receivers.py @@ -33,7 +33,7 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule): results=results, ) - self.results = results if results else {} + self.results = results if results else [] def run(self) -> None: content = self._get_dumpstate_file() @@ -49,4 +49,4 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule): ) self.parse(dumpsys_section) - self.log.info("Extracted receivers for %d intents", len(self.results)) + self.log.info("Extracted %d package receivers", len(self.results)) diff --git a/tests/android/test_artifact_dumpsys_package_activities.py b/tests/android/test_artifact_dumpsys_package_activities.py index 5eab63d..66ef790 100644 --- a/tests/android/test_artifact_dumpsys_package_activities.py +++ b/tests/android/test_artifact_dumpsys_package_activities.py @@ -21,12 +21,16 @@ class TestDumpsysPackageActivitiesArtifact: assert len(dpa.results) == 0 dpa.parse(data) - assert len(dpa.results) == 4 - assert dpa.results[0]["package_name"] == "com.samsung.android.app.social" + assert len(dpa.results) == 10 + assert dpa.results[0]["package_name"] == "com.samsung.android.messaging" assert ( - dpa.results[0]["activity"] - == "com.samsung.android.app.social/.feed.FeedsActivity" + dpa.results[0]["component"] + == "com.samsung.android.messaging/.ui.RcsTransferContent" ) + assert {result["resolver_type"] for result in dpa.results} == { + "full_mime_type", + "non_data_action", + } def test_ioc_check(self, indicator_file): dpa = DumpsysPackageActivitiesArtifact() @@ -41,4 +45,4 @@ class TestDumpsysPackageActivitiesArtifact: dpa.indicators = ind assert len(dpa.alertstore.alerts) == 0 dpa.check_indicators() - assert len(dpa.alertstore.alerts) == 1 + assert len(dpa.alertstore.alerts) == 2 diff --git a/tests/android/test_artifact_dumpsys_receivers.py b/tests/android/test_artifact_dumpsys_receivers.py index 7875a52..6e8b92e 100644 --- a/tests/android/test_artifact_dumpsys_receivers.py +++ b/tests/android/test_artifact_dumpsys_receivers.py @@ -19,17 +19,14 @@ class TestDumpsysReceiversArtifact: assert len(dr.results) == 0 dr.parse(data) - assert len(dr.results) == 4 - assert ( - list(dr.results.keys())[0] - == "com.android.storagemanager.automatic.SHOW_NOTIFICATION" - ) - assert ( - dr.results["com.android.storagemanager.automatic.SHOW_NOTIFICATION"][0][ - "package_name" - ] - == "com.android.storagemanager" + assert len(dr.results) == 9 + assert dr.results[0]["resolver_type"] == "full_mime_type" + storage_manager = next( + result + for result in dr.results + if result["key"] == "com.android.storagemanager.automatic.SHOW_NOTIFICATION" ) + assert storage_manager["package_name"] == "com.android.storagemanager" def test_parsing_misindented_action(self): dr = DumpsysReceiversArtifact() @@ -44,12 +41,8 @@ Receiver Resolver Table: dr.parse(data) - assert ( - dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][ - "package_name" - ] - == "com.psycatgames.nhiegame" - ) + assert dr.results[1]["key"] == "android.intent.action.MY_PACKAGE_REPLACED" + assert dr.results[1]["package_name"] == "com.psycatgames.nhiegame" def test_parsing_misindented_first_action(self): dr = DumpsysReceiversArtifact() @@ -62,12 +55,8 @@ Receiver Resolver Table: dr.parse(data) - assert ( - dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][ - "package_name" - ] - == "com.psycatgames.nhiegame" - ) + assert dr.results[0]["key"] == "android.intent.action.MY_PACKAGE_REPLACED" + assert dr.results[0]["package_name"] == "com.psycatgames.nhiegame" def test_ioc_check(self, indicator_file): dr = DumpsysReceiversArtifact() From 008844b440e4c7a31dd569703ab6d73de01c4e74 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 10/43] Parse accessibility service states per user --- .../artifacts/dumpsys_accessibility.py | 101 +++++++++++------- .../bugreport/dumpsys_accessibility.py | 4 +- .../test_artifact_dumpsys_accessibility.py | 22 +++- 3 files changed, 86 insertions(+), 41 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_accessibility.py b/src/mvt/android/artifacts/dumpsys_accessibility.py index 3d8fa6c..da46043 100644 --- a/src/mvt/android/artifacts/dumpsys_accessibility.py +++ b/src/mvt/android/artifacts/dumpsys_accessibility.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ import re +from typing import Any from .artifact import AndroidArtifact @@ -20,10 +21,10 @@ class DumpsysAccessibilityArtifact(AndroidArtifact): continue self.alertstore.medium( - f'Found accessibility service: "{result["service"]}"', + f'Found accessibility service: "{result["component"]}"', "", result, - ) + ) def parse(self, content: str) -> None: """ @@ -33,41 +34,69 @@ class DumpsysAccessibilityArtifact(AndroidArtifact): :param content: content of the accessibility section (string) """ - # "Old" syntax - in_services = False + self.results: list[dict[str, Any]] = [] + services: dict[tuple[int | None, str], dict] = {} + user_id: int | None = None + state: str | None = None + for line in content.splitlines(): - if line.strip().startswith("installed services:"): - in_services = True - continue + user_match = re.search(r"attributes:\{id=(\d+)", line) + if user_match: + user_id = int(user_match.group(1)) - if not in_services: - continue - - if line.strip() == "}": - # At end of installed services - break - - service = line.split(":")[1].strip() - - self.results.append( - { - "package_name": service.split("/")[0], - "service": service, - } + stripped = line.strip() + state_match = re.match( + r"(?i)(installed|enabled|binding|bound|crashed) services\s*:\s*\{(.*)", + stripped, ) - - # "New" syntax - AOSP >= 14 (?) - # Looks like: - # Enabled services:{{com.azure.authenticator/com.microsoft.brooklyn.module.accessibility.BrooklynAccessibilityService}, {com.agilebits.onepassword/com.agilebits.onepassword.filling.accessibility.FillingAccessibilityService}} - - for line in content.splitlines(): - if line.strip().startswith("Enabled services:"): - matches = re.finditer(r"{([^{]+?)}", line) - - for match in matches: - # Each match is in format: / - package_name, _, service = match.group(1).partition("/") - - self.results.append( - {"package_name": package_name, "service": service} + if state_match: + state = state_match.group(1).lower() + inline = state_match.group(2) + for component in re.findall( + r"\{?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?\}?", inline + ): + service = services.setdefault( + (user_id, component), self._new_service(component, user_id) ) + service[self._state_field(state)] = True + service["accessibility_tool"] = "(A11yTool)" in inline + continue + + if not state: + continue + if stripped == "}" or stripped.startswith("AccessibilityInputFilter"): + state = None + continue + component_match = re.search( + r"(?:\d+\s*:\s*)?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?", + stripped, + ) + if component_match: + component = component_match.group(1) + service = services.setdefault( + (user_id, component), self._new_service(component, user_id) + ) + service[self._state_field(state)] = True + service["accessibility_tool"] = "(A11yTool)" in stripped + + self.results.extend(services.values()) + + @staticmethod + def _state_field(state: str) -> str: + return {"binding": "binding", "bound": "bound"}.get(state, state) + + @staticmethod + def _new_service(component: str, user_id: int | None) -> dict: + package_name, service_name = component.split("/", 1) + return { + "user_id": user_id, + "component": component, + "package_name": package_name, + "service_name": service_name, + "installed": False, + "enabled": False, + "binding": False, + "bound": False, + "crashed": False, + "accessibility_tool": False, + } diff --git a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py index 72208c0..02d85a3 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py +++ b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py @@ -49,9 +49,7 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule): self.parse(content) for result in self.results: - self.log.info( - 'Found installed accessibility service "%s"', result.get("service") - ) + self.log.info('Found accessibility service "%s"', result.get("component")) self.log.info( "Identified a total of %d accessibility services", len(self.results) diff --git a/tests/android/test_artifact_dumpsys_accessibility.py b/tests/android/test_artifact_dumpsys_accessibility.py index 1f0b234..c727c57 100644 --- a/tests/android/test_artifact_dumpsys_accessibility.py +++ b/tests/android/test_artifact_dumpsys_accessibility.py @@ -23,7 +23,7 @@ class TestDumpsysAccessibilityArtifact: assert len(da.results) == 4 assert da.results[0]["package_name"] == "com.android.settings" assert ( - da.results[0]["service"] + da.results[0]["component"] == "com.android.settings/com.samsung.android.settings.development.gpuwatch.GPUWatchInterceptor" ) @@ -37,7 +37,9 @@ class TestDumpsysAccessibilityArtifact: da.parse(data) assert len(da.results) == 1 assert da.results[0]["package_name"] == "com.malware.accessibility" - assert da.results[0]["service"] == "com.malware.service.malwareservice" + assert da.results[0]["service_name"] == "com.malware.service.malwareservice" + assert da.results[0]["enabled"] is True + assert da.results[0]["installed"] is False def test_accessibility_service_alert(self): da = DumpsysAccessibilityArtifact() @@ -52,6 +54,22 @@ class TestDumpsysAccessibilityArtifact: assert da.alertstore.alerts[0].level == AlertLevel.MEDIUM assert da.alertstore.alerts[0].event == da.results[0] + def test_same_component_is_kept_for_each_user(self): + da = DumpsysAccessibilityArtifact() + da.parse( + """User state[attributes:{id=0 + installed services: { + 0 : com.example/.Service + } +User state[attributes:{id=10 + installed services: { + 0 : com.example/.Service + } +""" + ) + + assert [result["user_id"] for result in da.results] == [0, 10] + def test_ioc_check(self, indicator_file): da = DumpsysAccessibilityArtifact() file = get_artifact("android_data/dumpsys_accessibility.txt") From b904414bb08e27aac484cb862c225d7ac8e705a9 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 11/43] Parse ADB binary XML keys --- src/mvt/android/artifacts/dumpsys_adb.py | 42 +++++++++++++++++++++- tests/android/test_artifact_dumpsys_adb.py | 13 +++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/mvt/android/artifacts/dumpsys_adb.py b/src/mvt/android/artifacts/dumpsys_adb.py index e5438c9..cbd80e2 100644 --- a/src/mvt/android/artifacts/dumpsys_adb.py +++ b/src/mvt/android/artifacts/dumpsys_adb.py @@ -6,6 +6,7 @@ import base64 import binascii import hashlib +import re from .artifact import AndroidArtifact @@ -98,6 +99,34 @@ class DumpsysADBArtifact(AndroidArtifact): return keystore + def parse_binary_xml(self, data: bytes) -> list[dict]: + """Recover ADB key records from Android binary XML (ABX). + + Some dumpstate implementations embed ABX in a text stream and replace + binary token bytes. The public key remains intact, while unavailable + numeric metadata is represented as ``None`` rather than corrupt text. + """ + keystore = [] + seen = set() + for match in re.finditer( + rb"(? dict: if b" " in user_key: @@ -118,7 +147,7 @@ class DumpsysADBArtifact(AndroidArtifact): return { "user": user.decode("utf-8"), "fingerprint": key_fingerprint_colon, - "key": key_base64, + "key": key_base64.decode("ascii", errors="replace"), } def check_indicators(self) -> None: @@ -175,12 +204,23 @@ class DumpsysADBArtifact(AndroidArtifact): # Keystore is in XML format on some devices and we need to parse it if keystore_data and keystore_data.startswith(b" Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 12/43] Retain AppOps UID and event details --- src/mvt/android/artifacts/dumpsys_appops.py | 203 +++++++++++------- tests/android/test_artifact_dumpsys_appops.py | 29 ++- 2 files changed, 150 insertions(+), 82 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_appops.py b/src/mvt/android/artifacts/dumpsys_appops.py index 492eac7..611ab5f 100644 --- a/src/mvt/android/artifacts/dumpsys_appops.py +++ b/src/mvt/android/artifacts/dumpsys_appops.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ from datetime import datetime +import re from typing import Any from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult @@ -27,14 +28,14 @@ class DumpsysAppopsArtifact(AndroidArtifact): continue for entry in perm["entries"]: - if "timestamp" in entry: + if entry.get("timestamp"): records.append( { "timestamp": entry["timestamp"], "module": self.__class__.__name__, - "event": entry["access"], + "event": entry["event"], "data": f"{result['package_name']} access to " - f"{perm['name']}: {entry['access']}", + f"{perm['name']}: {entry['event']}", } ) @@ -51,7 +52,7 @@ class DumpsysAppopsArtifact(AndroidArtifact): continue # We use a placeholder entry to create a basic alert even without permission entries. - placeholder_entry = {"access": "Unknown", "timestamp": ""} + placeholder_entry = {"event": "unknown", "timestamp": ""} for perm in result["permissions"]: if ( @@ -60,12 +61,12 @@ class DumpsysAppopsArtifact(AndroidArtifact): ): for entry in sorted( perm["entries"] or [placeholder_entry], - key=lambda x: x["timestamp"], + key=lambda x: x.get("timestamp") or "", ): cleaned_result = result.copy() cleaned_result["permissions"] = [perm] self.alertstore.medium( - f"Package '{result['package_name']}' had risky permission '{perm['name']}' set to '{entry['access']}' at {entry['timestamp']}", + f"Package '{result['package_name']}' had risky permission '{perm['name']}' set to '{entry['event']}' at {entry['timestamp']}", entry["timestamp"], cleaned_result, ) @@ -73,111 +74,151 @@ class DumpsysAppopsArtifact(AndroidArtifact): elif result["package_name"] in RISKY_PACKAGES: for entry in sorted( perm["entries"] or [placeholder_entry], - key=lambda x: x["timestamp"], + key=lambda x: x.get("timestamp") or "", ): cleaned_result = result.copy() cleaned_result["permissions"] = [perm] self.alertstore.medium( - f"Risky package '{result['package_name']}' had '{perm['name']}' permission set to '{entry['access']}' at {entry['timestamp']}", + f"Risky package '{result['package_name']}' had '{perm['name']}' permission set to '{entry['event']}' at {entry['timestamp']}", entry["timestamp"], cleaned_result, ) def parse(self, output: str) -> None: - # self.results: List[Dict[str, Any]] = [] - perm: dict[str, Any] = {} - package: dict[str, Any] = {} - entry: dict[str, Any] = {} - uid = None + self.results: list[dict[str, Any]] = [] + permission: dict[str, Any] | None = None + package: dict[str, Any] | None = None + uid: str | None = None + uid_details: dict[str, Any] = {} + attribution: str | None = None in_packages = False + def finish_permission() -> None: + nonlocal permission + if package is not None and permission is not None: + package["permissions"].append(permission) + permission = None + + def finish_package() -> None: + nonlocal package + finish_permission() + if package is not None: + self.results.append(package) + package = None + for line in output.splitlines(): - if line.startswith(" Uid 0:"): + uid_match = re.match(r"^ Uid ([^:]+):$", line) + if uid_match: in_packages = True + finish_package() + uid = uid_match.group(1) + uid_details = { + "uid_state": None, + "capability": None, + "app_widget_visible": None, + "default_modes": {}, + } + continue if not in_packages: continue - if line.startswith(" Uid "): - uid = line[6:-1] - if entry: - perm["entries"].append(entry) - entry = {} - if package: - if perm: - package["permissions"].append(perm) + uid_property = re.match( + r"^ (state|capability|appWidgetVisible)=(.*)$", line + ) + if uid_property: + key = { + "state": "uid_state", + "appWidgetVisible": "app_widget_visible", + }.get(uid_property.group(1), uid_property.group(1)) + value: Any = uid_property.group(2) + if value in ("true", "false"): + value = value == "true" + uid_details[key] = value + continue - perm = {} - self.results.append(package) - package = {} + default_mode = re.match(r"^ ([A-Z0-9_]+): mode=([^\s]+)", line) + if default_mode and package is None: + uid_details["default_modes"][default_mode.group(1)] = ( + default_mode.group(2) + ) continue if line.startswith(" Package "): - if entry: - perm["entries"].append(entry) - entry = {} - - if package: - if perm: - package["permissions"].append(perm) - - perm = {} - self.results.append(package) - + finish_package() package = { "package_name": line[12:-1], "permissions": [], "uid": uid, + **uid_details, } continue - if package and line.startswith(" ") and line[6] != " ": - if entry: - perm["entries"].append(entry) - entry = {} - if perm: - package["permissions"].append(perm) - perm = {} - - perm["name"] = line.split()[0] - perm["entries"] = [] - if len(line.split()) > 1: - perm["access"] = line.split()[1][1:-2] - + operation_match = re.match( + r"^ ([A-Z0-9_]+)(?: \(([^)]+)\))?:\s*$", line + ) + if package is not None and operation_match: + finish_permission() + permission = { + "name": operation_match.group(1), + "mode": operation_match.group(2), + "entries": [], + } + attribution = None continue - if line.startswith(" "): - # Permission entry like: - # Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms) - access_type = line.split(":")[0].strip() - if access_type not in ["Access", "Reject"]: - # Skipping invalid access type. Some entries are not in the format we expect - continue + attribution_match = re.match(r"^\s{8,}([^=]+)=\[$", line) + if attribution_match: + attribution = attribution_match.group(1).strip() + continue + if line.strip() == "]": + attribution = None + continue - if entry: - perm["entries"].append(entry) - entry = {} - - entry["access"] = access_type - entry["type"] = line[line.find("[") + 1 : line.find("]")] - - try: - entry["timestamp"] = convert_datetime_to_iso( - datetime.strptime( - line[line.find("]") + 1 : line.find("(")].strip(), - "%Y-%m-%d %H:%M:%S.%f", - ) + if permission is None: + continue + event_match = re.match( + r"^\s*(Access|Reject):\s*\[([^]]+)\]\s*" + r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)\s*" + r"(\([^)]*\))?(?:\s+duration=([^\s]+))?", + line, + ) + running_match = re.match( + r"^\s*Running start at:\s*(\S+(?: \S+)?)", + line, + ) + if event_match: + entry = { + "event": event_match.group(1).lower(), + "access": event_match.group(1), + "uid_state": event_match.group(2), + "timestamp": convert_datetime_to_iso( + datetime.strptime(event_match.group(3), "%Y-%m-%d %H:%M:%S.%f") + ), + "relative_time": event_match.group(4), + "duration": event_match.group(5), + "attribution": attribution, + } + permission["entries"].append(entry) + elif running_match: + raw_start = running_match.group(1) + timestamp = None + if re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", raw_start): + timestamp = convert_datetime_to_iso( + datetime.strptime(raw_start, "%Y-%m-%d %H:%M:%S.%f") ) - except ValueError: - # Invalid date format - pass + permission["entries"].append( + { + "event": "running", + "access": "Running", + "uid_state": None, + "timestamp": timestamp, + "relative_time": raw_start + if raw_start.startswith("+") + else None, + "duration": None, + "attribution": attribution, + } + ) - if line.strip() == "": - break - - if entry: - perm["entries"].append(entry) - if perm: - package["permissions"].append(perm) - if package: - self.results.append(package) + finish_package() diff --git a/tests/android/test_artifact_dumpsys_appops.py b/tests/android/test_artifact_dumpsys_appops.py index 862b872..a86084f 100644 --- a/tests/android/test_artifact_dumpsys_appops.py +++ b/tests/android/test_artifact_dumpsys_appops.py @@ -25,10 +25,37 @@ class TestDumpsysAppopsArtifact: assert da.results[0]["uid"] == "0" assert len(da.results[0]["permissions"]) == 1 assert da.results[0]["permissions"][0]["name"] == "MANAGE_IPSEC_TUNNELS" - assert da.results[0]["permissions"][0]["access"] == "allow" + assert da.results[0]["permissions"][0]["mode"] == "allow" assert da.results[6]["package_name"] == "com.sec.factory.camera" assert len(da.results[6]["permissions"][1]["entries"]) == 1 assert len(da.results[11]["permissions"]) == 4 + wake_lock = next( + permission + for permission in da.results[11]["permissions"] + if permission["name"] == "WAKE_LOCK" + ) + assert wake_lock["entries"][0]["duration"] == "+126ms" + + def test_running_and_attribution_are_retained(self): + da = DumpsysAppopsArtifact() + da.parse( + """ Uid 0: + state=cch + Package com.example: + CAMERA (allow): + camera=[ + Access: [fg-s] 2025-01-01 00:00:00.000 (-1s) duration=+2ms + ] + RECORD_AUDIO (allow): + Running start at: +3s +""" + ) + + camera = da.results[0]["permissions"][0]["entries"][0] + running = da.results[0]["permissions"][1]["entries"][0] + assert camera["attribution"] == "camera" + assert running["event"] == "running" + assert running["relative_time"] == "+3s" def test_ioc_check(self, indicator_file): da = DumpsysAppopsArtifact() From 87a27f6e3746d73b5208a4e93421626642267fe3 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 13/43] Normalize battery daily update records --- .../artifacts/dumpsys_battery_daily.py | 58 ++++++++++++------- .../test_artifact_dumpsys_battery_daily.py | 23 ++++++-- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_battery_daily.py b/src/mvt/android/artifacts/dumpsys_battery_daily.py index c0f2c5e..2ccb6b7 100644 --- a/src/mvt/android/artifacts/dumpsys_battery_daily.py +++ b/src/mvt/android/artifacts/dumpsys_battery_daily.py @@ -18,18 +18,18 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: action = record.get("action", "update") package_name = record["package_name"] - vers = record["vers"] + vers = record["version_code"] - if vers == "0": + if vers == 0: data = f"Recorded uninstall of package {package_name} (vers 0)" elif action == "downgrade": - prev_vers = record.get("previous_vers", "unknown") + prev_vers = record.get("previous_version_code", "unknown") data = f"Recorded downgrade of package {package_name} from vers {prev_vers} to vers {vers}" else: data = f"Recorded update of package {package_name} with vers {vers}" return { - "timestamp": record["from"], + "timestamp": record["period_start"], "module": self.__class__.__name__, "event": "battery_daily", "data": data, @@ -48,6 +48,7 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): continue def parse(self, output: str) -> None: + self.results = [] daily = None daily_updates: list[dict[str, Any]] = [] records: list[dict[str, Any]] = [] @@ -59,7 +60,10 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): timeframe = line[13:].strip() date_from, date_to = timeframe.strip(":").split(" to ", 1) - daily = {"from": date_from[0:10], "to": date_to[0:10]} + daily = { + "period_start": self._format_daily_timestamp(date_from), + "period_end": self._format_daily_timestamp(date_to), + } continue if not daily: @@ -70,21 +74,30 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): line = line.strip().replace("Update ", "") package_name, vers = line.split(" ", 1) - vers_nr = vers.split("=", 1)[1] + vers_raw = vers.split("=", 1)[1] + try: + version_code: int | str = int(vers_raw) + except ValueError: + version_code = vers_raw already_seen = False for update in daily_updates: - if package_name == update["package_name"] and vers_nr == update["vers"]: + if ( + package_name == update["package_name"] + and version_code == update["version_code"] + ): + update["occurrences"] += 1 already_seen = True break if not already_seen: update_record: dict[str, Any] = { "action": "update", - "from": daily["from"], - "to": daily["to"], + "period_start": daily["period_start"], + "period_end": daily["period_end"], "package_name": package_name, - "vers": vers_nr, + "version_code": version_code, + "occurrences": 1, } daily_updates.append(update_record) @@ -95,26 +108,31 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): self._detect_uninstalls_and_downgrades(records) self.results.extend(records) - def _detect_uninstalls_and_downgrades( - self, records: list[dict[str, Any]] - ) -> None: + @staticmethod + def _format_daily_timestamp(value: str) -> str: + if len(value) >= 19 and value[10] == "-": + return f"{value[:10]} {value[11:].replace('-', ':')}" + return value + + def _detect_uninstalls_and_downgrades(self, records: list[dict[str, Any]]) -> None: package_versions: dict[str, int] = {} for record in sorted( records, key=lambda record: ( - record["from"], - record["to"], + record["period_start"], + record["period_end"], record["package_name"], ), ): package_name = record["package_name"] - vers_nr = record["vers"] + vers_nr = record["version_code"] - if vers_nr == "0": + if vers_nr == 0: + record["action"] = "uninstall" self.alertstore.medium( f"Detected uninstall of package {package_name} (vers 0)", - record["from"], + record["period_start"], record, ) package_versions.pop(package_name, None) @@ -128,11 +146,11 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): previous_vers = package_versions.get(package_name) if previous_vers is not None and current_vers < previous_vers: record["action"] = "downgrade" - record["previous_vers"] = str(previous_vers) + record["previous_version_code"] = previous_vers self.alertstore.medium( f"Detected downgrade of package {package_name} " f"from vers {previous_vers} to vers {current_vers}", - record["from"], + record["period_start"], record, ) diff --git a/tests/android/test_artifact_dumpsys_battery_daily.py b/tests/android/test_artifact_dumpsys_battery_daily.py index dbb9c36..5f7f997 100644 --- a/tests/android/test_artifact_dumpsys_battery_daily.py +++ b/tests/android/test_artifact_dumpsys_battery_daily.py @@ -57,18 +57,19 @@ class TestDumpsysBatteryDailyArtifact: assert uninstall_alert.message == ( "Detected uninstall of package com.example.removed (vers 0)" ) - assert uninstall_alert.event_time == "2022-08-16" + assert uninstall_alert.event_time == "2022-08-16 15:56:39" assert uninstall_alert.event["package_name"] == "com.example.removed" - assert uninstall_alert.event["vers"] == "0" + assert uninstall_alert.event["version_code"] == 0 + assert uninstall_alert.event["action"] == "uninstall" assert downgrade_alert.level == AlertLevel.MEDIUM assert downgrade_alert.message == ( "Detected downgrade of package com.example.app from vers 10 to vers 9" ) - assert downgrade_alert.event_time == "2022-08-17" + assert downgrade_alert.event_time == "2022-08-17 15:56:39" assert downgrade_alert.event["package_name"] == "com.example.app" assert downgrade_alert.event["action"] == "downgrade" - assert downgrade_alert.event["previous_vers"] == "10" + assert downgrade_alert.event["previous_version_code"] == 10 def test_newest_first_update_is_not_reported_as_downgrade(self): dba = DumpsysBatteryDailyArtifact() @@ -107,7 +108,19 @@ class TestDumpsysBatteryDailyArtifact: assert downgrade_alert.event_time == "2026-01-10" assert downgrade_alert.event["package_name"] == "com.example.app" assert downgrade_alert.event["action"] == "downgrade" - assert downgrade_alert.event["previous_vers"] == "102" + assert downgrade_alert.event["previous_version_code"] == 102 + + def test_duplicate_updates_retain_occurrence_count(self): + dba = DumpsysBatteryDailyArtifact() + dba.parse( + """ Daily from 2026-01-10-01-02-03 to 2026-01-11-04-05-06: + Update com.example.app vers=12 + Update com.example.app vers=12 +""" + ) + + assert dba.results[0]["occurrences"] == 2 + assert dba.results[0]["period_start"] == "2026-01-10 01:02:03" def test_reinstall_after_uninstall_is_not_reported_as_downgrade(self): dba = DumpsysBatteryDailyArtifact() From 288b3136498c0746ae0dffa979c71fe6c07966dc Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 14/43] Parse complete battery history events --- .../artifacts/dumpsys_battery_history.py | 225 ++++++++++++------ .../test_artifact_dumpsys_battery_history.py | 21 ++ 2 files changed, 174 insertions(+), 72 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_battery_history.py b/src/mvt/android/artifacts/dumpsys_battery_history.py index 77d9d3c..6cb36f1 100644 --- a/src/mvt/android/artifacts/dumpsys_battery_history.py +++ b/src/mvt/android/artifacts/dumpsys_battery_history.py @@ -1,96 +1,177 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import datetime +import re + +from mvt.common.utils import convert_datetime_to_iso + from .artifact import AndroidArtifact class DumpsysBatteryHistoryArtifact(AndroidArtifact): - """ - Parser for dumpsys dattery history events. - """ + """Parser for package-related events in dumpsys batterystats history.""" def check_indicators(self) -> None: if not self.indicators: return - for result in self.results: - ioc_match = self.indicators.check_app_id(result["package_name"]) + package_name = result.get("package_name") + if not package_name: + continue + ioc_match = self.indicators.check_app_id(package_name) if ioc_match: self.alertstore.critical( ioc_match.message, "", result, matched_indicator=ioc_match.ioc ) - continue + + @staticmethod + def _parse_wall_time(value: str) -> datetime.datetime | None: + if re.fullmatch(r"\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", value): + value = f"1900-{value}" + for date_format in ( + "%Y-%m-%d-%H-%M-%S-%f", + "%Y-%m-%d-%H-%M-%S", + "%Y-%m-%d %H:%M:%S.%f", + ): + try: + return datetime.datetime.strptime(value, date_format) + except ValueError: + pass + return None + + @staticmethod + def _elapsed_seconds(value: str) -> float | None: + if not value.startswith("+"): + return None + units = {"d": 86400, "h": 3600, "m": 60, "s": 1, "ms": 0.001} + total = 0.0 + for number, unit in re.findall(r"(\d+)(ms|d|h|m|s)", value): + total += int(number) * units[unit] + return total + + @staticmethod + def _package_from_name(name: str) -> str | None: + clean = name.removeprefix("*walarm*:").removeprefix("*alarm*:") + slash_parts = clean.split("/") + if len(slash_parts) > 1: + first = slash_parts[0].lstrip("@") + if first.startswith(("com.", "org.", "net.")): + return first + for part in reversed(slash_parts[1:]): + candidate = part.lstrip("@").split(":", 1)[0] + if candidate.startswith(("com.", "org.", "net.")): + return candidate + parts = clean.split(".") + package_parts = [] + for part in parts: + if part and part[0].islower(): + package_parts.append(part) + else: + break + return ".".join(package_parts) if len(package_parts) >= 2 else None + + @staticmethod + def _normalize_service(service: str) -> str: + # WorkManager decorates jobs with one or more scheduler prefixes. + if "@" in service: + candidates = [part for part in service.split("@") if "/" in part] + if candidates: + return candidates[-1] + return service def parse(self, data: str) -> None: + self.results: list[dict[str, str | None]] = [] + anchor_time: datetime.datetime | None = None + anchor_elapsed = 0.0 + has_history_heading = any( + line.startswith("Battery History") for line in data.splitlines() + ) + in_history = not has_history_heading + for line in data.splitlines(): - if line.startswith("Battery History "): + stripped = line.strip() + if line.startswith("Battery History"): + if in_history: + break + in_history = True continue - - if line.strip() == "": + if not in_history: + continue + if has_history_heading and not stripped: break - - time_parts = line.strip().split() - time_elapsed = time_parts[0] - if ( - len(time_parts) > 1 - and len(time_parts[0]) == 5 - and time_parts[0][2] == "-" - and ":" in time_parts[1] - ): - time_elapsed = " ".join(time_parts[:2]) - - event = "" - if line.find("+job") > 0: - event = "start_job" - payload = line.split("+job=", 1)[1] - uid, separator, service = payload.partition(":") - if not separator: - continue - service = service.strip().strip('"') - package_name = service.split("/")[0] - elif line.find("-job") > 0: - event = "end_job" - payload = line.split("-job=", 1)[1] - uid, separator, service = payload.partition(":") - if not separator: - continue - service = service.strip().strip('"') - package_name = service.split("/")[0] - elif line.find("+running +wake_lock=") > 0: - payload = line.split("+running +wake_lock=", 1)[1] - uid, separator, _ = payload.partition(":") - if not separator: - continue - event = "wake" - service = ( - line[line.find("*walarm*:") + 9 :].split(" ")[0].strip('"').strip() - ) - if service == "" or "/" not in service: - continue - - package_name = service.split("/")[0] - elif (line.find("+top=") > 0) or (line.find("-top") > 0): - if line.find("+top=") > 0: - event = "start_top" - top_pos = line.find("+top=") - else: - event = "end_top" - top_pos = line.find("-top=") - colon_pos = top_pos + line[top_pos:].find(":") - uid = line[top_pos + 5 : colon_pos] - service = "" - package_name = line[colon_pos + 1 :].strip('"') - else: + reset_match = re.search(r"(?:RESET:)?TIME:\s*(\S+)", stripped) + if reset_match: + parsed_time = self._parse_wall_time(reset_match.group(1)) + if parsed_time is not None: + elapsed_token = stripped.split()[0] + anchor_elapsed = self._elapsed_seconds(elapsed_token) or 0.0 + anchor_time = parsed_time continue - self.results.append( - { - "time_elapsed": time_elapsed, - "event": event, - "uid": uid, - "package_name": package_name, - "service": service, - } - ) + fields = stripped.split() + if not fields: + continue + if len(fields) > 1 and re.fullmatch(r"\d{2}-\d{2}", fields[0]): + time_elapsed = " ".join(fields[:2]) + line_time = self._parse_wall_time(time_elapsed) + elapsed = None + else: + time_elapsed = fields[0] + elapsed = self._elapsed_seconds(time_elapsed) + line_time = None + + timestamp = line_time + if timestamp is None and anchor_time is not None and elapsed is not None: + timestamp = anchor_time + datetime.timedelta( + seconds=elapsed - anchor_elapsed + ) + + def add( + event: str, uid: str, service: str, package_name: str | None + ) -> None: + self.results.append( + { + "time_elapsed": time_elapsed, + "timestamp": convert_datetime_to_iso(timestamp) + if timestamp + else None, + "event": event, + "uid": uid, + "package_name": package_name, + "service": service, + } + ) + + for sign, uid, raw_service in re.findall( + r"([+-])job=([^:\s]+):\"([^\"]+)\"", line + ): + service = self._normalize_service(raw_service) + add( + "start_job" if sign == "+" else "end_job", + uid, + service, + self._package_from_name(service) or service.split("/", 1)[0], + ) + + for sign, uid, package_name in re.findall( + r"([+-])top=([^:\s]+):\"([^\"]+)\"", line + ): + add( + "start_top" if sign == "+" else "end_top", + uid, + "", + package_name, + ) + + wake_match = re.search(r"\+wake_lock=([^:\s]+):\"([^\"]+)\"", line) + if wake_match: + wake_name = wake_match.group(2) + add( + "wake", + wake_match.group(1), + wake_name, + self._package_from_name(wake_name), + ) diff --git a/tests/android/test_artifact_dumpsys_battery_history.py b/tests/android/test_artifact_dumpsys_battery_history.py index dd0b120..091ce55 100644 --- a/tests/android/test_artifact_dumpsys_battery_history.py +++ b/tests/android/test_artifact_dumpsys_battery_history.py @@ -54,6 +54,7 @@ class TestDumpsysBatteryHistoryArtifact: assert len(dba.results) == 2 assert dba.results[0] == { "time_elapsed": "07-15 20:27:39.431", + "timestamp": "1900-07-15 20:27:39.431000", "event": "start_job", "uid": "u0a123", "package_name": "com.example", @@ -61,3 +62,23 @@ class TestDumpsysBatteryHistoryArtifact: } assert dba.results[1]["event"] == "end_job" assert dba.results[1]["uid"] == "u0a123" + + def test_wake_lock_without_component_is_retained(self): + dba = DumpsysBatteryHistoryArtifact() + dba.parse( + "Battery History:\n" + " 0 (2) 100 RESET:TIME: 2025-09-05-01-04-52-139\n" + ' +1s (2) 100 +running +wake_lock=1000:"*alarm*:TIME_TICK"\n' + ' +2s (2) 100 +running +wake_lock=u0a1:"*walarm*:com.whatsapp.MessageHandler.LOGOUT_ACTION"\n' + "\n" + ) + + assert [record["event"] for record in dba.results] == ["wake", "wake"] + assert dba.results[0]["package_name"] is None + assert dba.results[1]["package_name"] == "com.whatsapp" + + def test_decorated_sync_job_uses_component_package(self): + dba = DumpsysBatteryHistoryArtifact() + dba.parse('+1s (2) 100 +job=u0a1:"@SyncManager@gmail-ls/com.google:android"\n') + + assert dba.results[0]["package_name"] == "com.google" From e9bf197ec61a75be966df689d4405e758de4b22a Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 15/43] Parse all database operation details --- src/mvt/android/artifacts/dumpsys_dbinfo.py | 36 +++++++++++++------ tests/android/test_artifact_dumpsys_dbinfo.py | 25 ++++++++++++- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_dbinfo.py b/src/mvt/android/artifacts/dumpsys_dbinfo.py index 050582d..0972b43 100644 --- a/src/mvt/android/artifacts/dumpsys_dbinfo.py +++ b/src/mvt/android/artifacts/dumpsys_dbinfo.py @@ -29,16 +29,27 @@ class DumpsysDBInfoArtifact(AndroidArtifact): def parse(self, output: str) -> None: rxp = re.compile( - r".*\[((?:[0-9]{4}-)?[0-9]{2}-[0-9]{2} " - r"[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\]\s*" - r"(?:\[Pid:\((\d+)\)\])?([\w-]+).*?sql=\"(.+?)\"" - ) # pylint: disable=line-too-long + r"^\s*\d+:\s*\[((?:\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]\s*" + r"(?:\[Pid:\((\d+)\)\])?\s*([\w-]+) took (\d+)ms - ([^,]+)," + r"\s*sql=\"(.*)\"(?:, path=(.*))?$" + ) - pool = None + pool: str | None = None + connection_number: int | None = None + is_primary: bool | None = None in_operations = False for line in output.splitlines(): if line.startswith("Connection pool for "): pool = line.replace("Connection pool for ", "").rstrip(":") + in_operations = False + + connection_match = re.match(r"\s+Connection #(\d+):", line) + if connection_match: + connection_number = int(connection_match.group(1)) + is_primary = None + + if line.strip().startswith("isPrimaryConnection:"): + is_primary = line.strip().split(":", 1)[1].strip() == "true" if not pool: continue @@ -52,7 +63,6 @@ class DumpsysDBInfoArtifact(AndroidArtifact): if not line.startswith(" "): in_operations = False - pool = None continue match = rxp.match(line) @@ -60,11 +70,15 @@ class DumpsysDBInfoArtifact(AndroidArtifact): continue result = { - "isodate": match.group(1), + "timestamp": match.group(1), + "pid": int(match.group(2)) if match.group(2) else None, "action": match.group(3), - "sql": match.group(4), - "path": pool, + "duration_ms": int(match.group(4)), + "status": match.group(5), + "sql": match.group(6), + "path": match.group(7) or pool, + "pool_path": pool, + "connection_number": connection_number, + "is_primary": is_primary, } - if match.group(2): - result["pid"] = match.group(2) self.results.append(result) diff --git a/tests/android/test_artifact_dumpsys_dbinfo.py b/tests/android/test_artifact_dumpsys_dbinfo.py index 691d43c..0dcc0bc 100644 --- a/tests/android/test_artifact_dumpsys_dbinfo.py +++ b/tests/android/test_artifact_dumpsys_dbinfo.py @@ -53,9 +53,32 @@ Connection pool for /data/user/0/com.example/databases/current.db: assert dbi.results == [ { - "isodate": "07-15 20:27:39.431", + "timestamp": "07-15 20:27:39.431", + "pid": None, "action": "executeForCursorWindow", + "duration_ms": 1, + "status": "succeeded", "sql": "SELECT 1", "path": "/data/user/0/com.example/databases/current.db", + "pool_path": "/data/user/0/com.example/databases/current.db", + "connection_number": None, + "is_primary": None, } ] + + def test_parses_operations_from_multiple_connections(self): + dbi = DumpsysDBInfoArtifact() + dbi.parse( + """Connection pool for /data/example.db: + Connection #0: + isPrimaryConnection: true + Most recently executed operations: + 0: [2025-01-01 00:00:00.000] execute took 1ms - succeeded, sql="SELECT 1", path=/data/example.db + Connection #1: + isPrimaryConnection: false + Most recently executed operations: + 0: [2025-01-01 00:00:01.000] execute took 2ms - succeeded, sql="SELECT 2", path=/data/example.db +""" + ) + + assert [record["connection_number"] for record in dbi.results] == [0, 1] From a3c8b56102e38a004f1ee605702091cfea562f47 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 16/43] Parse typed multi-user package details --- src/mvt/android/artifacts/dumpsys_packages.py | 388 +++++++++--------- .../modules/bugreport/dumpsys_packages.py | 5 +- .../android/test_artifact_dumpsys_packages.py | 61 ++- 3 files changed, 234 insertions(+), 220 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_packages.py b/src/mvt/android/artifacts/dumpsys_packages.py index bef7744..1194a75 100644 --- a/src/mvt/android/artifacts/dumpsys_packages.py +++ b/src/mvt/android/artifacts/dumpsys_packages.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ import re -from typing import Any, Dict, List, Optional +from typing import Any from mvt.android.utils import ROOT_PACKAGES from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult @@ -12,222 +12,244 @@ from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult from .artifact import AndroidArtifact +def _value(raw: str) -> Any: + if raw == "null": + return None + if raw in ("true", "false"): + return raw == "true" + try: + return int(raw) + except ValueError: + return raw + + class DumpsysPackagesArtifact(AndroidArtifact): def check_indicators(self) -> None: alerted_root_packages = set() for result in self.results: - if result["package_name"] in ROOT_PACKAGES: - if result["package_name"] in alerted_root_packages: - continue - alerted_root_packages.add(result["package_name"]) + package_name = result["package_name"] + if ( + package_name in ROOT_PACKAGES + and package_name not in alerted_root_packages + ): + alerted_root_packages.add(package_name) self.alertstore.medium( - f'Found an installed package related to rooting/jailbreaking: "{result["package_name"]}"', + f'Found an installed package related to rooting/jailbreaking: "{package_name}"', "", result, ) - continue - if not self.indicators: continue - - ioc_match = self.indicators.check_app_id(result.get("package_name", "")) + ioc_match = self.indicators.check_app_id(package_name) if ioc_match: self.alertstore.critical( ioc_match.message, "", result, matched_indicator=ioc_match.ioc ) def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: - records = [] timestamps = [ - {"event": "package_install", "timestamp": record["timestamp"]}, + ("package_install", record.get("timestamp")), + ("package_last_update", record.get("last_update_time")), + ] + timestamps.extend( + ("package_first_install", user.get("first_install_time")) + for user in record.get("users", []) + ) + return [ { - "event": "package_first_install", - "timestamp": record["first_install_time"], - }, - {"event": "package_last_update", "timestamp": record["last_update_time"]}, + "timestamp": timestamp, + "module": self.__class__.__name__, + "event": event, + "data": f"Install or update of package {record['package_name']}", + } + for event, timestamp in timestamps + if timestamp ] - for timestamp in timestamps: - records.append( - { - "timestamp": timestamp["timestamp"], - "module": self.__class__.__name__, - "event": timestamp["event"], - "data": f"Install or update of package {record['package_name']}", - } - ) - - return records - @staticmethod - def parse_dumpsys_package_for_details(output: str) -> Dict[str, Any]: - """ - Parse one entry of a dumpsys package information - """ - details: Dict[str, Any] = { - "uid": "", - "version_name": "", - "version_code": "", - "timestamp": "", - "first_install_time": "", - "last_update_time": "", - "installer": "", - "system": False, - "permissions": list(), - "requested_permissions": list(), + def _permission(line: str, permission_type: str) -> dict: + name, _, details = line.strip().partition(":") + granted_match = re.search(r"granted=(true|false)", details) + flags_match = re.search(r"flags=\[\s*([^]]*)\]", details) + return { + "name": name, + "type": permission_type, + "granted": granted_match.group(1) == "true" if granted_match else None, + "flags": [ + flag.strip() + for flag in (flags_match.group(1).split("|") if flags_match else []) + if flag.strip() + ], } - in_install_permissions = False - in_runtime_permissions = False - in_declared_permissions = False - in_requested_permissions = True - current_user: Optional[int] = None - first_install_times: Dict[Optional[int], str] = {} - runtime_permissions: Dict[Optional[int], List[Dict[str, Any]]] = {} + + @classmethod + def parse_dumpsys_package_for_details(cls, output: str) -> dict[str, Any]: + details: dict[str, Any] = { + "app_id": None, + "version_name": None, + "version_code": None, + "min_sdk": None, + "target_sdk": None, + "timestamp": None, + "last_update_time": None, + "installer": None, + "system": False, + "permissions": [], + "requested_permissions": [], + "users": [], + } + permission_section: str | None = None + current_user: dict[str, Any] | None = None + legacy_first_install: str | None = None + for line in output.splitlines(): - user_match = re.match(r"User (\d+):", line.strip()) + stripped = line.strip() + user_match = re.match(r"User (\d+):\s*(.*)", stripped) if user_match: - current_user = int(user_match.group(1)) + current_user = {"user_id": int(user_match.group(1)), "permissions": []} + for key, raw in re.findall(r"(\w+)=([^\s]+)", user_match.group(2)): + clean_key = { + "notLaunched": "not_launched", + "installReason": "install_reason", + "uninstallReason": "uninstall_reason", + "dataDir": "data_dir", + }.get(key, re.sub(r"(? List[Dict[str, Any]]: - """ - Parse the dumpsys package service data - """ - pkg_rxp = re.compile(r" Package \[(.+?)\].*") - - results = [] - package_name = None - package = {} - lines: list[str] = [] - for line in output.splitlines(): - if line.startswith(" Package ["): - if len(lines) > 0: - details = self.parse_dumpsys_package_for_details("\n".join(lines)) - package.update(details) - results.append(package) - lines = [] - package = {} - - matches = pkg_rxp.findall(line) - if not matches: + if current_user is not None: + user_property = re.match( + r"(installReason|uninstallReason|dataDir|firstInstallTime)=(.*)", + stripped, + ) + if user_property: + key = { + "installReason": "install_reason", + "uninstallReason": "uninstall_reason", + "dataDir": "data_dir", + "firstInstallTime": "first_install_time", + }[user_property.group(1)] + current_user[key] = _value(user_property.group(2)) continue - package_name = matches[0] - package["package_name"] = package_name + if permission_section == "requested" and line.startswith(" "): + details["requested_permissions"].append(stripped) + continue + if permission_section in ("declared", "install") and line.startswith( + " " + ): + details["permissions"].append( + cls._permission(stripped, permission_section) + ) + continue + if ( + permission_section == "runtime" + and line.startswith(" ") + and current_user is not None + ): + current_user["permissions"].append(cls._permission(stripped, "runtime")) continue - if not package_name: + simple_match = re.match( + r"(appId|userId|versionName|timeStamp|lastUpdateTime|installerPackageName)=(.*)", + stripped, + ) + if simple_match: + key = { + "appId": "app_id", + "userId": "app_id", + "versionName": "version_name", + "timeStamp": "timestamp", + "lastUpdateTime": "last_update_time", + "installerPackageName": "installer", + }[simple_match.group(1)] + raw_value = simple_match.group(2) + details[key] = ( + _value(raw_value) + if key == "app_id" or raw_value == "null" + else raw_value + ) continue + if stripped.startswith("pkgFlags="): + details["system"] = "SYSTEM" in stripped.split("=", 1)[1].split() + continue + version_match = re.match( + r"versionCode=([^\s]+)(?:\s+minSdk=([^\s]+))?(?:\s+targetSdk=([^\s]+))?", + stripped, + ) + if version_match: + details["version_code"] = _value(version_match.group(1)) + details["min_sdk"] = ( + _value(version_match.group(2)) if version_match.group(2) else None + ) + details["target_sdk"] = ( + _value(version_match.group(3)) if version_match.group(3) else None + ) + elif stripped.startswith("firstInstallTime="): + legacy_first_install = stripped.split("=", 1)[1] - lines.append(line) + if legacy_first_install: + user_zero = next( + (user for user in details["users"] if user["user_id"] == 0), None + ) + if user_zero is None: + user_zero = {"user_id": 0, "permissions": []} + details["users"].append(user_zero) + user_zero.setdefault("first_install_time", legacy_first_install) + return details - if len(lines) > 0: - details = self.parse_dumpsys_package_for_details("\n".join(lines)) - package.update(details) - results.append(package) + def parse(self, content: str) -> None: + self.results: list[dict[str, Any]] = [] + category: str | None = None + package: dict[str, Any] | None = None + block: list[str] = [] - return results + def finish() -> None: + nonlocal package, block + if package is not None: + package.update(self.parse_dumpsys_package_for_details("\n".join(block))) + self.results.append(package) + package = None + block = [] - def parse(self, content: str): - """ - Parse the Dumpsys Package section for activities - Adds results to self.results - - :param content: content of the package section (string) - """ - self.results = [] - package = [] - - in_package_list = False for line in content.splitlines(): - if line.startswith("Packages:"): - in_package_list = True + if line == "Packages:": + finish() + category = "active" continue - - if not in_package_list: + if line == "Hidden system packages:": + finish() + category = "hidden_system" continue - - if line.strip() == "": - break - - package.append(line) - - self.results = self.parse_dumpsys_packages("\n".join(package)) + package_match = re.match(r"^ Package \[([^]]+)\]", line) + if package_match and category: + finish() + package = { + "package_name": package_match.group(1), + "package_type": category, + } + continue + if ( + category + and line + and not line.startswith(" ") + and not line.endswith(" overlay paths:") + ): + finish() + category = None + continue + if package is not None: + block.append(line) + finish() diff --git a/src/mvt/android/modules/bugreport/dumpsys_packages.py b/src/mvt/android/modules/bugreport/dumpsys_packages.py index aebec4c..ff944e3 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_packages.py +++ b/src/mvt/android/modules/bugreport/dumpsys_packages.py @@ -50,7 +50,10 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule): for result in self.results: dangerous_permissions_count = 0 - for perm in result["permissions"]: + permissions = list(result["permissions"]) + for user in result.get("users", []): + permissions.extend(user.get("permissions", [])) + for perm in permissions: if perm["name"] in DANGEROUS_PERMISSIONS: dangerous_permissions_count += 1 diff --git a/tests/android/test_artifact_dumpsys_packages.py b/tests/android/test_artifact_dumpsys_packages.py index bd5fb75..f22cc67 100644 --- a/tests/android/test_artifact_dumpsys_packages.py +++ b/tests/android/test_artifact_dumpsys_packages.py @@ -25,7 +25,11 @@ class TestDumpsysPackagesArtifact: == "com.samsung.android.provider.filterprovider" ) assert dpa.results[0]["version_name"] == "5.0.07" - assert dpa.results[0]["first_install_time"] == "2008-12-31 16:00:00" + assert dpa.results[0]["version_code"] == 500700000 + assert dpa.results[0]["min_sdk"] == 28 + assert dpa.results[0]["target_sdk"] == 28 + assert dpa.results[0]["package_type"] == "active" + assert dpa.results[0]["users"][0]["user_id"] == 0 assert dpa.results[0]["system"] is True def test_parsing_system_flag(self): @@ -60,42 +64,27 @@ class TestDumpsysPackagesArtifact: dpa.check_indicators() assert len(dpa.alertstore.alerts) == 1 - def test_per_user_fields_use_primary_user(self): - details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details( - """ User 0: installed=true - firstInstallTime=2024-01-10 09:19:39 - runtime permissions: - android.permission.CAMERA: granted=true - User 95: installed=false - firstInstallTime=1970-01-01 01:00:00 - runtime permissions: - android.permission.CAMERA: granted=false - android.permission.RECORD_AUDIO: granted=false + def test_hidden_packages_and_per_user_state(self): + dpa = DumpsysPackagesArtifact() + dpa.parse( + """Packages: + Package [com.example.active]: + appId=10001 + versionCode=12 minSdk=29 targetSdk=35 + User 0: installed=true hidden=false + firstInstallTime=2025-01-01 01:02:03 + User 10: installed=false hidden=true + firstInstallTime=2025-01-02 01:02:03 +Hidden system packages: + Package [com.example.hidden]: + appId=10002 + installerPackageName=null """ ) - assert details["first_install_time"] == "2024-01-10 09:19:39" - runtime_permissions = [ - permission - for permission in details["permissions"] - if permission["type"] == "runtime" + assert [record["package_type"] for record in dpa.results] == [ + "active", + "hidden_system", ] - assert runtime_permissions == [ - { - "name": "android.permission.CAMERA", - "granted": True, - "type": "runtime", - } - ] - - def test_per_user_fields_fall_back_when_user_zero_is_missing(self): - details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details( - """ User 10: installed=true - firstInstallTime=2024-02-10 09:19:39 - runtime permissions: - android.permission.CAMERA: granted=true -""" - ) - - assert details["first_install_time"] == "2024-02-10 09:19:39" - assert details["permissions"][-1]["name"] == "android.permission.CAMERA" + assert len(dpa.results[0]["users"]) == 2 + assert dpa.results[1]["installer"] is None From 832e46604d40dd14532cb3ad560ae4f6414e94d3 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:23 +0200 Subject: [PATCH 17/43] Parse all platform compatibility overrides --- .../artifacts/dumpsys_platform_compat.py | 50 +++++++++++++------ .../bugreport/dumpsys_platform_compat.py | 4 +- .../test_artifact_dumpsys_platform_compat.py | 3 ++ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/mvt/android/artifacts/dumpsys_platform_compat.py b/src/mvt/android/artifacts/dumpsys_platform_compat.py index c01fbfd..08cb46e 100644 --- a/src/mvt/android/artifacts/dumpsys_platform_compat.py +++ b/src/mvt/android/artifacts/dumpsys_platform_compat.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import re + from .artifact import AndroidArtifact @@ -25,19 +27,37 @@ class DumpsysPlatformCompatArtifact(AndroidArtifact): def parse(self, data: str) -> None: for line in data.splitlines(): - if not line.startswith("ChangeId(168419799; name=DOWNSCALED;"): + match = re.match(r"ChangeId\((\d+);\s*(.*)\)$", line.strip()) + if not match or "rawOverrides={" not in line: continue - - if line.strip() == "": - break - - # Look for rawOverrides field - if "rawOverrides={" in line: - # Extract the content inside the braces for rawOverrides - overrides_field = line.split("rawOverrides={", 1)[1].split("};", 1)[0] - - for entry in overrides_field.split(", "): - # Extract app name - uninstall_app = entry.split("=")[0].strip() - - self.results.append({"package_name": uninstall_app}) + body = match.group(2) + name_match = re.search(r"(?:^|;\s*)name=([^;]+)", body) + state = ( + "enabled" + if re.search(r"(?:^|;\s*)enabled(?:;|$)", body) + else "disabled" + ) + overridable = bool(re.search(r"(?:^|;\s*)overridable(?:;|$)", body)) + overrides_field = body.split("rawOverrides={", 1)[1].split("}", 1)[0] + for entry in overrides_field.split(","): + package_name, separator, raw_value = entry.strip().partition("=") + if not separator: + continue + value: bool | int | str + if raw_value in ("true", "false"): + value = raw_value == "true" + else: + try: + value = int(raw_value) + except ValueError: + value = raw_value + self.results.append( + { + "change_id": int(match.group(1)), + "change_name": name_match.group(1) if name_match else None, + "change_state": state, + "overridable": overridable, + "package_name": package_name, + "override_value": value, + } + ) diff --git a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py index 968bc25..dd06b40 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py +++ b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py @@ -13,7 +13,7 @@ from mvt.common.module_types import ModuleResults class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): - """This module extracts details on uninstalled apps.""" + """This module extracts raw per-package compatibility overrides.""" def __init__( self, @@ -48,4 +48,4 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): ) self.parse(content) - self.log.info("Found %d uninstalled apps", len(self.results)) + self.log.info("Found %d package compatibility overrides", len(self.results)) diff --git a/tests/android/test_artifact_dumpsys_platform_compat.py b/tests/android/test_artifact_dumpsys_platform_compat.py index c8c59b3..c202645 100644 --- a/tests/android/test_artifact_dumpsys_platform_compat.py +++ b/tests/android/test_artifact_dumpsys_platform_compat.py @@ -22,6 +22,9 @@ class TestDumpsysPlatformCompatArtifact: assert len(dbi.results) == 2 assert dbi.results[0]["package_name"] == "org.torproject.torbrowser" assert dbi.results[1]["package_name"] == "org.article19.circulo.next" + assert dbi.results[0]["change_id"] == 168419799 + assert dbi.results[0]["change_name"] == "DOWNSCALED" + assert dbi.results[0]["override_value"] is False def test_ioc_check(self, indicator_file): dbi = DumpsysPlatformCompatArtifact() From f747751b09a11f702c8215238dfb3c449e61a285 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 18/43] Add timezone provenance to file timestamps --- .../modules/bugreport/fs_timestamps.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py index 5a2ca48..3c7e324 100644 --- a/src/mvt/android/modules/bugreport/fs_timestamps.py +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -4,12 +4,15 @@ # https://license.mvt.re/1.1/ import logging +import datetime from typing import Optional +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from mvt.common.utils import convert_datetime_to_iso from .base import BugReportModule from mvt.common.module_types import ModuleResults from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact +from mvt.android.artifacts.getprop import GetProp class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): @@ -38,15 +41,44 @@ class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): def run(self) -> None: filesystem_files = self._get_files_by_pattern("FS/*") + timezone_name = None + dumpstate = self._get_dumpstate_file() + if dumpstate: + section = self.extract_command_section( + dumpstate.decode("utf-8", errors="replace"), + "------ SYSTEM PROPERTIES", + ) + properties = GetProp() + properties.parse(section) + timezone_name = properties.get_device_timezone() + timezone = None + if timezone_name: + try: + timezone = ZoneInfo(timezone_name) + except ZoneInfoNotFoundError: + self.log.warning("Unknown device timezone %s", timezone_name) + self.results = [] for file in filesystem_files: # Only the modification time is available in the zip file metadata. # The timezone is the local timezone of the machine the phone. modification_time = self._get_file_modification_time(file) + utc_time = None + if timezone is not None: + utc_time = convert_datetime_to_iso( + modification_time.replace(tzinfo=timezone).astimezone( + datetime.timezone.utc + ) + ) self.results.append( { "path": file, "modified_time": convert_datetime_to_iso(modification_time), + "modified_time_utc": utc_time, + "timezone": timezone_name, + "timestamp_source": ( + "zip_metadata" if self.zip_archive else "filesystem_metadata" + ), } ) From 8b85972cc310d6d7ba2f8f82eb3573d89ff57b96 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 19/43] Add bugreport process table parser --- src/mvt/android/artifacts/processes.py | 127 ++++++++++-------- .../android/modules/bugreport/processes.py | 22 +++ tests/android/test_artifact_processes.py | 13 +- 3 files changed, 107 insertions(+), 55 deletions(-) create mode 100644 src/mvt/android/modules/bugreport/processes.py diff --git a/src/mvt/android/artifacts/processes.py b/src/mvt/android/artifacts/processes.py index 5471eb2..97e27a9 100644 --- a/src/mvt/android/artifacts/processes.py +++ b/src/mvt/android/artifacts/processes.py @@ -1,72 +1,91 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +FIELD_NAMES = { + "LABEL": "label", + "USER": "user", + "PID": "pid", + "TID": "tid", + "PPID": "ppid", + "VSZ": "virtual_memory_size", + "RSS": "resident_set_size", + "WCHAN": "wchan", + "ADDR": "address", + "S": "state", + "PRI": "priority", + "NI": "nice", + "RTPRIO": "realtime_priority", + "SCH": "scheduler", + "PCY": "policy", + "TIME": "cpu_time", + "CMD": "command", + "NAME": "command", +} +INTEGER_FIELDS = { + "pid", + "tid", + "ppid", + "virtual_memory_size", + "resident_set_size", + "priority", + "nice", +} + + class Processes(AndroidArtifact): def parse(self, entry: str) -> None: - for line in entry.splitlines()[1:]: - proc = line.split() + self.results = [] + lines = [line for line in entry.splitlines() if line.strip()] + if not lines: + return + headers = lines[0].split() + if not all(header in FIELD_NAMES for header in headers): + return - # Skip empty lines - if len(proc) == 0: + for line in lines[1:]: + values = line.split(None, len(headers) - 1) + if len(values) != len(headers): continue - - # Sometimes WCHAN is empty. - if len(proc) == 8: - proc = proc[:5] + [""] + proc[5:] - - # Sometimes there is the security label. - if proc[0].startswith("u:r"): - label = proc[0] - proc = proc[1:] - else: - label = "" - - # Sometimes there is no WCHAN. - if len(proc) < 9: - proc = proc[:5] + [""] + proc[5:] - - self.results.append( - { - "user": proc[0], - "pid": int(proc[1]), - "ppid": int(proc[2]), - "virtual_memory_size": int(proc[3]), - "resident_set_size": int(proc[4]), - "wchan": proc[5], - "aprocress": proc[6], - "stat": proc[7], - "proc_name": proc[8].strip("[]"), - "label": label, - } - ) + result = {} + valid = True + for header, raw in zip(headers, values): + key = FIELD_NAMES[header] + value: str | int = raw.strip("[]") if key == "command" else raw + if key in INTEGER_FIELDS: + try: + value = int(value) + except ValueError: + valid = False + break + result[key] = value + if valid: + self.results.append(result) def check_indicators(self) -> None: if not self.indicators: return - for result in self.results: - proc_name = result.get("proc_name", "") - if not proc_name: + command = result.get("command", "") + if not isinstance(command, str): continue - - # Skipping this process because of false positives. - if result["proc_name"] == "gatekeeperd": + process_name = command.rsplit("/", 1)[-1] + if not process_name or process_name == "gatekeeperd": continue - - ioc_match = self.indicators.check_app_id(proc_name) - if ioc_match: - self.alertstore.critical( - ioc_match.message, "", result, matched_indicator=ioc_match.ioc - ) - continue - - ioc_match = self.indicators.check_process(proc_name) - if ioc_match: - self.alertstore.critical( - ioc_match.message, "", result, matched_indicator=ioc_match.ioc - ) + for checker in ( + self.indicators.check_app_id, + self.indicators.check_process, + ): + ioc_match = checker(process_name) + if ioc_match: + self.alertstore.critical( + ioc_match.message, + "", + result, + matched_indicator=ioc_match.ioc, + ) + break diff --git a/src/mvt/android/modules/bugreport/processes.py b/src/mvt/android/modules/bugreport/processes.py new file mode 100644 index 0000000..4f9b334 --- /dev/null +++ b/src/mvt/android/modules/bugreport/processes.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +from mvt.android.artifacts.processes import Processes as ProcessesArtifact + +from .base import BugReportModule + + +class Processes(ProcessesArtifact, BugReportModule): + """Extract the process and thread table from dumpstate.""" + + def run(self) -> None: + data = self._get_dumpstate_file() + if not data: + self.log.error("Unable to find dumpstate file") + return + section = self.extract_command_section( + data.decode("utf-8", errors="replace"), + "------ PROCESSES AND THREADS", + ) + self.parse(section) + self.log.info("Identified %d running process threads", len(self.results)) diff --git a/tests/android/test_artifact_processes.py b/tests/android/test_artifact_processes.py index 2806472..4d51037 100644 --- a/tests/android/test_artifact_processes.py +++ b/tests/android/test_artifact_processes.py @@ -20,7 +20,7 @@ class TestProcessesArtifact: assert len(p.results) == 0 p.parse(data) assert len(p.results) == 17 - assert p.results[0]["proc_name"] == "init" + assert p.results[0]["command"] == "init" def test_ioc_check(self, indicator_file): p = Processes() @@ -36,3 +36,14 @@ class TestProcessesArtifact: assert len(p.alertstore.alerts) == 0 p.check_indicators() assert len(p.alertstore.alerts) == 1 + + def test_bugreport_thread_columns(self): + p = Processes() + p.parse( + "LABEL USER PID TID PPID VSZ RSS WCHAN ADDR S PRI NI RTPRIO SCH PCY TIME CMD\n" + "u:r:init:s0 root 1 2 0 100 20 0 0 S 19 0 - 0 fg 00:00:01 init\n" + ) + + assert p.results[0]["label"] == "u:r:init:s0" + assert p.results[0]["tid"] == 2 + assert p.results[0]["command"] == "init" From 627ab32d427fb9d43a4bbc552a586f0a06ae22d9 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 20/43] Add bugreport settings parser --- src/mvt/android/artifacts/settings.py | 24 +++++++++++++++++++ src/mvt/android/modules/bugreport/settings.py | 22 +++++++++++++++++ tests/android_androidqf/test_settings.py | 15 ++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 src/mvt/android/modules/bugreport/settings.py diff --git a/src/mvt/android/artifacts/settings.py b/src/mvt/android/artifacts/settings.py index 17c3bf2..e0cd5f0 100644 --- a/src/mvt/android/artifacts/settings.py +++ b/src/mvt/android/artifacts/settings.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import re + from .artifact import AndroidArtifact ANDROID_DANGEROUS_SETTINGS = [ @@ -60,6 +62,28 @@ ANDROID_DANGEROUS_SETTINGS = [ class Settings(AndroidArtifact): + def parse(self, content: str) -> None: + self.results: dict[str, dict[str, str]] = {} + namespace: str | None = None + for line in content.splitlines(): + heading = re.match( + r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$", + line.strip(), + ) + if heading: + namespace = f"{heading.group(1).lower()}:user_{heading.group(2)}" + self.results[namespace] = {} + continue + if namespace is None or not line.startswith("_id:"): + continue + setting = re.match( + r"^_id:\S+\s+name:(.*?)\s+pkg:.*?\s+value:(.*?)" + r"(?:\s+default:.*\s+defaultSystemSet:(?:true|false))?$", + line, + ) + if setting: + self.results[namespace][setting.group(1)] = setting.group(2) + def check_indicators(self) -> None: for namespace, settings in self.results.items(): for key, value in settings.items(): diff --git a/src/mvt/android/modules/bugreport/settings.py b/src/mvt/android/modules/bugreport/settings.py new file mode 100644 index 0000000..20180ad --- /dev/null +++ b/src/mvt/android/modules/bugreport/settings.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +from mvt.android.artifacts.settings import Settings as SettingsArtifact + +from .base import BugReportModule + + +class Settings(SettingsArtifact, BugReportModule): + """Extract all SettingsProvider namespaces and users.""" + + def run(self) -> None: + data = self._get_dumpstate_file() + if not data: + self.log.error("Unable to find dumpstate file") + return + section = self.extract_dumpsys_section( + data.decode("utf-8", errors="replace"), "DUMP OF SERVICE settings:" + ) + self.parse(section) + count = sum(len(settings) for settings in self.results.values()) + self.log.info("Identified %d Android settings", count) diff --git a/tests/android_androidqf/test_settings.py b/tests/android_androidqf/test_settings.py index ce14460..6edfec8 100644 --- a/tests/android_androidqf/test_settings.py +++ b/tests/android_androidqf/test_settings.py @@ -6,12 +6,27 @@ from pathlib import Path from mvt.android.modules.androidqf.aqf_settings import AQFSettings +from mvt.android.artifacts.settings import Settings from mvt.common.module import run_module from ..utils import get_android_androidqf, list_files class TestSettingsModule: + def test_bugreport_settings_format(self): + settings = Settings() + settings.parse( + "GLOBAL SETTINGS (user 0)\n" + "_id:1 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true\n" + "SECURE SETTINGS (user 10)\n" + "_id:2 name:accessibility_enabled pkg:android value:1\n" + ) + + assert settings.results == { + "global:user_0": {"adb_wifi_enabled": "0"}, + "secure:user_10": {"accessibility_enabled": "1"}, + } + def test_parsing(self): data_path = get_android_androidqf() m = AQFSettings(target_path=data_path) From 45fdf8579083dac618d57150429cc1d1149cd2b9 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 21/43] Add bugreport mountinfo parser --- src/mvt/android/artifacts/mounts.py | 45 +++++++++++++++++++++ src/mvt/android/modules/bugreport/mounts.py | 41 +++++++++++++++++++ tests/android_androidqf/test_mounts.py | 13 ++++++ 3 files changed, 99 insertions(+) create mode 100644 src/mvt/android/modules/bugreport/mounts.py diff --git a/src/mvt/android/artifacts/mounts.py b/src/mvt/android/artifacts/mounts.py index a35d889..5a5537b 100644 --- a/src/mvt/android/artifacts/mounts.py +++ b/src/mvt/android/artifacts/mounts.py @@ -117,6 +117,51 @@ class Mounts(AndroidArtifact): # Skip lines that don't match expected format continue + @staticmethod + def parse_mountinfo(entry: str, process_id: int) -> list[dict[str, Any]]: + """Parse Linux /proc/PID/mountinfo records.""" + results = [] + for line in entry.splitlines(): + fields = line.split() + if "-" not in fields: + continue + separator = fields.index("-") + if separator < 6 or len(fields) < separator + 4: + continue + try: + mount_id = int(fields[0]) + parent_id = int(fields[1]) + except ValueError: + continue + mount_options = fields[5].split(",") + super_options = fields[separator + 3].split(",") + options = list(dict.fromkeys(mount_options + super_options)) + mount_point = fields[4].replace("\\040", " ") + device = fields[separator + 2].replace("\\040", " ") + filesystem_type = fields[separator + 1] + is_system = mount_point in SUSPICIOUS_MOUNT_POINTS or any( + mount_point.startswith(f"{prefix}/") + for prefix in SUSPICIOUS_MOUNT_POINTS + ) + results.append( + { + "mount_id": mount_id, + "parent_id": parent_id, + "major_minor": fields[2], + "root": fields[3].replace("\\040", " "), + "mount_point": mount_point, + "device": device, + "filesystem_type": filesystem_type, + "mount_options": ",".join(options), + "options_list": options, + "optional_fields": fields[6:separator], + "is_system_partition": is_system, + "is_read_write": "rw" in options, + "process_ids": [process_id], + } + ) + return results + def check_indicators(self) -> None: """ Check for suspicious mount configurations that may indicate root access diff --git a/src/mvt/android/modules/bugreport/mounts.py b/src/mvt/android/modules/bugreport/mounts.py new file mode 100644 index 0000000..ecef40c --- /dev/null +++ b/src/mvt/android/modules/bugreport/mounts.py @@ -0,0 +1,41 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +import re + +from mvt.android.artifacts.mounts import Mounts as MountsArtifact + +from .base import BugReportModule + + +class Mounts(MountsArtifact, BugReportModule): + """Extract and deduplicate process mount namespaces from mountinfo.""" + + def run(self) -> None: + mount_files = self._get_files_by_pattern("FS/proc/*/mountinfo") + if not mount_files: + mount_files = self._get_files_by_pattern("*/proc/*/mountinfo") + unique: dict[tuple, dict] = {} + for file_path in mount_files: + pid_match = re.search(r"/proc/(\d+)/mountinfo$", file_path) + if not pid_match: + continue + entries = self.parse_mountinfo( + self._get_file_content(file_path).decode("utf-8", errors="replace"), + int(pid_match.group(1)), + ) + for entry in entries: + identity = ( + entry["major_minor"], + entry["root"], + entry["mount_point"], + entry["device"], + entry["filesystem_type"], + entry["mount_options"], + ) + if identity in unique: + unique[identity]["process_ids"].extend(entry["process_ids"]) + else: + unique[identity] = entry + self.results = list(unique.values()) + self.log.info("Extracted %d unique mount records", len(self.results)) diff --git a/tests/android_androidqf/test_mounts.py b/tests/android_androidqf/test_mounts.py index d66925d..5f47176 100644 --- a/tests/android_androidqf/test_mounts.py +++ b/tests/android_androidqf/test_mounts.py @@ -13,6 +13,19 @@ from ..utils import get_android_androidqf, list_files class TestAndroidqfMountsArtifact: + def test_parse_proc_mountinfo(self): + from mvt.android.artifacts.mounts import Mounts as MountsArtifact + + results = MountsArtifact.parse_mountinfo( + "41 40 254:13 / / ro,relatime shared:1 - erofs /dev/block/dm-13 ro,seclabel\n", + 123, + ) + + assert results[0]["mount_id"] == 41 + assert results[0]["mount_point"] == "/" + assert results[0]["filesystem_type"] == "erofs" + assert results[0]["process_ids"] == [123] + def test_parse_mounts_token_checks(self): """ Test the artifact-level `parse` method using tolerant token checks. From 3a43f6fcc7d5d4dd3a0adf15e97894494ebeb84e Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 22/43] Merge plaintext and protobuf tombstones --- .../android/artifacts/tombstone_crashes.py | 32 ++++++-- .../android/modules/bugreport/tombstones.py | 81 ++++++++++++++++--- tests/android/test_artifact_tombstones.py | 7 +- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index bcfb389..d22e65a 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -131,6 +131,13 @@ class TombstoneCrashArtifact(AndroidArtifact): self, file_name: str, file_timestamp: datetime.datetime, data: bytes ) -> None: """Parse Android tombstone crash files from a protobuf object.""" + self.results.append(self.parse_protobuf_record(file_name, file_timestamp, data)) + + def parse_protobuf_record( + self, file_name: str, file_timestamp: datetime.datetime, data: bytes + ) -> dict: + if not data: + raise ValueError("empty protobuf tombstone") tombstone_pb = Tombstone().parse(data) tombstone_dict = tombstone_pb.to_dict( casing=betterproto2.Casing.SNAKE, include_default_values=True @@ -143,20 +150,31 @@ class TombstoneCrashArtifact(AndroidArtifact): tombstone_dict["file_name"] = file_name tombstone_dict["file_timestamp"] = convert_datetime_to_iso(file_timestamp) tombstone_dict["process_name"] = self._proccess_name_from_thread(tombstone_dict) + if isinstance(tombstone_dict.get("selinux_label"), str): + tombstone_dict["selinux_label"] = tombstone_dict["selinux_label"].rstrip( + "\x00" + ) # Confirm the tombstone is valid, and matches the output model tombstone = TombstoneCrashResult.model_validate(tombstone_dict) - self.results.append(tombstone.model_dump()) + return tombstone.model_dump() def parse( self, file_name: str, file_timestamp: datetime.datetime, content: bytes ) -> None: """Parse text Android tombstone crash files.""" + self.results.append(self.parse_text_record(file_name, file_timestamp, content)) + + def parse_text_record( + self, file_name: str, file_timestamp: datetime.datetime, content: bytes + ) -> dict: + if not content: + raise ValueError("empty plaintext tombstone") tombstone_dict = { "file_name": file_name, "file_timestamp": convert_datetime_to_iso(file_timestamp), } - lines = content.decode("utf-8").splitlines() + lines = content.decode("utf-8", errors="replace").splitlines() for line_num, line in enumerate(lines, 1): if not line.strip() or TOMBSTONE_DELIMITER in line: continue @@ -171,7 +189,7 @@ class TombstoneCrashArtifact(AndroidArtifact): # Validate the tombstone and add it to the results tombstone = TombstoneCrashResult.model_validate(tombstone_dict) - self.results.append(tombstone.model_dump()) + return tombstone.model_dump() def _parse_tombstone_line( self, line: str, key: str, destination_key: str, tombstone: dict @@ -195,7 +213,9 @@ class TombstoneCrashArtifact(AndroidArtifact): if line_key != key: raise ValueError(f"Expected key {key}, got {line_key}") - value_clean = value.strip().strip("'") + value_clean = value.strip() + if len(value_clean) >= 2 and value_clean[0] == value_clean[-1] == "'": + value_clean = value_clean[1:-1] if destination_key == "uid": tombstone[destination_key] = int(value_clean) elif destination_key == "process_uptime": @@ -269,9 +289,7 @@ class TombstoneCrashArtifact(AndroidArtifact): @staticmethod def _parse_timestamp_string(timestamp: str) -> str: timestamp_parsed = parser.parse(timestamp) - # Preserve the source wall-clock time while returning the project-wide ISO format. - local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc) - return convert_datetime_to_iso(local_timestamp) + return convert_datetime_to_iso(timestamp_parsed) @staticmethod def _proccess_name_from_thread(tombstone_dict: dict) -> str: diff --git a/src/mvt/android/modules/bugreport/tombstones.py b/src/mvt/android/modules/bugreport/tombstones.py index c4a7afb..4d121a1 100644 --- a/src/mvt/android/modules/bugreport/tombstones.py +++ b/src/mvt/android/modules/bugreport/tombstones.py @@ -43,21 +43,76 @@ class Tombstones(TombstoneCrashArtifact, BugReportModule): ) return - for tombstone_file in sorted(tombstone_files): - tombstone_filename = tombstone_file.split("/")[-1] - modification_time = self._get_file_modification_time(tombstone_file) - tombstone_data = self._get_file_content(tombstone_file) + grouped: dict[str, dict[str, str]] = {} + for file_path in tombstone_files: + file_name = file_path.rsplit("/", 1)[-1] + source = "protobuf" if file_name.endswith(".pb") else "text" + crash_id = file_name.removesuffix(".pb") + grouped.setdefault(crash_id, {})[source] = file_path - try: - if tombstone_file.endswith(".pb"): - self.parse_protobuf( - tombstone_filename, modification_time, tombstone_data + for crash_id, paths in sorted(grouped.items()): + parsed_sources: dict[str, dict] = {} + source_records: dict[str, dict] = {} + for source in ("text", "protobuf"): + file_path = paths.get(source) + if file_path is None: + continue + file_name = file_path.rsplit("/", 1)[-1] + file_timestamp = self._get_file_modification_time(file_path) + source_info = { + "file_name": file_name, + "file_timestamp": file_timestamp.isoformat(), + "parsed": False, + "error": None, + "record": None, + } + try: + data = self._get_file_content(file_path) + if source == "protobuf": + record = self.parse_protobuf_record( + file_name, file_timestamp, data + ) + else: + record = self.parse_text_record(file_name, file_timestamp, data) + source_info["parsed"] = True + source_info["record"] = record + source_records[source] = record + except Exception as exc: + source_info["error"] = str(exc) + self.log.error( + "Error parsing tombstone file %s: %s", file_path, exc ) - else: - self.parse(tombstone_filename, modification_time, tombstone_data) - except ValueError as e: - # Catch any exceptions raised during parsing or validation. - self.log.error(f"Error parsing tombstone file {tombstone_file}: {e}") + parsed_sources[source] = source_info + + if not source_records: + continue + preferred = source_records.get("protobuf") or source_records["text"] + canonical = dict(preferred) + text_record = source_records.get("text") + if text_record: + for key, value in text_record.items(): + if canonical.get(key) in (None, "", [], {}): + canonical[key] = value + + differences = {} + protobuf_record = source_records.get("protobuf") + if text_record and protobuf_record: + for key in text_record.keys() & protobuf_record.keys(): + if key in ("file_name", "file_timestamp"): + continue + if text_record[key] != protobuf_record[key]: + differences[key] = { + "text": text_record[key], + "protobuf": protobuf_record[key], + } + canonical.update( + { + "crash_id": crash_id, + "sources": parsed_sources, + "differences": differences, + } + ) + self.results.append(canonical) self.log.info( "Extracted a total of %d tombstone files", diff --git a/tests/android/test_artifact_tombstones.py b/tests/android/test_artifact_tombstones.py index f88a9c2..e9159bd 100644 --- a/tests/android/test_artifact_tombstones.py +++ b/tests/android/test_artifact_tombstones.py @@ -128,8 +128,5 @@ class TestTombstoneCrashArtifact: assert tombstone_result.get("pid") == 25541 assert tombstone_result.get("process_name") == "mtk.ape.decoder" - # With Android logs we want to keep timestamps as device local time for consistency. - # We often don't know the time offset for a log entry and so can't convert everything to UTC. - # MVT should output the local time only: - # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000 - assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.518290" + # Tombstones include an explicit offset, so normalize them to UTC. + assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.518290" From 7ef26779bf2edb01efe0aefb9f987be54b314454 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 23/43] Register added bugreport parsers --- src/mvt/android/modules/bugreport/__init__.py | 10 ++++++++- tests/android_bugreport/test_bugreport.py | 22 ++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 1594af9..e23bf5a 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .dumpsys_accessibility import DumpsysAccessibility from .dumpsys_activities import DumpsysActivities from .dumpsys_appops import DumpsysAppops @@ -16,8 +18,11 @@ from .dumpsys_receivers import DumpsysReceivers from .dumpsys_adb_state import DumpsysADBState from .fs_timestamps import BugReportTimestamps from .tombstones import Tombstones +from .mounts import Mounts +from .processes import Processes +from .settings import Settings -BUGREPORT_MODULES = [ +BUGREPORT_MODULES: list[type[MVTModule]] = [ DumpsysAccessibility, DumpsysActivities, DumpsysAppops, @@ -31,4 +36,7 @@ BUGREPORT_MODULES = [ DumpsysADBState, BugReportTimestamps, Tombstones, + Processes, + Settings, + Mounts, ] diff --git a/tests/android_bugreport/test_bugreport.py b/tests/android_bugreport/test_bugreport.py index 75df106..413bb18 100644 --- a/tests/android_bugreport/test_bugreport.py +++ b/tests/android_bugreport/test_bugreport.py @@ -54,10 +54,11 @@ class TestBugreportAnalysis: == "com.samsung.android.provider.filterprovider" ) assert m.results[1]["package_name"] == "com.instagram.android" - assert m.results[0]["installer"] == "" + assert m.results[0]["installer"] is None assert m.results[1]["installer"] == "com.android.vending" assert len(m.results[0]["permissions"]) == 4 - assert len(m.results[1]["permissions"]) == 32 + assert len(m.results[1]["permissions"]) == 20 + assert len(m.results[1]["users"][0]["permissions"]) == 19 def test_getprop_module(self): m = self.launch_bug_report_module(DumpsysGetProp) @@ -66,29 +67,34 @@ class TestBugreportAnalysis: def test_receivers_match_exact_package_name(self, indicators_factory): intent = "android.intent.action.PHONE_STATE" false_positive = { + "resolver_type": "non_data_action", + "key": intent, "package_name": "com.android.phone", - "receiver": ( + "component": ( "com.android.phone/" "com.android.services.telephony.sip.SipIncomingCallReceiver" ), + "filter_count": 1, } malicious_receiver = { + "resolver_type": "non_data_action", + "key": intent, "package_name": "com.android.services", - "receiver": "com.android.services/com.example.SomeReceiver", + "component": "com.android.services/com.example.SomeReceiver", + "filter_count": 1, } - module = DumpsysReceivers( - results={intent: [false_positive, malicious_receiver]} - ) + module = DumpsysReceivers(results=[false_positive, malicious_receiver]) module.indicators = indicators_factory(app_ids=["com.android.services"]) module.check_indicators() assert len(module.alertstore.alerts) == 1 alert = module.alertstore.alerts[0] - assert alert.event == {intent: malicious_receiver} + assert alert.event == malicious_receiver assert alert.matched_indicator.value == "com.android.services" def test_tombstones_modules(self): m = self.launch_bug_report_module(Tombstones) assert len(m.results) == 2 assert m.results[1]["pid"] == 3559 + assert m.results[0]["sources"]["text"]["parsed"] is True From fd27fe33784991592114711070a4d6381b550e82 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Sat, 22 Aug 2026 14:16:24 +0200 Subject: [PATCH 24/43] Fix module registry type annotations --- src/mvt/android/modules/androidqf/__init__.py | 4 +++- src/mvt/android/modules/backup/__init__.py | 4 +++- src/mvt/android/modules/intrusion_logs/__init__.py | 4 +++- src/mvt/ios/modules/backup/__init__.py | 9 ++++++++- src/mvt/ios/modules/fs/__init__.py | 4 +++- src/mvt/ios/modules/mixed/__init__.py | 4 +++- 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/mvt/android/modules/androidqf/__init__.py b/src/mvt/android/modules/androidqf/__init__.py index 1d8d619..05e7cde 100644 --- a/src/mvt/android/modules/androidqf/__init__.py +++ b/src/mvt/android/modules/androidqf/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .aqf_files import AQFFiles from .aqf_getprop import AQFGetProp from .aqf_log_timestamps import AQFLogTimestamps @@ -12,7 +14,7 @@ from .aqf_settings import AQFSettings from .mounts import Mounts from .root_binaries import RootBinaries -ANDROIDQF_MODULES = [ +ANDROIDQF_MODULES: list[type[MVTModule]] = [ AQFPackages, AQFProcesses, AQFGetProp, diff --git a/src/mvt/android/modules/backup/__init__.py b/src/mvt/android/modules/backup/__init__.py index 47a5e1c..683a383 100644 --- a/src/mvt/android/modules/backup/__init__.py +++ b/src/mvt/android/modules/backup/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .sms import SMS -BACKUP_MODULES = [SMS] +BACKUP_MODULES: list[type[MVTModule]] = [SMS] diff --git a/src/mvt/android/modules/intrusion_logs/__init__.py b/src/mvt/android/modules/intrusion_logs/__init__.py index f8be973..a0e7bdd 100644 --- a/src/mvt/android/modules/intrusion_logs/__init__.py +++ b/src/mvt/android/modules/intrusion_logs/__init__.py @@ -3,11 +3,13 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .connect_event import ConnectEvent from .dns_event import DnsEvent from .security_event import SecurityEvent -INTRUSION_LOGS_MODULES = [ +INTRUSION_LOGS_MODULES: list[type[MVTModule]] = [ DnsEvent, ConnectEvent, SecurityEvent, diff --git a/src/mvt/ios/modules/backup/__init__.py b/src/mvt/ios/modules/backup/__init__.py index 0fb833a..ce8499b 100644 --- a/src/mvt/ios/modules/backup/__init__.py +++ b/src/mvt/ios/modules/backup/__init__.py @@ -3,9 +3,16 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .backup_info import BackupInfo from .configuration_profiles import ConfigurationProfiles from .manifest import Manifest from .profile_events import ProfileEvents -BACKUP_MODULES = [BackupInfo, ConfigurationProfiles, Manifest, ProfileEvents] +BACKUP_MODULES: list[type[MVTModule]] = [ + BackupInfo, + ConfigurationProfiles, + Manifest, + ProfileEvents, +] diff --git a/src/mvt/ios/modules/fs/__init__.py b/src/mvt/ios/modules/fs/__init__.py index 06dba24..b00ee35 100644 --- a/src/mvt/ios/modules/fs/__init__.py +++ b/src/mvt/ios/modules/fs/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .analytics import Analytics from .analytics_ios_versions import AnalyticsIOSVersions from .cache_files import CacheFiles @@ -15,7 +17,7 @@ from .webkit_indexeddb import WebkitIndexedDB from .webkit_localstorage import WebkitLocalStorage from .webkit_safariviewservice import WebkitSafariViewService -FS_MODULES = [ +FS_MODULES: list[type[MVTModule]] = [ CacheFiles, Filesystem, Netusage, diff --git a/src/mvt/ios/modules/mixed/__init__.py b/src/mvt/ios/modules/mixed/__init__.py index a846bf2..027dce0 100644 --- a/src/mvt/ios/modules/mixed/__init__.py +++ b/src/mvt/ios/modules/mixed/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .applications import Applications from .calendar import Calendar from .calls import Calls @@ -28,7 +30,7 @@ from .webkit_session_resource_log import WebkitSessionResourceLog from .whatsapp import Whatsapp from .whatsapp_contacts import WhatsappContacts -MIXED_MODULES = [ +MIXED_MODULES: list[type[MVTModule]] = [ Calls, ChromeFavicon, ChromeHistory, From 65df48325828f317268750c205c39a22518bbf35 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Tue, 25 Aug 2026 19:19:47 +0200 Subject: [PATCH 25/43] Parse multiline Android properties --- src/mvt/android/artifacts/getprop.py | 21 +++++++++------------ tests/android/test_artifact_getprop.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index a03491b..094eb5d 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -28,19 +28,16 @@ INTERESTING_PROPERTIES = [ class GetProp(AndroidArtifact): def parse(self, entry: str) -> None: self.results: List[Dict[str, str]] = [] - rxp = re.compile(r"^\[([^]]+)\]: \[(.*)\]$") + # A property value may span several lines: persist.sys.boot.reason.history + # prints one boot per line. Matching the whole section instead of line by + # line lets a value run to the first closing bracket that ends a line. + rxp = re.compile( + r"^[ \t]*\[([^]]+)\]: \[(.*?)\][ \t\r]*$", + re.MULTILINE | re.DOTALL, + ) - for line in entry.splitlines(): - line = line.strip() - if line == "": - continue - - match = rxp.match(line) - if not match: - continue - - prop_entry = {"name": match.group(1), "value": match.group(2)} - self.results.append(prop_entry) + for name, value in rxp.findall(entry): + self.results.append({"name": name, "value": value}) def get_device_timezone(self) -> str | None: """ diff --git a/tests/android/test_artifact_getprop.py b/tests/android/test_artifact_getprop.py index 7815fbd..0fbe742 100644 --- a/tests/android/test_artifact_getprop.py +++ b/tests/android/test_artifact_getprop.py @@ -48,3 +48,20 @@ class TestGetPropArtifact: {"name": "empty", "value": ""}, {"name": "valid", "value": "value"}, ] + + def test_multiline_value(self): + gp = GetProp() + gp.parse( + "[persist.sys.boot.reason.history]: [" + "reboot,ota,1697044974\n" + "reboot,watchdog,1696958574]\n" + "[ro.build.version.sdk]: [35]\n" + ) + + assert gp.results == [ + { + "name": "persist.sys.boot.reason.history", + "value": "reboot,ota,1697044974\nreboot,watchdog,1696958574", + }, + {"name": "ro.build.version.sdk", "value": "35"}, + ] From 3c8a581fd027f4caccbc7174549791994263695d Mon Sep 17 00:00:00 2001 From: "va@resident" Date: Tue, 25 Aug 2026 21:18:36 +0300 Subject: [PATCH 26/43] Do not end the whole androidqf run on an encrypted backup.ab (#891) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit from_ab() raises InvalidAndroidBackup instead of exiting when it runs as a sub-command, which check-androidqf catches to skip the backup modules. The two password branches still called sys.exit(1) unconditionally, and since run_backup_cmd() runs inside finish(), that ended the parent run before the intrusion-logs command and before the timeline, alerts, urls, info and run manifest were stored — leaving an output directory that looks complete but has no alerts.json. Also drop "as backup.ab is malformed" from the skip warning: it covers a missing or wrong password too. --- src/mvt/android/cmd_check_androidqf.py | 4 +- src/mvt/android/cmd_check_backup.py | 4 ++ .../test_check_backup_optional_failure.py | 39 +++++++++++++++++++ tests/test_check_android_androidqf.py | 2 +- 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 tests/android/test_check_backup_optional_failure.py diff --git a/src/mvt/android/cmd_check_androidqf.py b/src/mvt/android/cmd_check_androidqf.py index ffa3e13..99ac5a3 100644 --- a/src/mvt/android/cmd_check_androidqf.py +++ b/src/mvt/android/cmd_check_androidqf.py @@ -292,9 +292,7 @@ class CmdAndroidCheckAndroidQF(Command): try: cmd.from_ab(backup) except InvalidAndroidBackup as exc: - self.log.warning( - "Skipping backup modules as backup.ab is malformed: %s", exc - ) + self.log.warning("Skipping backup modules: %s", exc) return False cmd.run() diff --git a/src/mvt/android/cmd_check_backup.py b/src/mvt/android/cmd_check_backup.py index b75bb34..94be4ee 100644 --- a/src/mvt/android/cmd_check_backup.py +++ b/src/mvt/android/cmd_check_backup.py @@ -87,11 +87,15 @@ class CmdAndroidCheckBackup(Command): if header["encryption"] != "none": password = prompt_or_load_android_backup_password(log, self.module_options) if not password: + if self.sub_command: + raise InvalidAndroidBackup("No backup password provided") log.critical("No backup password provided.") sys.exit(1) try: tardata = parse_backup_file(ab_file_bytes, password=password) except InvalidBackupPassword: + if self.sub_command: + raise InvalidAndroidBackup("Invalid backup password") log.critical("Invalid backup password") sys.exit(1) except AndroidBackupParsingError as exc: diff --git a/tests/android/test_check_backup_optional_failure.py b/tests/android/test_check_backup_optional_failure.py new file mode 100644 index 0000000..72dae42 --- /dev/null +++ b/tests/android/test_check_backup_optional_failure.py @@ -0,0 +1,39 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""An encrypted backup.ab must not take the whole check-androidqf run with it. + +`CmdAndroidCheckBackup.from_ab()` already raises `InvalidAndroidBackup` instead +of exiting when it runs as a sub-command (`check-androidqf` catches that and +skips the backup modules), for a wrong file format and for a parse error. The +password branches used to call `sys.exit(1)` unconditionally, which ends the +parent run inside `finish()` — before the intrusion-logs command and before the +timeline, alerts, urls, info and run-manifest are written. +""" + +import pytest + +from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBackup + +ENCRYPTED_AB_HEADER = b"ANDROID BACKUP\n5\n0\nAES-256\n" + b"\x00" * 64 + + +class TestCheckBackupOptionalFailure: + def _cmd(self, tmp_path, sub_command): + return CmdAndroidCheckBackup( + target_path=None, + results_path=str(tmp_path), + module_options={"interactive": False}, + sub_command=sub_command, + ) + + def test_missing_password_raises_when_nested(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=True) + with pytest.raises(InvalidAndroidBackup): + cmd.from_ab(ENCRYPTED_AB_HEADER) + + def test_missing_password_still_exits_on_its_own_command(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=False) + with pytest.raises(SystemExit): + cmd.from_ab(ENCRYPTED_AB_HEADER) diff --git a/tests/test_check_android_androidqf.py b/tests/test_check_android_androidqf.py index 2253a50..4e6ef98 100644 --- a/tests/test_check_android_androidqf.py +++ b/tests/test_check_android_androidqf.py @@ -155,7 +155,7 @@ class TestCheckAndroidqfCommand: result = runner.invoke(check_androidqf, [str(path)]) assert result.exit_code == 0 - assert "Skipping backup modules as backup.ab is malformed" in caplog.text + assert "Skipping backup modules: Invalid backup format" in caplog.text assert not any( record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records ) From 874f75bb4cf7a8bd3860bab6e8b56afa7c52f6f6 Mon Sep 17 00:00:00 2001 From: "va@resident" Date: Tue, 25 Aug 2026 22:35:33 +0300 Subject: [PATCH 27/43] Do not discard the whole tombstone on a "Caused by:" line (#892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keys are matched as bare prefixes, so `Caused by:` inside an abort message reaches the `Cause` key, fails the key comparison and raises — and the per-line loop turns that into an error that drops the entire text tombstone, stack trace included. A key mismatch means "this line is not that key", not "this file is broken": decline the line and let the remaining keys have their turn. A line with no colon is declined the same way instead of raising on the unpack. The same trap has a second form in the field, HiSilicon/Huawei tombstones printing `code around pc:` against the `code` key. --- .../android/artifacts/tombstone_crashes.py | 11 ++-- .../test_artifact_tombstone_caused_by.py | 54 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 tests/android/test_artifact_tombstone_caused_by.py diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index bcfb389..1b6e115 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -191,9 +191,14 @@ class TombstoneCrashArtifact(AndroidArtifact): def _load_key_value_line( self, line: str, key: str, destination_key: str, tombstone: dict ) -> bool: - line_key, value = line.split(":", 1) - if line_key != key: - raise ValueError(f"Expected key {key}, got {line_key}") + # The caller matched the key as a bare prefix, so a longer word starting + # with it arrives here: `Caused by: …` inside an abort message reaches + # the `Cause` key. That is a different line, not a broken file — say so + # by declining it, and let the remaining keys have their turn. Raising + # here discarded the whole tombstone, crash and stack trace included. + line_key, separator, value = line.partition(":") + if not separator or line_key != key: + return False value_clean = value.strip().strip("'") if destination_key == "uid": diff --git a/tests/android/test_artifact_tombstone_caused_by.py b/tests/android/test_artifact_tombstone_caused_by.py new file mode 100644 index 0000000..9a2df85 --- /dev/null +++ b/tests/android/test_artifact_tombstone_caused_by.py @@ -0,0 +1,54 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""A `Caused by:` line must not discard the whole text tombstone. + +Keys are matched as bare prefixes, so `Caused by: …` — an ordinary line inside +an abort message — reached the `Cause` key, failed the key comparison and +raised, which `Tombstones.run()` logged while dropping the entire crash record. +Seen on a 1.6 MB tombstone whose protobuf twin was zero bytes: the crash then +had no representation at all. +""" + +import datetime + +from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact + +TOMBSTONE = b"""\ +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +Build fingerprint: 'Xiaomi/vili_eea/vili:13/TKQ1.220829.002/V14.0.10.0:user/release-keys' +Revision: '0' +ABI: 'arm64' +Timestamp: 2023-08-24 14:54:47.999124034+0300 +Process uptime: 12199s +Cmdline: com.example.game +pid: 8044, tid: 26222, name: UnityMain >>> com.example.game <<< +uid: 10235 +signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr -------- +Abort message: 'No pending exception expected: java.lang.SecurityException: listen + at void android.os.Parcel.readException() (Parcel.java:2920) +Caused by: android.os.RemoteException: Remote stack trace: +\tat com.android.server.TelephonyRegistry.listen(TelephonyRegistry.java:1096) +""" + +WITH_CAUSE = TOMBSTONE + b"Cause: null pointer dereference\n" + + +class TestTombstoneCausedBy: + def _parse(self, content): + artifact = TombstoneCrashArtifact() + artifact.results = [] + artifact.parse("tombstone_23", datetime.datetime(2023, 8, 24), content) + return artifact.results + + def test_caused_by_line_does_not_discard_the_tombstone(self): + results = self._parse(TOMBSTONE) + assert len(results) == 1 + assert results[0]["pid"] == 8044 + assert results[0]["process_name"] == "UnityMain" + assert results[0]["uid"] == 10235 + + def test_the_real_cause_key_is_still_parsed(self): + results = self._parse(WITH_CAUSE) + assert results[0]["cause"] == "null pointer dereference" From 65b8114469cd8db65a4ac24c2fbd1624123da210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:13 +0200 Subject: [PATCH 28/43] Move development documentation under docs/development (#893) The Development section holds more than one page and will grow further, so give it its own directory. development.md becomes development/index.md, which keeps its published URL, and the custom CLI command page moves alongside it. Update the nav, the cross-link between the two pages, and the README link to the custom command documentation, which is now published at /development/custom_commands/. --- README.md | 2 +- docs/{ => development}/custom_commands.md | 2 +- docs/{development.md => development/index.md} | 0 mkdocs.yml | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename docs/{ => development}/custom_commands.md (97%) rename docs/{development.md => development/index.md} (100%) diff --git a/README.md b/README.md index 65fcdee..d4ac9b4 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ details. Users can also add top-level commands to `mvt-ios` and `mvt-android` from installed Python packages or local files and folders. See the -[custom CLI command documentation](https://docs.mvt.re/en/latest/custom_commands/) +[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/) for the plugin entry points and `--load-command` interface. diff --git a/docs/custom_commands.md b/docs/development/custom_commands.md similarity index 97% rename from docs/custom_commands.md rename to docs/development/custom_commands.md index ab89fd9..764c70b 100644 --- a/docs/custom_commands.md +++ b/docs/development/custom_commands.md @@ -1,7 +1,7 @@ # Custom CLI Commands MVT can load additional top-level commands into `mvt-ios` and `mvt-android`. -Custom commands are different from [custom forensic modules](development.md#custom-modules): +Custom commands are different from [custom forensic modules](index.md#custom-modules): commands add new CLI operations, while modules add analysis steps to existing `check-*` commands. diff --git a/docs/development.md b/docs/development/index.md similarity index 100% rename from docs/development.md rename to docs/development/index.md diff --git a/mkdocs.yml b/mkdocs.yml index a7b22ac..11a91dc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,6 @@ nav: - Check Android Intrusion Logs: "android/intrusion_logs.md" - Indicators of Compromise: "iocs.md" - Development: - - Development Instructions: "development.md" - - Custom CLI Commands: "custom_commands.md" + - Development Instructions: "development/index.md" + - Custom CLI Commands: "development/custom_commands.md" - License: "license.md" From 097766a63b5edcb8364c12368b4141288f1f5a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:13 +0200 Subject: [PATCH 29/43] Add namespaced plugin configuration support (#894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add namespaced plugin configuration support Plugin packages need somewhere to keep their own settings, but MVT rewrites its config.yaml with only the fields it knows about, so any foreign section is dropped. Add MVTPluginSettings, a pydantic-settings base class that gives each plugin its own file under the MVT config folder and its own MVT_PLUGIN__ environment variable namespace. Settings resolve from constructor arguments, then the environment, then the plugin file, then the field defaults. Saving skips the values the environment currently supplies, so credentials passed as environment variables are not copied to disk, and writes through a private temporary file so a settings file is never partially written or briefly readable by other users. * Add per-plugin data folders Plugins had no sanctioned place to keep the data they persist, so the plugin configuration documentation suggested a CACHE_FOLDER setting defaulting to ~/.cache/example-plugin. That is a Linux convention which is wrong on macOS, nothing expands or creates it, and it turns a path into a setting a user can be asked to configure. Add plugin_data_folder(), which returns the folder a plugin should use for caches, downloaded artifacts, synchronization state or anything else it writes to disk, and creates it if it is missing. The plugin name is validated before anything is created, so a name holding a path separator raises an error and leaves no folder behind, and calling the function again returns the same folder with its contents untouched. The folder sits under plugin-data rather than under the plugins folder which holds the settings files. On macOS the configuration folder and the data folder are the same directory, so reusing the plugins name would leave each plugin's data folder in among the settings files. Both the plugin-data folder and the folder of each plugin are created with 0700 permissions. MVT is a forensic tool, and what a plugin keeps there, such as API responses or sample metadata, is private by default. The path is resolved on every call, as the configuration folder already is, so it follows the current environment rather than whatever it was when MVT was imported. The documentation now points plugins at the helper, and the example settings class carries a plain integer setting in place of its cache folder. * Derive the data folder of a plugin from its settings class A plugin with a settings class already names itself in `plugin_name`; passing the name again to plugin_data_folder() repeats it and can drift. Add a `data_folder()` class method on MVTPluginSettings which returns plugin_data_folder() for the class's validated plugin name (works on the class and on an instance); plugin_data_folder(name) stays as the function underneath for plugins without a settings class. --------- Co-authored-by: Donncha Ó Cearbhaill --- docs/development/custom_commands.md | 4 + docs/development/index.md | 4 + docs/development/plugin_configuration.md | 149 +++++++++ mkdocs.yml | 1 + src/mvt/common/plugin_config.py | 299 +++++++++++++++++ tests/common/test_plugin_config.py | 391 +++++++++++++++++++++++ 6 files changed, 848 insertions(+) create mode 100644 docs/development/plugin_configuration.md create mode 100644 src/mvt/common/plugin_config.py create mode 100644 tests/common/test_plugin_config.py diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md index 764c70b..1d012f9 100644 --- a/docs/development/custom_commands.md +++ b/docs/development/custom_commands.md @@ -56,6 +56,10 @@ pipx inject mvt my-mvt-plugin When MVT is installed in an active virtual environment, install the plugin with `pip` in that environment. +Command packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + ## Load a Command File For local commands that are not packaged, create a Python file that exports one diff --git a/docs/development/index.md b/docs/development/index.md index 53a8f61..487127f 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -160,6 +160,10 @@ For a `pipx` installation of MVT, inject the package into MVT's environment: pipx inject mvt mvt-plugin-amnesty-custom ``` +Module packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + ### Naming module packages Name module packages `mvt-plugin-` (import package `mvt_plugin_`), diff --git a/docs/development/plugin_configuration.md b/docs/development/plugin_configuration.md new file mode 100644 index 0000000..e41f4e4 --- /dev/null +++ b/docs/development/plugin_configuration.md @@ -0,0 +1,149 @@ +# Plugin Configuration + +Plugin packages that add [custom CLI commands](custom_commands.md) or +[modules](index.md#installed-module-packages) often need to store their +own settings, such as an API key, a server URL or the timestamp of the last +synchronization. MVT provides a namespaced settings base class so each plugin +keeps its configuration in its own file, and a data folder for anything else a +plugin needs to keep on disk. + +!!! warning + + Do not write plugin settings to MVT's own `config.yaml`. MVT rewrites that + file with the settings it knows about every time it starts, so any other + section is deleted. + +## Where Settings Are Stored + +Each plugin gets one YAML file in a `plugins` folder next to MVT's own +configuration: + +``` +~/.config/mvt/plugins/.yaml +``` + +The exact parent folder follows the platform convention used for MVT's +`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use +`mvt.common.plugin_config.plugin_config_path()` instead of building the path by +hand. + +Plugin names must be lowercase and may only contain letters, digits and dashes, +matching the `mvt-plugin-` package naming convention. MVT creates the +`plugins` folder with `0700` permissions and writes the settings files with +`0600` permissions, because they commonly hold credentials. Files are written +through a temporary file and moved into place, so an interrupted save never +leaves a partially written settings file behind. + +## Plugin Data Folder + +Everything else a plugin keeps on disk, such as a cache, a downloaded artifact +or synchronization state, belongs in the folder returned by the `data_folder()` +class method of the plugin's settings class, or by +`mvt.common.plugin_config.plugin_data_folder()` called with the plugin name if +the plugin has no settings class: + +``` +~/.local/share/mvt/plugin-data// # Linux +~/Library/Application Support/mvt/plugin-data// # macOS +``` + +The folder sits beside MVT's own data, such as the downloaded indicators. It is +created if it is missing, with `0700` permissions. Asking for it again returns +the same path and leaves the contents alone, so a plugin can ask for it every +time it needs the folder. `ExamplePluginSettings` below is the settings class +defined in the next section: + +```python +import os + + +def cache_path() -> str: + folder = ExamplePluginSettings.data_folder() + return os.path.join(folder, "virustotal_lookups_cache.json") +``` + +A plugin which has no settings class calls +`plugin_data_folder("example-plugin")` instead. + +Do not fall back on a path of your own such as `~/.cache/example-plugin`: it +is a Linux-only convention, and MVT will not create it for you. + +## Defining Plugin Settings + +Subclass `MVTPluginSettings`, set `plugin_name` and declare typed fields with +defaults: + +```python +from typing import Optional + +from mvt.common.plugin_config import MVTPluginSettings + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + MAX_RESULTS: int = 25 + LAST_SYNC: Optional[str] = None +``` + +`load()` returns the current settings and `save()` writes them back: + +```python +from datetime import datetime, timezone + +import click + + +def sync(): + settings = ExamplePluginSettings.load() + if not settings.API_KEY: + raise click.ClickException( + "No API key configured. Set MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY or " + "run 'example-plugin configure'." + ) + + settings.LAST_SYNC = datetime.now(timezone.utc).isoformat() + settings.save() +``` + +A missing settings file is not an error: the plugin then runs on the field +defaults and on whatever the environment provides. `save()` only persists the +values that differ from the defaults, and it never touches MVT's `config.yaml`. +A settings file that cannot be parsed, or that does not hold a mapping of +setting names to values, raises a `PluginConfigLoadError` naming the file. + +Every subclass that sets its own `plugin_name` gets its own file and its own +environment namespace. A subclass that does not redefine `plugin_name` inherits +it, and therefore shares the file and the environment variables of its parent +class. + +## Environment Variables + +Every field can also be set with an environment variable. The prefix is +`MVT_PLUGIN_`, followed by the plugin name upper-cased with dashes replaced by +underscores, followed by the field name. For the example above: + +```bash +export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=... +export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50 +``` + +Settings resolve in this order, from highest to lowest priority: + +1. Arguments passed to the settings class directly, such as + `ExamplePluginSettings(API_KEY="...")` +2. Environment variables +3. The plugin's YAML file +4. The field defaults declared on the settings class + +!!! tip + + On shared or multi-user machines, prefer passing API keys through + environment variables rather than saving them to the plugin file. `save()` + skips every value that the environment currently supplies, so a credential + provided that way is not copied into the settings file when a plugin saves + an unrelated setting. + +Unknown keys in a plugin's YAML file are ignored, so a settings file written by +a newer version of a plugin does not break an older one. diff --git a/mkdocs.yml b/mkdocs.yml index 11a91dc..6250cdc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,4 +52,5 @@ nav: - Development: - Development Instructions: "development/index.md" - Custom CLI Commands: "development/custom_commands.md" + - Plugin Configuration: "development/plugin_configuration.md" - License: "license.md" diff --git a/src/mvt/common/plugin_config.py b/src/mvt/common/plugin_config.py new file mode 100644 index 0000000..f5e2099 --- /dev/null +++ b/src/mvt/common/plugin_config.py @@ -0,0 +1,299 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import contextlib +import json +import os +import re +import tempfile +from typing import Any, ClassVar, Dict, List, Tuple, Type, TypeVar + +import yaml +from appdirs import user_config_dir, user_data_dir +from pydantic import ValidationError +from pydantic_settings import ( + BaseSettings, + EnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +PLUGIN_CONFIG_FOLDER_NAME = "plugins" +# Not "plugins": on macOS the configuration and data folders are the same +# directory, and that name already holds the settings files. +PLUGIN_DATA_FOLDER_NAME = "plugin-data" +PLUGIN_ENV_PREFIX = "MVT_PLUGIN_" + +PLUGIN_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*") + +PluginSettingsType = TypeVar("PluginSettingsType", bound="MVTPluginSettings") + + +class PluginConfigLoadError(Exception): + pass + + +def validate_plugin_name(plugin_name: Any) -> str: + """ + Check that a plugin name is safe to use in a file name and an env variable. + + :param plugin_name: Name to validate. + :returns: The validated plugin name. + """ + if not isinstance(plugin_name, str): + raise TypeError( + f"Plugin name must be a string, not {type(plugin_name).__name__}" + ) + if not PLUGIN_NAME_PATTERN.fullmatch(plugin_name): + raise ValueError( + f"Invalid plugin name {plugin_name!r}: plugin names must start with a " + "lowercase letter or a digit and may only contain lowercase letters, " + "digits and dashes" + ) + return plugin_name + + +def plugin_config_folder() -> str: + """ + Return the folder where plugins store their configuration files. + + The path is resolved on every call so it always reflects the current + environment. + """ + return os.path.join(user_config_dir("mvt"), PLUGIN_CONFIG_FOLDER_NAME) + + +def plugin_config_path(plugin_name: str) -> str: + """ + Return the path of the configuration file of a given plugin. + + :param plugin_name: Name of the plugin. + """ + return os.path.join( + plugin_config_folder(), f"{validate_plugin_name(plugin_name)}.yaml" + ) + + +def plugin_data_folder(plugin_name: str) -> str: + """ + Return the folder where a given plugin stores its data, creating it. + + Plugins should keep whatever they persist, such as caches or downloaded + artifacts, in this folder. It is created with owner-only permissions. The + path is resolved on every call so it always reflects the current + environment. A plugin with a settings class calls + `MVTPluginSettings.data_folder()` instead, which passes `plugin_name` here. + + :param plugin_name: Name of the plugin. + :returns: The path of the data folder of the plugin. + """ + # Validate the name before anything is created, so an unsafe name cannot + # leave a folder behind. + name = validate_plugin_name(plugin_name) + + # makedirs() applies its mode only to the last folder of the path, so + # MVT's own data folder keeps the default permissions while the two + # plugin folders are private. + data_folder = os.path.join(user_data_dir("mvt"), PLUGIN_DATA_FOLDER_NAME) + os.makedirs(data_folder, mode=0o700, exist_ok=True) + + folder = os.path.join(data_folder, name) + os.makedirs(folder, mode=0o700, exist_ok=True) + return folder + + +def plugin_env_prefix(plugin_name: str) -> str: + """ + Return the environment variable prefix used by a given plugin. + + Dashes are replaced by underscores. Plugin names cannot contain underscores, + so two different plugin names never share an environment namespace. + + :param plugin_name: Name of the plugin. + """ + name = validate_plugin_name(plugin_name).upper().replace("-", "_") + return f"{PLUGIN_ENV_PREFIX}{name}_" + + +def _settings_plugin_name(settings_cls: Type[BaseSettings]) -> str: + plugin_name = getattr(settings_cls, "plugin_name", None) + if plugin_name is None: + raise TypeError( + f"{settings_cls.__name__} must set a 'plugin_name' class attribute to " + "namespace its configuration file and environment variables" + ) + return validate_plugin_name(plugin_name) + + +def _plugin_yaml_source( + settings_cls: Type[BaseSettings], config_path: str +) -> YamlConfigSettingsSource: + """ + Build the YAML settings source of a plugin, reporting unusable files. + + A missing file is not an error, but a file which cannot be parsed or which + does not hold a mapping of setting names is reported with its path. + """ + try: + return YamlConfigSettingsSource(settings_cls, config_path) + except yaml.YAMLError as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: {exc}" + ) from exc + except (TypeError, ValueError) as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: the file must " + "contain a mapping of setting names to values" + ) from exc + + +class MVTPluginSettings(BaseSettings): + """ + Base class for plugin-namespaced settings. + + Subclass with typed fields and set `plugin_name`. Values resolve from + constructor arguments, then environment variables (MVT_PLUGIN__*), + then the plugin's YAML file (~/.config/mvt/plugins/.yaml), then field + defaults. + + Plugins must not store their settings in MVT's own configuration file: MVT + rewrites it with the fields it knows about, dropping anything else. + + `data_folder()` returns the folder the plugin keeps its data in. + """ + + model_config = SettingsConfigDict(extra="ignore") + + plugin_name: ClassVar[str] + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + plugin_name = _settings_plugin_name(cls) + # Namespace the environment variables of this plugin. Each pydantic + # model gets its own configuration dictionary, so this does not leak + # into other plugins. + cls.model_config["env_prefix"] = plugin_env_prefix(plugin_name) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + config_path = plugin_config_path(_settings_plugin_name(settings_cls)) + yaml_source = _plugin_yaml_source(settings_cls, config_path) + # Explicit arguments take precedence over environment variables, which + # in turn take precedence over the configuration file. + return (init_settings, env_settings, yaml_source) + + @classmethod + def load(cls: Type[PluginSettingsType]) -> PluginSettingsType: + """ + Load the settings of the plugin. + + A missing configuration file is not an error: the settings then come + from the environment and from the field defaults. + """ + return cls() + + @classmethod + def data_folder(cls) -> str: + """ + Return the data folder of the plugin, creating it. + + The folder is the one plugin_data_folder() returns for `plugin_name`, + so a plugin with a settings class does not repeat its name. + """ + return plugin_data_folder(_settings_plugin_name(cls)) + + def _environment_values(self) -> Dict[str, Any]: + """ + Return the settings values currently supplied by the environment. + + Values are validated by the model, so they can be compared with the + values held by this instance. + """ + settings_cls = type(self) + raw_values = EnvSettingsSource(settings_cls)() + names = [name for name in raw_values if name in settings_cls.model_fields] + if not names: + return {} + + # Fall back on the current values for the fields the environment does + # not set, so that required fields do not fail validation here. + current_values = json.loads(self.model_dump_json()) + try: + from_environment = settings_cls.model_validate( + {**current_values, **raw_values} + ) + except ValidationError: + # An environment variable which the model cannot validate must not + # stop the other environment values from being recognised, or a + # credential would be written to the configuration file. + return self._environment_values_by_field(current_values, raw_values, names) + return {name: getattr(from_environment, name) for name in names} + + def _environment_values_by_field( + self, + current_values: Dict[str, Any], + raw_values: Dict[str, Any], + names: List[str], + ) -> Dict[str, Any]: + """ + Validate each environment value on its own, skipping unusable ones. + + :param current_values: Serialized values held by this instance. + :param raw_values: Values supplied by the environment. + :param names: Names of the fields set by the environment. + """ + settings_cls = type(self) + values = {} + for name in names: + try: + from_environment = settings_cls.model_validate( + {**current_values, name: raw_values[name]} + ) + except ValidationError: + continue + values[name] = getattr(from_environment, name) + return values + + def save(self) -> None: + """ + Save the current settings to the configuration file of the plugin. + + Only values which differ from the field defaults are persisted. Values + which come from the environment are not written to disk, so credentials + passed as environment variables stay out of the configuration file. + MVT's own configuration file is never modified. + """ + config_folder = plugin_config_folder() + if not os.path.isdir(config_folder): + os.makedirs(config_folder, mode=0o700, exist_ok=True) + + values = json.loads(self.model_dump_json(exclude_defaults=True)) + for name, environment_value in self._environment_values().items(): + if name in values and getattr(self, name, None) == environment_value: + del values[name] + + # Settings files can hold credentials, so write them through a private + # temporary file and move it in place. The file is then never partially + # written and never briefly readable by other users. + config_path = plugin_config_path(self.plugin_name) + descriptor, temporary_path = tempfile.mkstemp( + dir=config_folder, prefix=f".{self.plugin_name}-", suffix=".yaml" + ) + try: + with os.fdopen(descriptor, "w") as config_file: + config_file.write(yaml.dump(values, default_flow_style=False)) + os.replace(temporary_path, config_path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(temporary_path) + raise diff --git a/tests/common/test_plugin_config.py b/tests/common/test_plugin_config.py new file mode 100644 index 0000000..9bcf9eb --- /dev/null +++ b/tests/common/test_plugin_config.py @@ -0,0 +1,391 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import stat +import sys +from typing import Optional + +import pytest +import yaml + +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_folder, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + CACHE_FOLDER: str = "cache" + MAX_RESULTS: int = 25 + + +class OtherPluginSettings(MVTPluginSettings): + plugin_name = "other-plugin" + + API_KEY: Optional[str] = None + + +@pytest.fixture +def config_folder(tmp_path, monkeypatch): + monkeypatch.setattr( + "mvt.common.plugin_config.user_config_dir", + lambda *args, **kwargs: str(tmp_path), + ) + return tmp_path + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "data" + monkeypatch.setattr( + "mvt.common.plugin_config.user_data_dir", + lambda *args, **kwargs: str(folder), + ) + return folder + + +def _write_plugin_file(plugin_name, values): + config_path = plugin_config_path(plugin_name) + os.makedirs(os.path.dirname(config_path), exist_ok=True) + content = values if isinstance(values, str) else yaml.dump(values) + with open(config_path, "w") as config_file: + config_file.write(content) + return config_path + + +def test_plugin_paths_and_prefixes_are_namespaced(config_folder): + assert plugin_config_folder() == str(config_folder / "plugins") + assert plugin_config_path("example-plugin") == str( + config_folder / "plugins" / "example-plugin.yaml" + ) + assert plugin_env_prefix("example-plugin") == "MVT_PLUGIN_EXAMPLE_PLUGIN_" + assert plugin_env_prefix("other-plugin") == "MVT_PLUGIN_OTHER_PLUGIN_" + + +def test_defaults_are_used_without_file_or_environment(config_folder): + settings = ExamplePluginSettings.load() + + assert settings.API_KEY is None + assert settings.CACHE_FOLDER == "cache" + assert settings.MAX_RESULTS == 25 + assert not os.path.exists(plugin_config_path("example-plugin")) + + +def test_values_are_loaded_from_the_plugin_file(config_folder): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert settings.MAX_RESULTS == 5 + assert settings.CACHE_FOLDER == "cache" + + +def test_environment_overrides_the_plugin_file(config_folder, monkeypatch): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-environment" + assert settings.MAX_RESULTS == 5 + + +def test_arguments_override_the_environment_and_the_plugin_file( + config_folder, monkeypatch +): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "10") + + settings = ExamplePluginSettings(API_KEY="from-argument") + + assert settings.API_KEY == "from-argument" + assert settings.MAX_RESULTS == 10 + + +def test_save_and_load_round_trip(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + settings.MAX_RESULTS = 100 + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert os.path.isfile(config_path) + with open(config_path) as config_file: + assert yaml.safe_load(config_file) == { + "API_KEY": "saved-key", + "MAX_RESULTS": 100, + } + + reloaded = ExamplePluginSettings.load() + assert reloaded.API_KEY == "saved-key" + assert reloaded.MAX_RESULTS == 100 + assert reloaded.CACHE_FOLDER == "cache" + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_saved_file_is_only_readable_by_the_user(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + folder_mode = stat.S_IMODE(os.stat(plugin_config_folder()).st_mode) + assert folder_mode & 0o077 == 0 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_save_restricts_the_permissions_of_an_existing_file(config_folder): + config_path = _write_plugin_file("example-plugin", {"API_KEY": "from-file"}) + os.chmod(config_path, 0o644) + + settings = ExamplePluginSettings.load() + settings.MAX_RESULTS = 100 + settings.save() + + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + assert os.listdir(plugin_config_folder()) == ["example-plugin.yaml"] + + +def test_save_only_persists_non_default_values(config_folder): + settings = ExamplePluginSettings.load() + settings.CACHE_FOLDER = "another-cache" + + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"CACHE_FOLDER": "another-cache"} + + +def test_save_does_not_persist_values_coming_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + assert settings.API_KEY == "environment-secret" + settings.MAX_RESULTS = 100 + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"MAX_RESULTS": 100} + + +def test_an_invalid_environment_variable_still_protects_the_other_values( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "not-an-int") + + settings = ExamplePluginSettings(MAX_RESULTS=100) + assert settings.API_KEY == "environment-secret" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + saved_values = yaml.safe_load(config_file) + assert saved_values == {"MAX_RESULTS": 100} + assert "API_KEY" not in saved_values + + +def test_save_persists_values_which_differ_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + settings.API_KEY = "chosen-key" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "chosen-key"} + + +def test_save_does_not_write_the_mvt_configuration_file(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + assert os.listdir(config_folder) == ["plugins"] + + +def test_unknown_keys_in_the_plugin_file_are_ignored(config_folder): + _write_plugin_file( + "example-plugin", + {"API_KEY": "from-file", "UNKNOWN_SETTING": "ignored"}, + ) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert not hasattr(settings, "UNKNOWN_SETTING") + + +def test_unparsable_plugin_file_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "API_KEY: [unclosed\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + + +def test_plugin_file_which_is_not_a_mapping_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "- one\n- two\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + assert "mapping of setting names" in str(raised.value) + + +def test_plugins_do_not_interfere_with_each_other(config_folder, monkeypatch): + _write_plugin_file("other-plugin", {"API_KEY": "other-file-key"}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "example-environment-key") + + example_settings = ExamplePluginSettings.load() + other_settings = OtherPluginSettings.load() + + assert example_settings.API_KEY == "example-environment-key" + assert other_settings.API_KEY == "other-file-key" + + example_settings.MAX_RESULTS = 100 + example_settings.save() + assert sorted(os.listdir(plugin_config_folder())) == [ + "example-plugin.yaml", + "other-plugin.yaml", + ] + with open(plugin_config_path("other-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "other-file-key"} + + +def test_subclass_without_plugin_name_is_rejected(): + with pytest.raises(TypeError, match="plugin_name"): + + class MissingNameSettings(MVTPluginSettings): + API_KEY: Optional[str] = None + + +def test_subclass_with_invalid_plugin_name_is_rejected(): + with pytest.raises(ValueError, match="Invalid plugin name"): + + class InvalidNameSettings(MVTPluginSettings): + plugin_name = "Bad/Name" + + +def test_underscores_are_not_allowed_in_plugin_names(): + # Underscores are replaced by dashes in the environment prefix, so allowing + # both would let two plugin names share one environment namespace. + with pytest.raises(ValueError, match="Invalid plugin name"): + + class UnderscoreNameSettings(MVTPluginSettings): + plugin_name = "under_score" + + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path("under_score") + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_env_prefix("under_score") + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_configuration_path(plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path(plugin_name) + + +def test_data_folder_is_namespaced_and_created(data_folder): + folder = plugin_data_folder("example-plugin") + + assert folder == str(data_folder / "plugin-data" / "example-plugin") + assert os.path.isdir(folder) + + +def test_data_folder_can_be_requested_repeatedly(data_folder): + folder = plugin_data_folder("example-plugin") + with open(os.path.join(folder, "kept.json"), "w") as data_file: + data_file.write("{}") + + assert plugin_data_folder("example-plugin") == folder + assert os.listdir(folder) == ["kept.json"] + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_data_folder_is_only_accessible_by_the_user(data_folder): + folder = plugin_data_folder("example-plugin") + + assert stat.S_IMODE(os.stat(folder).st_mode) & 0o077 == 0 + parent_mode = stat.S_IMODE(os.stat(os.path.dirname(folder)).st_mode) + assert parent_mode & 0o077 == 0 + + +def test_plugins_get_their_own_data_folder(data_folder): + example_folder = plugin_data_folder("example-plugin") + other_folder = plugin_data_folder("other-plugin") + + assert example_folder != other_folder + assert sorted(os.listdir(data_folder / "plugin-data")) == [ + "example-plugin", + "other-plugin", + ] + + +def test_data_folder_does_not_touch_the_configuration_folder( + config_folder, data_folder +): + plugin_data_folder("example-plugin") + + assert not os.path.exists(plugin_config_folder()) + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_data_folder(data_folder, plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_data_folder(plugin_name) + + assert not os.path.exists(data_folder) + + +def test_settings_class_knows_its_data_folder(data_folder): + folder = ExamplePluginSettings.data_folder() + + assert folder == plugin_data_folder("example-plugin") + assert os.path.isdir(folder) + assert OtherPluginSettings.data_folder() != folder + + +def test_settings_instance_uses_the_same_data_folder(config_folder, data_folder): + settings = ExamplePluginSettings.load() + + assert settings.data_folder() == ExamplePluginSettings.data_folder() + + +def test_subclass_without_its_own_name_shares_the_data_folder(data_folder): + class InheritingSettings(ExamplePluginSettings): + pass + + assert InheritingSettings.data_folder() == ExamplePluginSettings.data_folder() From 104ffb167f60a08e4aa7802442295060390f1d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:14 +0200 Subject: [PATCH 30/43] Skip modules with unavailable dependencies instead of aborting the run (#895) A module declaring a dependency its command does not provide made _ordered_modules() give up on the whole run, so a single wrong declaration in a module scoped to several commands turned a forensic analysis into zero executed modules with one warning to explain it. Drop only the modules that cannot run: the one with the unavailable dependency, and anything depending on it. Each gets its own warning naming the module missing a dependency and the dependency it is missing, and the remaining modules run in the same stable topological order as before. A cycle in the dependency graph is still a programming error and still stops the run. --- docs/development/index.md | 13 +++- src/mvt/common/command.py | 111 ++++++++++++++++++++++++++++++----- tests/common/test_command.py | 91 ++++++++++++++++++++++++++-- 3 files changed, 192 insertions(+), 23 deletions(-) diff --git a/docs/development/index.md b/docs/development/index.md index 487127f..c2b8464 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -35,9 +35,16 @@ class DependentModule(MVTModule): prerequisite_results = self.get_dependency_results(PrerequisiteModule) ``` -Selecting a single module also runs its transitive dependencies. If a dependency -is unavailable or the dependency graph contains a cycle, the command logs a -warning and does not run any modules. +Selecting a single module also runs its transitive dependencies. + +A module can only depend on modules the command it runs in also has. When a +declared dependency is not among them, the command logs a warning naming the +module and the missing dependency, skips that module and everything depending +on it, and runs the rest of the analysis. Selecting such a module with +`--module` therefore leaves nothing to run, which the warning explains. + +A cycle in the dependency graph is a programming error rather than a +configuration problem: the command logs a warning and runs no modules at all. ## Custom modules diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index 6111dc7..f2f211d 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -317,6 +317,81 @@ class Command: console.print("") console.print(panel) + def _skipped_modules( + self, + required: list[type[MVTModule]], + module_indexes: dict[type[MVTModule], int], + ) -> dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]]: + """Return the modules to drop because a dependency is unavailable. + + A module declaring a dependency this command cannot provide is unable + to run, and so is every module depending on it. Dropping only those + keeps a single wrong declaration - in a module scoped to several + commands, for example - from silencing an entire analysis. + + The returned mapping gives, for each skipped module, the module which + is missing a dependency and the dependency it is missing. + """ + skipped: dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]] = {} + # Skipping the one module a run was asked for leaves nothing to run, + # which the caller reports instead. + remainder = ( + "" if self.module_name else " The rest of the analysis will still run." + ) + + # Skipping one module can skip the modules depending on it, which the + # pass over the module list may already have gone past, so repeat the + # pass until nothing changes. + changed = True + while changed: + changed = False + for module in required: + if module in skipped: + continue + + for dependency in module.dependencies: + if dependency not in module_indexes: + skipped[module] = (module, dependency) + changed = True + self.log.warning( + "Module %s will be SKIPPED: it depends on module " + "%s, which is not available in this command.%s", + module.__name__, + dependency.__name__, + remainder, + ) + break + + if dependency in skipped: + root, missing = skipped[dependency] + skipped[module] = (root, missing) + changed = True + if dependency is root: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "module %s, itself skipped for depending on " + "unavailable module %s.%s", + module.__name__, + dependency.__name__, + missing.__name__, + remainder, + ) + else: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "skipped module %s, in a chain starting at " + "module %s, which depends on unavailable " + "module %s.%s", + module.__name__, + dependency.__name__, + root.__name__, + missing.__name__, + remainder, + ) + break + + return skipped + def _ordered_modules(self) -> Optional[list[type[MVTModule]]]: """Return enabled modules in stable topological order.""" modules = self._available_modules() @@ -329,30 +404,34 @@ class Command: else: selected = [module for module in modules if module.enabled] - required = set(selected) + required: set[type[MVTModule]] = set() pending = list(selected) while pending: module = pending.pop() + if module in required: + continue + required.add(module) for dependency in module.dependencies: - if dependency not in module_indexes: - self.log.warning( - "Module %s depends on unavailable module %s. " - "No modules will be run.", - module.__name__, - dependency.__name__, - ) - return None - if dependency not in required: - required.add(dependency) + # Unavailable dependencies are reported by _skipped_modules(). + if dependency in module_indexes: pending.append(dependency) + ordered_required = sorted(required, key=lambda module: module_indexes[module]) + skipped = self._skipped_modules(ordered_required, module_indexes) + runnable = [module for module in ordered_required if module not in skipped] + if skipped and not runnable: + self.log.warning( + "Every selected module was skipped for an unavailable " + "dependency. No modules will be run." + ) + dependents: dict[type[MVTModule], list[type[MVTModule]]] = { - module: [] for module in required + module: [] for module in runnable } - indegree = {module: 0 for module in required} - for module in required: + indegree = {module: 0 for module in runnable} + for module in runnable: for dependency in module.dependencies: - if dependency not in required: + if dependency not in indegree: continue dependents[dependency].append(module) indegree[module] += 1 @@ -371,7 +450,7 @@ class Command: if indegree[dependent] == 0: heappush(ready, (module_indexes[dependent], dependent)) - if len(ordered) != len(required): + if len(ordered) != len(runnable): cyclic_modules = sorted( (module.__name__ for module, count in indegree.items() if count > 0) ) diff --git a/tests/common/test_command.py b/tests/common/test_command.py index 4dbfe1a..4835fd3 100644 --- a/tests/common/test_command.py +++ b/tests/common/test_command.py @@ -157,7 +157,7 @@ class TestCommand: assert not hasattr(cmd, "initialized") assert "Circular module dependency detected" in caplog.text - def test_unavailable_dependency_warns_and_stops(self, caplog): + def test_unavailable_dependency_only_skips_the_dependent_module(self, caplog): class UnavailableModule(RecordingModule): pass @@ -165,14 +165,97 @@ class TestCommand: dependencies = (UnavailableModule,) cmd = RecordingCommand() - cmd.modules = [DependentModule] + cmd.modules = [DependentModule, IndependentModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "FirstModule"] + assert cmd.initialized + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "depends on module UnavailableModule" in caplog.text + + def test_modules_depending_on_a_skipped_module_are_skipped_too(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + class DependsOnSkippedModule(RecordingModule): + dependencies = (SkippedModule,) + + class DependsOnTheChain(RecordingModule): + dependencies = (DependsOnSkippedModule,) + + cmd = RecordingCommand() + cmd.modules = [ + DependsOnTheChain, + DependsOnSkippedModule, + SkippedModule, + IndependentModule, + ] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule"] + skip_warnings = [ + record.getMessage() + for record in caplog.records + if "will be SKIPPED" in record.getMessage() + ] + assert len(skip_warnings) == 3 + assert [warning.split()[1] for warning in skip_warnings] == [ + "SkippedModule", + "DependsOnSkippedModule", + "DependsOnTheChain", + ] + # Every warning names the root cause: the module missing a dependency + # and the dependency it is missing. + assert all("UnavailableModule" in warning for warning in skip_warnings) + assert all("module SkippedModule" in warning for warning in skip_warnings[1:]) + + def test_explicitly_selected_module_with_missing_dependency_runs_nothing( + self, caplog + ): + class UnavailableModule(RecordingModule): + pass + + class DependentModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand(module_name="DependentModule") + cmd.modules = [DependentModule, IndependentModule] with caplog.at_level(logging.WARNING): cmd.run() assert RecordingModule.run_order == [] - assert not hasattr(cmd, "initialized") - assert "depends on unavailable module UnavailableModule" in caplog.text + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warning must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + + def test_unaffected_dependency_chains_keep_their_order(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand() + cmd.modules = [ThirdModule, SkippedModule, SecondModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "FirstModule", + "SecondModule", + "ThirdModule", + ] def test_custom_modules_are_filtered_before_ordering(self): cmd = RecordingCommand() From 24645cb718dad02efa2477cc22e13b16cbb9d81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:14 +0200 Subject: [PATCH 31/43] Register installed CLI plugins at program start (#896) register_cli_plugins() ran while mvt.ios and mvt.android were being imported, so importing any part of MVT executed the entry points of every installed command package. That made plugin loading depend on import order: a plugin importing from MVT while MVT was still initializing got an ImportError and was quietly demoted to a broken command, and the same plugin worked when MVT happened to be imported first. Move the call into a main() function in each CLI module and point the console scripts at it, so registration happens once when the program starts and importing MVT no longer runs third-party code. For packagers: mvt.ios:cli and mvt.android:cli stay importable, but a wrapper invoking cli() directly no longer registers the installed plugin commands and should call main() instead. --- pyproject.toml | 4 +- src/mvt/android/__init__.py | 2 +- src/mvt/android/cli.py | 21 +++- src/mvt/ios/__init__.py | 2 +- src/mvt/ios/cli.py | 21 +++- tests/plugin_fixtures.py | 85 +++++++++++++ tests/test_cli_entry_points.py | 215 +++++++++++++++++++++++++++++++++ 7 files changed, 336 insertions(+), 14 deletions(-) create mode 100644 tests/plugin_fixtures.py create mode 100644 tests/test_cli_entry_points.py diff --git a/pyproject.toml b/pyproject.toml index 151d4e7..ad92423 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,8 @@ homepage = "https://docs.mvt.re/en/latest/" repository = "https://github.com/mvt-project/mvt" [project.scripts] -mvt-ios = "mvt.ios:cli" -mvt-android = "mvt.android:cli" +mvt-ios = "mvt.ios:main" +mvt-android = "mvt.android:main" [dependency-groups] dev = [ diff --git a/src/mvt/android/__init__.py b/src/mvt/android/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/android/__init__.py +++ b/src/mvt/android/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index ca7e3dc..cabd083 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -514,8 +514,19 @@ def download_indicators(): ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=ANDROID_CLI_PLUGIN_GROUP, - environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-android console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-android CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/ios/__init__.py b/src/mvt/ios/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/ios/__init__.py +++ b/src/mvt/ios/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index c338fa5..52a1c29 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -532,8 +532,19 @@ def download_iocs(): ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=IOS_CLI_PLUGIN_GROUP, - environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-ios console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-ios CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/tests/plugin_fixtures.py b/tests/plugin_fixtures.py new file mode 100644 index 0000000..ab3359b --- /dev/null +++ b/tests/plugin_fixtures.py @@ -0,0 +1,85 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Helpers building throwaway plugin distributions for the tests. + +Some plugin behaviour only shows up in a fresh interpreter: what an import +executes, and what a plugin sees when MVT is imported before or after it. +These helpers write an importable distribution with a real entry point and +run a script against it in a subprocess, with a temporary home so that the +subprocess cannot touch the configuration of whoever runs the tests. +""" + +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional + +FIXTURE_COMMAND_NAME = "fixture-plugin" +FIXTURE_MODULE_NAME = "fixture_cli_plugin" +FIXTURE_DISTRIBUTION_NAME = "fixture-cli-plugin" + + +def write_cli_plugin_distribution( + site_path: Path, + entry_point_group: str, + module_source: str, +) -> Path: + """Write a distribution registering a CLI plugin entry point. + + :param site_path: Folder to write the distribution into, to be added to + the import path of the interpreter loading it. + :param entry_point_group: Entry-point group to register the command in. + :param module_source: Source of the plugin module, which must define a + Click command named `cli`. + :returns: The folder the distribution was written to. + """ + site_path.mkdir(parents=True, exist_ok=True) + (site_path / f"{FIXTURE_MODULE_NAME}.py").write_text( + module_source, encoding="utf-8" + ) + + dist_info = ( + site_path / f"{FIXTURE_DISTRIBUTION_NAME.replace('-', '_')}-1.0.dist-info" + ) + dist_info.mkdir(exist_ok=True) + (dist_info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {FIXTURE_DISTRIBUTION_NAME}\nVersion: 1.0\n", + encoding="utf-8", + ) + (dist_info / "entry_points.txt").write_text( + f"[{entry_point_group}]\n{FIXTURE_COMMAND_NAME} = {FIXTURE_MODULE_NAME}:cli\n", + encoding="utf-8", + ) + return site_path + + +def run_isolated_python( + script: str, + home: Path, + site_path: Optional[Path] = None, + **environment: str, +) -> subprocess.CompletedProcess: + """Run a script in a fresh interpreter with its own configuration folder. + + Importing MVT writes its configuration file, so the subprocess gets a + temporary home and no MVT environment variables from the test session. + """ + isolated_environment = { + key: value for key, value in os.environ.items() if not key.startswith("MVT_") + } + isolated_environment["HOME"] = str(home) + isolated_environment["XDG_CONFIG_HOME"] = str(home / "config") + if site_path is not None: + isolated_environment["PYTHONPATH"] = str(site_path) + isolated_environment.update(environment) + + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=isolated_environment, + ) diff --git a/tests/test_cli_entry_points.py b/tests/test_cli_entry_points.py new file mode 100644 index 0000000..f5834ac --- /dev/null +++ b/tests/test_cli_entry_points.py @@ -0,0 +1,215 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import sys +from types import SimpleNamespace + +import click +import pytest + +import mvt.android +import mvt.ios +from mvt.android.cli import cli as android_cli +from mvt.android.cli import main as android_main +from mvt.common.cli_plugins import ANDROID_CLI_PLUGIN_GROUP, IOS_CLI_PLUGIN_GROUP +from mvt.ios.cli import cli as ios_cli +from mvt.ios.cli import main as ios_main + +from .plugin_fixtures import ( + FIXTURE_COMMAND_NAME, + run_isolated_python, + write_cli_plugin_distribution, +) + +MARKER_PLUGIN_TEMPLATE = """ +import os + +import click + +# Touched when this module is imported, so a test can tell whether loading MVT +# executed the plugin. +open(os.environ["FIXTURE_PLUGIN_MARKER"], "a").close() + + +@click.command() +def cli(): + click.echo("fixture plugin ran") +""" + +PROGRAMS = { + "mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP), + "mvt-android": (mvt.android, android_cli, ANDROID_CLI_PLUGIN_GROUP), +} + + +@pytest.fixture +def restore_cli_commands(): + """Undo the plugin registration main() performs on the shared CLI groups.""" + originals = { + program: dict(group.commands) for program, (_, group, _) in PROGRAMS.items() + } + yield + for program, (_, group, _) in PROGRAMS.items(): + group.commands.clear() + group.commands.update(originals[program]) + if hasattr(group, "_mvt_external_command_sources"): + delattr(group, "_mvt_external_command_sources") + + +def _install_fixture_entry_point(monkeypatch, entry_point_group, command): + def entry_points(*, group): + if group != entry_point_group: + return [] + return [ + SimpleNamespace( + name=FIXTURE_COMMAND_NAME, + value="fixture_cli_plugin:cli", + load=lambda: command, + dist=SimpleNamespace( + metadata={"Name": "fixture-cli-plugin"}, version="1.0" + ), + ) + ] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", entry_points + ) + + +def _offline_argv(program, *arguments): + """Build an argument list which keeps the CLI from checking for updates.""" + return [ + program, + "--disable-update-check", + "--disable-indicator-update-check", + *arguments, + ] + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_registers_installed_plugins_before_running_the_cli( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, entry_point_group = PROGRAMS[program] + + @click.command() + def fixture_command(): + click.echo("fixture plugin ran") + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + monkeypatch.setattr(sys, "argv", _offline_argv(program, FIXTURE_COMMAND_NAME)) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "fixture plugin ran" in capsys.readouterr().out + assert FIXTURE_COMMAND_NAME in group.commands + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_completes_plugin_command_names( + program, monkeypatch, capsys, restore_cli_commands +): + package, _, entry_point_group = PROGRAMS[program] + + @click.command() + def fixture_command(): + pass + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + complete_variable = f"_{program.upper().replace('-', '_')}_COMPLETE" + monkeypatch.setenv(complete_variable, "bash_complete") + monkeypatch.setenv("COMP_WORDS", f"{program} fixture") + monkeypatch.setenv("COMP_CWORD", "1") + monkeypatch.setattr(sys, "argv", [program]) + + with pytest.raises(SystemExit): + package.main() + + assert f"plain,{FIXTURE_COMMAND_NAME}" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_still_loads_commands_from_a_file( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + package, _, entry_point_group = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text( + "import click\n" + "\n" + "\n" + '@click.command("case-summary")\n' + "def cli():\n" + ' click.echo("case summary ran")\n', + encoding="utf-8", + ) + _install_fixture_entry_point( + monkeypatch, entry_point_group, click.Command("unused") + ) + monkeypatch.setattr( + sys, + "argv", + _offline_argv(program, "--load-command", str(command_path), "case-summary"), + ) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +def test_the_console_script_targets_are_importable(): + # [project.scripts] points at these, so they must stay on the packages. + assert mvt.ios.main is ios_main + assert mvt.android.main is android_main + + +def test_importing_mvt_does_not_run_installed_plugins(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import mvt.ios.cli\nimport mvt.android.cli\nprint('imported')", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "imported" in result.stdout + assert not marker.exists() + + +def test_registering_the_plugins_runs_the_entry_point(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import click\n" + "from mvt.common.cli_plugins import (\n" + " IOS_CLI_PLUGIN_GROUP,\n" + " BrokenPluginCommand,\n" + " register_installed_cli_commands,\n" + ")\n" + "group = click.Group()\n" + "register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)\n" + f"command = group.commands[{FIXTURE_COMMAND_NAME!r}]\n" + "assert not isinstance(command, BrokenPluginCommand), command.help\n" + "print('registered')\n", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "registered" in result.stdout + assert marker.exists() From a78894aaa5c897ad2154435412e7f759e9de57b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:15 +0200 Subject: [PATCH 32/43] Add a platform-neutral mvt command (#897) * Add a platform-neutral mvt command Several MVT commands have nothing to do with the acquisition of one platform, yet they were reachable only through mvt-ios and mvt-android. Asking which version is installed or downloading the public indicators meant picking one of the two platform commands arbitrarily, and each of those tasks had to be written, documented and maintained twice. Add a third console script, mvt, hosting the commands which belong to no platform: version and download-iocs for now, with completion following in a later commit. Commands installed in the new mvt.cli_plugins entry-point group are registered on mvt, and on mvt only, so that a command package chooses the CLI each of its commands is added to: mvt.ios.cli_plugins for mvt-ios, mvt.android.cli_plugins for mvt-android and mvt.cli_plugins for mvt. A command wanted on both platform CLIs is registered in both platform groups; no group adds a command to every CLI. The MVT_CUSTOM_COMMANDS variable loads command files and folders into mvt the way the platform variables already do for mvt-ios and mvt-android. Run on its own, mvt prints the banner and its help instead of a usage error. The help text reminds that the forensic analysis of an acquisition runs through mvt-ios and mvt-android, so that the command which knows nothing about acquisitions says where they are analysed. version and download-iocs stay on mvt-ios and mvt-android for now, so that no documented invocation stops working. They are to be dropped from the platform CLIs in a later release, once mvt has been available long enough for the change to be announced. Unlike mvt-ios and mvt-android, which point at their subpackages, the console script points at mvt.cli:main and the mvt package re-exports nothing of it. Importing mvt has to stay cheap and free of side effects: it is the package plugins import from, and pulling in Click, the CLI and everything the commands import merely because something imported mvt would work against that. While here, give the version command of both platform CLIs the context settings every other command already has, so that "mvt-ios version -h" prints its help instead of failing on an unknown option. * Reduce indent for MVT CLI header * Move shell completion to the mvt command and generate one script for every MVT command Setting up shell completion had nothing to do with the acquisition of one platform, yet it was a command of mvt-ios and mvt-android, each generating the script of the program it ran under only. Completion now leaves the platform CLIs for mvt, which emits one script covering mvt, mvt-ios and mvt-android, installed as a single file loaded when the shell starts. Click names the completion function of each program after the program, so the three scripts concatenate without colliding, and fish takes the file in conf.d rather than one named after a single command. With one script there is nothing left for the platform commands to generate, so their completion command goes, and with it the banner suppression which was keyed on the command name: mvt-ios and mvt-android now print the banner for every command they have. The long help line says which commands the completion covers, so a short_help keeps "mvt --help" from truncating it. * Point the README and the command docs at plugin packages Loading a command by path is for local development; a package is how a command is distributed. The README's summary of what extends MVT now names plugin packages and nothing else, and the section on loading a command file says what it is for: keeping a command being written loadable without reinstalling its package after every change. --- README.md | 25 ++-- docs/command_completion.md | 44 +++---- docs/development/custom_commands.md | 60 +++++++-- docs/index.md | 2 +- docs/install.md | 6 +- docs/iocs.md | 2 +- pyproject.toml | 1 + src/mvt/android/cli.py | 52 +------- src/mvt/cli.py | 102 +++++++++++++++ src/mvt/common/cli_plugins.py | 13 ++ src/mvt/common/completion.py | 121 ++++++++++++----- src/mvt/common/help.py | 4 +- src/mvt/common/logo.py | 20 +-- src/mvt/ios/cli.py | 52 +------- tests/common/test_cli_plugins.py | 196 +++++++++++++++++++++++++++- tests/conftest.py | 35 +++++ tests/test_cli.py | 49 +++++++ tests/test_cli_entry_points.py | 121 +++++++++++++---- tests/test_completion.py | 66 ++++++---- 19 files changed, 727 insertions(+), 244 deletions(-) create mode 100644 src/mvt/cli.py create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index d4ac9b4..5f00600 100644 --- a/README.md +++ b/README.md @@ -58,34 +58,31 @@ For alternative installation options and known issues, please refer to the [docu ## Usage -MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) ### Shell completion -MVT can generate shell completion scripts for Bash, Zsh, and Fish: +MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`: ```bash -mvt-ios completion -mvt-android completion +mvt completion ``` -The commands print setup instructions by default. To generate a completion script directly, pass the shell name: +The command prints setup instructions by default. To generate the completion script directly, pass the shell name: ```bash -mvt-ios completion bash -mvt-android completion zsh +mvt completion bash ``` MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details. -Module-running `check-*` commands can load custom Python modules with -`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the -[development documentation](https://docs.mvt.re/en/latest/development/) for -details. -Users can also add top-level commands to `mvt-ios` and `mvt-android` from -installed Python packages or local files and folders. See the +Plugin packages extend MVT with additional forensic modules, which run inside +the `check-*` commands, and with top-level commands on `mvt`, `mvt-ios` and +`mvt-android`. See the +[development documentation](https://docs.mvt.re/en/latest/development/) for +writing and installing them, and the [custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/) -for the plugin entry points and `--load-command` interface. +for the entry points a package registers commands in. ## License diff --git a/docs/command_completion.md b/docs/command_completion.md index 7204df5..a51cc7a 100644 --- a/docs/command_completion.md +++ b/docs/command_completion.md @@ -6,61 +6,61 @@ Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fi To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using. -The following describes how to generate the command completion scripts and add them to your shell configuration. +`mvt completion` generates one script which covers `mvt`, `mvt-ios` and `mvt-android`. The following describes how to generate that script and add it to your shell configuration. > **Note: You will need to start a new shell for the changes to take effect.** ### For Bash ```bash -# Generate bash completion scripts -mvt-ios completion bash > ~/.mvt-ios-complete.bash -mvt-android completion bash > ~/.mvt-android-complete.bash +# Generate the bash completion script +mvt completion bash > ~/.mvt-complete.bash ``` Add the following to `~/.bashrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash -[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash +# source the mvt completion script +[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash ``` ### For Zsh ```bash -# Generate zsh completion scripts -mvt-ios completion zsh > ~/.mvt-ios-complete.zsh -mvt-android completion zsh > ~/.mvt-android-complete.zsh +# Generate the zsh completion script +mvt completion zsh > ~/.mvt-complete.zsh ``` Add the following to `~/.zshrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh -[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh +# source the mvt completion script +[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh ``` ### For Fish ```bash -# Generate fish completion scripts -mkdir -p ~/.config/fish/completions -mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish -mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish +# Generate the fish completion script +mkdir -p ~/.config/fish/conf.d +mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish ``` -Fish loads completion files from `~/.config/fish/completions` automatically. +Fish loads the files in `~/.config/fish/conf.d` automatically. ### Automatic Installation MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`: ```bash -mvt-ios completion bash --install -mvt-android completion bash --install +mvt completion bash --install ``` -Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`. +Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/conf.d` and changes no shell configuration. + +!!! note + + Earlier versions generated one script per command, with `mvt-ios completion` + and `mvt-android completion`. Files written by them keep working. When you + switch to the single script, remove the old files and the lines which load + them from your shell configuration. For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion). - diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md index 1d012f9..19cdb85 100644 --- a/docs/development/custom_commands.md +++ b/docs/development/custom_commands.md @@ -1,6 +1,8 @@ # Custom CLI Commands -MVT can load additional top-level commands into `mvt-ios` and `mvt-android`. +MVT can load additional top-level commands into `mvt`, `mvt-ios` and +`mvt-android`. A command package chooses which of the three each of its +commands is added to. Custom commands are different from [custom forensic modules](index.md#custom-modules): commands add new CLI operations, while modules add analysis steps to existing `check-*` commands. @@ -13,8 +15,8 @@ commands add new CLI operations, while modules add analysis steps to existing ## Install a Command Package -Python packages can register a Click command or group for either MVT CLI. A -minimal package can expose this command from `my_mvt_plugin.py`: +Python packages can register a Click command or group on one or more of the MVT CLIs. +A minimal package can expose this command from `my_mvt_plugin.py`: ```python import click @@ -38,9 +40,13 @@ summarize = "my_mvt_plugin:summarize" summarize = "my_mvt_plugin:summarize" ``` -Use only the iOS or Android group if the command is platform-specific. After -installing the package in the same environment as MVT, it appears directly in -the appropriate CLI: +Each entry-point group adds the command to one CLI: `mvt.ios.cli_plugins` to +`mvt-ios`, `mvt.android.cli_plugins` to `mvt-android` and `mvt.cli_plugins` to +`mvt`. Register the command in the group of every CLI which should offer it: a +platform-specific command belongs in one platform group, and a command which +handles acquisitions of both platforms, as above, in both. After installing the +package in the same environment as MVT, the command appears directly in those +CLIs: ```bash mvt-ios summarize ./ios-backup @@ -60,10 +66,32 @@ Command packages that need their own settings, such as an API key, should store them in a namespaced [plugin configuration file](plugin_configuration.md) rather than in MVT's own `config.yaml`. -## Load a Command File +### Commands on `mvt` -For local commands that are not packaged, create a Python file that exports one -Click command or group named `cli`: +The `mvt` command hosts what belongs to neither platform: `version`, +`completion` and `download-iocs`. A plugin command which is not about the +acquisition of one platform, such as one which configures the plugin or +synchronizes the indicators it uses, belongs there too, in the +`mvt.cli_plugins` group: + +```toml +[project.entry-points."mvt.cli_plugins"] +my-plugin = "my_mvt_plugin:my_plugin" +``` + +Commands in this group are added to `mvt` only, so this one is invoked as +`mvt my-plugin`. A command on `mvt` has nothing but its name to say which +plugin it belongs to, so name it after the plugin, and make it a Click group +when the plugin has several operations to offer, such as +`mvt my-plugin configure`. + +## Developing a Command Locally + +A package is how a command is distributed. While a command is being written, +MVT can load it straight from its file instead, so the package need not be +reinstalled after every change; an editable install of the package does the +same through its entry points. Create a Python file that exports one Click +command or group named `cli`: ```python import click @@ -86,10 +114,12 @@ mvt-ios --load-command ./case_summary.py case-summary ./ios-backup non-hidden top-level `*.py` files in sorted order and skips `__init__.py`. Every loaded file must export one `cli` object. -To load a file or folder on every invocation, set the platform-specific -environment variable: +To load a file or folder on every invocation, set the environment variable of +the CLI the commands belong on. Like the entry-point groups, each variable adds +its commands to one CLI only: ```bash +export MVT_CUSTOM_COMMANDS=./commands export MVT_IOS_CUSTOM_COMMANDS=./ios_commands export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands ``` @@ -97,9 +127,11 @@ export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands ## Naming and Errors Built-in MVT commands cannot be replaced. External command names must also be -unique; when installed packages or environment paths collide, MVT keeps the -first command and logs a warning. A collision from an explicit -`--load-command` is a usage error. +unique on each CLI; when installed packages or environment paths collide, MVT +keeps the first command and logs a warning. The environment path of a CLI is +registered before its installed packages, so a command loaded from there wins a +collision with a package. A collision from an explicit `--load-command` is a +usage error. A package entry point or environment command that cannot be imported appears as a marked broken command without preventing other MVT commands from working. diff --git a/docs/index.md b/docs/index.md index fada316..8cd733e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forens It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors. -In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results. +In this documentation you will find instructions on how to install and run the `mvt-ios`, `mvt-android` and `mvt` commands, and guidance on how to interpret the extracted results. ## Resources diff --git a/docs/install.md b/docs/install.md index c08c75d..c024338 100644 --- a/docs/install.md +++ b/docs/install.md @@ -64,7 +64,7 @@ It is recommended to try installing and running MVT from [Windows Subsystem Linu pipx install mvt ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. ### Installing from PyPI directly into a virtual environment You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool: @@ -84,7 +84,7 @@ source env/bin/activate pip install mvt ``` -The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. +The `mvt`, `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. ### Installing from git source with pipx If you want to have the latest features in development, you can install MVT directly from the source code in git. @@ -93,7 +93,7 @@ If you want to have the latest features in development, you can install MVT dire pipx install --force git+https://github.com/mvt-project/mvt.git ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. **Notes:** 1. The `--force` flag is necessary to force the reinstallation of the package. diff --git a/docs/iocs.md b/docs/iocs.md index c7586de..5f50eff 100644 --- a/docs/iocs.md +++ b/docs/iocs.md @@ -71,7 +71,7 @@ So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis - [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2). - We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns. -You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. +You can automatically download the latest public indicator files with the command `mvt download-iocs`. The per-platform forms `mvt-ios download-iocs` and `mvt-android download-iocs` do the same thing. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs. diff --git a/pyproject.toml b/pyproject.toml index ad92423..fb90f8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ homepage = "https://docs.mvt.re/en/latest/" repository = "https://github.com/mvt-project/mvt" [project.scripts] +mvt = "mvt.cli:main" mvt-ios = "mvt.ios:main" mvt-android = "mvt.android:main" diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index cabd083..4716168 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -15,12 +15,6 @@ from mvt.common.cli_plugins import ( register_cli_plugins, ) from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -31,7 +25,6 @@ from mvt.common.help import ( HELP_MSG_CHECK_IOCS, HELP_MSG_CHECK_INTRUSION_LOGS, HELP_MSG_DELAY_CHECKS, - HELP_MSG_COMPLETION, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_HASHES, @@ -102,55 +95,20 @@ def cli(ctx, disable_update_check, disable_indicator_update_check): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-android" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: check-adb (removed) # ============================================================================== diff --git a/src/mvt/cli.py b/src/mvt/cli.py new file mode 100644 index 0000000..a2e48dd --- /dev/null +++ b/src/mvt/cli.py @@ -0,0 +1,102 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import click + +from mvt.common.cli_plugins import ( + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, + load_cli_commands_option, + register_cli_plugins, +) +from mvt.common.completion import completion +from mvt.common.help import ( + HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, + HELP_MSG_DISABLE_UPDATE_CHECK, + HELP_MSG_STIX2, + HELP_MSG_VERSION, +) +from mvt.common.logo import logo +from mvt.common.updates import IndicatorsUpdates +from mvt.common.utils import init_logging + +init_logging() + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +# ============================================================================== +# Main +# ============================================================================== +@click.group(invoke_without_command=True) +@load_cli_commands_option +@click.option( + "--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK +) +@click.option( + "--disable-indicator-update-check", + is_flag=True, + help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, +) +@click.pass_context +def cli(ctx, disable_update_check, disable_indicator_update_check): + """Mobile Verification Toolkit. + + mvt-ios and mvt-android run the forensic analysis of an acquisition: each + provides the check-* commands of its platform. This command hosts what + belongs to neither platform; run it without a command to see the installed + version and the list of what it offers. + """ + ctx.ensure_object(dict) + ctx.obj["disable_version_check"] = disable_update_check + ctx.obj["disable_indicator_check"] = disable_indicator_update_check + if ctx.invoked_subcommand != "completion": + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +# ============================================================================== +# Command: download-iocs +# ============================================================================== +@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) +def download_iocs(): + ioc_updates = IndicatorsUpdates() + ioc_updates.update() + + +# ============================================================================== +# Command: completion +# ============================================================================== +cli.add_command(completion) + + +# ============================================================================== +# Command: version +# ============================================================================== +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) +def version(): + return + + +# ============================================================================== +# Entry point of the mvt console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index acf4c6d..4735222 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -17,6 +17,9 @@ import click IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" +# Commands in this group are registered on the platform-neutral mvt command only. +NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins" +MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS" MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS" MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS" @@ -255,6 +258,16 @@ def register_cli_plugins( entry_point_group: str, environment_variable: str, ) -> None: + """Register the external commands of one CLI on its group. + + Each CLI has one entry-point group and one environment variable of its + own, so a command package chooses the CLIs its commands are added to. + + :param group: CLI group to register the external commands on. + :param entry_point_group: Entry-point group of the CLI. + :param environment_variable: Name of the environment variable holding a + path to load commands from. + """ environment_path = os.environ.get(environment_variable) if environment_path: register_cli_commands_from_path(group, environment_path) diff --git a/src/mvt/common/completion.py b/src/mvt/common/completion.py index 6466a6d..0b72b4e 100644 --- a/src/mvt/common/completion.py +++ b/src/mvt/common/completion.py @@ -9,34 +9,83 @@ import shlex import click from click.shell_completion import get_completion_class +from .help import HELP_MSG_COMPLETION + SUPPORTED_SHELLS = ("bash", "zsh", "fish") +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +COMPLETION_INSTRUCTIONS = """Shell completion for mvt, mvt-ios and mvt-android -def completion_instructions(program_name: str) -> str: - return f"""Shell completion for {program_name} - -Print a completion script: - {program_name} completion bash > ~/.{program_name}-complete.bash - {program_name} completion zsh > ~/.{program_name}-complete.zsh - mkdir -p ~/.config/fish/completions - {program_name} completion fish > ~/.config/fish/completions/{program_name}.fish +Print one completion script covering the three commands: + mvt completion bash > ~/.mvt-complete.bash + mvt completion zsh > ~/.mvt-complete.zsh + mkdir -p ~/.config/fish/conf.d + mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish Load the generated Bash script from ~/.bashrc: - [ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash + [ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash Load the generated Zsh script from ~/.zshrc: - [ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh + [ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh -Fish loads completion files from ~/.config/fish/completions automatically. +Fish loads the files in ~/.config/fish/conf.d automatically. -To write these files and update Bash/Zsh shell configuration automatically: - {program_name} completion bash --install - {program_name} completion zsh --install - {program_name} completion fish --install +To write these files and update the Bash/Zsh shell configuration automatically: + mvt completion bash --install + mvt completion zsh --install + mvt completion fish --install """ +def _mvt_programs() -> list[tuple[str, click.Command]]: + """Return the console script name and CLI group of every MVT program. + + The three CLIs are imported here rather than at module level: mvt.cli + imports this module while it is being defined, and generating a completion + script should not make the start-up of `mvt` import the platform CLIs. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as mvt_cli + from mvt.ios.cli import cli as ios_cli + + return [("mvt", mvt_cli), ("mvt-ios", ios_cli), ("mvt-android", android_cli)] + + +@click.command( + "completion", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_COMPLETION, + short_help="Generate or install shell completion", +) +@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) +@click.option( + "--install", + is_flag=True, + help="Write completion files and update shell configuration.", +) +def completion(shell, install): + if shell is None: + if install: + raise click.UsageError("A shell is required when using --install.") + click.echo(COMPLETION_INSTRUCTIONS) + return + + if install: + script_path = install_completion_script(shell) + click.echo( + f"Installed {shell} completion for mvt, mvt-ios and mvt-android " + f"to {script_path}" + ) + if shell in ("bash", "zsh"): + click.echo(f"Updated ~/.{shell}rc") + else: + click.echo("Fish loads the files in ~/.config/fish/conf.d automatically.") + return + + click.echo(generate_mvt_completion_script(shell)) + + def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str: completion_class = get_completion_class(shell) if completion_class is None: @@ -46,41 +95,49 @@ def generate_completion_script(cli: click.Command, program_name: str, shell: str return completion_class(cli, {}, program_name, complete_var).source() -def install_completion_script( - cli: click.Command, - program_name: str, - shell: str, -) -> Path: - script = generate_completion_script(cli, program_name, shell) - script_path = _completion_script_path(program_name, shell) +def generate_mvt_completion_script(shell: str) -> str: + """Return one script completing every MVT command. + + Click names the completion function of each program after the program, so + the scripts of the three commands can simply be concatenated. + """ + scripts = [ + generate_completion_script(cli, program_name, shell).strip("\n") + for program_name, cli in _mvt_programs() + ] + return "\n\n".join(scripts) + + +def install_completion_script(shell: str) -> Path: + script = generate_mvt_completion_script(shell) + script_path = _completion_script_path(shell) script_path.parent.mkdir(parents=True, exist_ok=True) - script_path.write_text(script, encoding="utf-8") + script_path.write_text(f"{script}\n", encoding="utf-8") if shell in ("bash", "zsh"): - _install_shell_source_line(program_name, shell, script_path) + _install_shell_source_line(shell, script_path) return script_path -def _completion_script_path(program_name: str, shell: str) -> Path: +def _completion_script_path(shell: str) -> Path: home = Path.home() if shell == "fish": - return home / ".config" / "fish" / "completions" / f"{program_name}.fish" + # conf.d is sourced when the shell starts, unlike the completions + # folder, whose files fish loads on demand by command name. + return home / ".config" / "fish" / "conf.d" / "mvt-completion.fish" - return home / f".{program_name}-complete.{shell}" + return home / f".mvt-complete.{shell}" -def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None: +def _install_shell_source_line(shell: str, script_path: Path) -> None: shell_config_path = Path.home() / f".{shell}rc" source_line = ( f"[ -f {shlex.quote(str(script_path))} ] && " f". {shlex.quote(str(script_path))}" ) - block = ( - f"# MVT shell completion for {program_name}\n" - f"{source_line}\n" - ) + block = f"# MVT shell completion\n{source_line}\n" if shell_config_path.exists(): shell_config = shell_config_path.read_text(encoding="utf-8") diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 7514113..4eccef1 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -21,7 +21,9 @@ HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators" HELP_MSG_STIX2 = "Download public STIX2 indicators" HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check" HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check" -HELP_MSG_COMPLETION = "Generate or install shell completion" +HELP_MSG_COMPLETION = ( + "Generate or install shell completion for mvt, mvt-ios and mvt-android" +) # IOS Specific HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup" diff --git a/src/mvt/common/logo.py b/src/mvt/common/logo.py index 048ee22..862479f 100644 --- a/src/mvt/common/logo.py +++ b/src/mvt/common/logo.py @@ -24,7 +24,7 @@ def check_updates( latest_version = mvt_updates.check() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): rich_print( - "\t\t[bold]Note: Could not check for MVT updates.[/bold] " + "\t[bold]Note: Could not check for MVT updates.[/bold] " "You may be working offline. Please update MVT regularly." ) except Exception as e: @@ -34,7 +34,7 @@ def check_updates( else: if latest_version: rich_print( - f"\t\t[bold]Version {latest_version} is available! " + f"\t[bold]Version {latest_version} is available! " "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" ) @@ -46,7 +46,7 @@ def check_updates( # If not, there's no point in proceeding with the updates check. if ioc_updates.get_latest_update() == 0: rich_print( - "\t\t[bold]You have not yet downloaded any indicators, check " + "\t[bold]You have not yet downloaded any indicators, check " "the `download-iocs` command![/bold]" ) return @@ -57,7 +57,7 @@ def check_updates( should_check, hours = ioc_updates.should_check() if not should_check: rich_print( - f"\t\tIndicators updates checked recently, next automatic check " + f"\tIndicators updates checked recently, next automatic check " f"in {int(hours)} hours" ) return @@ -66,7 +66,7 @@ def check_updates( ioc_to_update = ioc_updates.check() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): rich_print( - "\t\t[bold]Note: Could not check for indicator updates.[/bold] " + "\t[bold]Note: Could not check for indicator updates.[/bold] " "You may be working offline. Please update MVT indicators regularly." ) except Exception as e: @@ -76,20 +76,20 @@ def check_updates( else: if ioc_to_update: rich_print( - "\t\t[bold]There are updates to your indicators files! " + "\t[bold]There are updates to your indicators files! " "Run the `download-iocs` command to update![/bold]" ) else: - rich_print("\t\tYour indicators files seem to be up to date.") + rich_print("\tYour indicators files seem to be up to date.") def logo( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: rich_print("\n") - rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit") - rich_print("\t\thttps://mvt.re") - rich_print(f"\t\tVersion: {MVT_VERSION}") + rich_print("\t[bold]MVT - Mobile Verification Toolkit[/bold]\n") + rich_print("\thttps://mvt.re") + rich_print(f"\tVersion: {MVT_VERSION}\n") check_updates(disable_version_check, disable_indicator_check) diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 52a1c29..6f633ea 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -15,12 +15,6 @@ from mvt.common.cli_plugins import ( register_cli_plugins, ) from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption from mvt.common.updates import IndicatorsUpdates @@ -51,7 +45,6 @@ from mvt.common.help import ( HELP_MSG_CHECK_SYSDIAGNOSE, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, - HELP_MSG_COMPLETION, ) from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.password import prompt_password @@ -106,55 +99,20 @@ def cli(ctx, disable_update_check, disable_indicator_update_check): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-ios" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: decrypt-backup # ============================================================================== diff --git a/tests/common/test_cli_plugins.py b/tests/common/test_cli_plugins.py index ea7c7fa..5abb2c9 100644 --- a/tests/common/test_cli_plugins.py +++ b/tests/common/test_cli_plugins.py @@ -3,9 +3,12 @@ from types import SimpleNamespace import click from click.testing import CliRunner +from mvt.cli import cli as mvt_cli from mvt.common.cli_plugins import ( ANDROID_CLI_PLUGIN_GROUP, IOS_CLI_PLUGIN_GROUP, + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, BrokenPluginCommand, load_cli_commands_option, register_cli_commands_from_path, @@ -14,6 +17,9 @@ from mvt.common.cli_plugins import ( ) +# Keep the banner of the mvt group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + COMMAND_TEMPLATE = """ import click @@ -345,7 +351,11 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( def entry_points(*, group): if group == IOS_CLI_PLUGIN_GROUP: return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] - return [_entry_point("android-package", "android_plugin:cli", android_package)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [ + _entry_point("android-package", "android_plugin:cli", android_package) + ] + return [] monkeypatch.setattr( "mvt.common.cli_plugins.importlib.metadata.entry_points", @@ -369,3 +379,187 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( assert set(ios_group.commands) == {"ios-file", "ios-package"} assert set(android_group.commands) == {"android-file", "android-package"} + + +def test_neutral_entry_point_group_is_not_registered_on_the_platform_clis( + monkeypatch, +): + @click.command() + def neutral_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point("neutral-package", "neutral_plugin:cli", neutral_package) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + ios_group = click.Group() + android_group = click.Group() + + register_cli_plugins( + ios_group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + register_cli_plugins( + android_group, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable="TEST_ANDROID_COMMANDS", + ) + + assert not ios_group.commands + assert not android_group.commands + + +def test_environment_command_wins_collision_with_installed_command( + tmp_path, monkeypatch, caplog +): + command_path = _write_command( + tmp_path / "duplicate.py", + "duplicate", + message="environment command ran", + ) + + @click.command() + def installed_command(): + pass + + def entry_points(*, group): + if group == IOS_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "duplicate", + "ios_plugin:cli", + installed_command, + distribution="ios-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + monkeypatch.setenv("TEST_IOS_COMMANDS", str(command_path)) + group = click.Group() + + register_cli_plugins( + group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + + assert group.commands["duplicate"] is not installed_command + result = CliRunner().invoke(group, ["duplicate"]) + assert result.exit_code == 0 + assert "environment command ran" in result.output + assert "the command name is already registered" in caplog.text + assert "ios-plugin 1.0 (ios_plugin:cli)" in caplog.text + + +def test_the_mvt_cli_gets_the_neutral_commands_and_no_platform_command( + monkeypatch, restore_cli_commands +): + @click.command() + def shared_package(): + click.echo("shared command ran") + + @click.command() + def ios_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared-package", "shared_plugin:cli", shared_package)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert "ios-package" not in mvt_cli.commands + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "shared-package"]) + assert result.exit_code == 0 + assert "shared command ran" in result.output + + +def test_builtin_mvt_command_wins_collision_with_neutral_command( + monkeypatch, caplog, restore_cli_commands +): + @click.command() + def neutral_version(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "version", + "neutral_plugin:cli", + neutral_version, + distribution="neutral-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + builtin_version = mvt_cli.commands["version"] + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert mvt_cli.commands["version"] is builtin_version + assert "the command name is already registered" in caplog.text + assert "neutral-plugin 1.0 (neutral_plugin:cli)" in caplog.text + + +def test_broken_neutral_plugin_does_not_break_the_mvt_cli( + monkeypatch, restore_cli_commands +): + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "broken", + "broken_plugin:cli", + exception=RuntimeError("missing dependency"), + distribution="broken-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert isinstance(mvt_cli.commands["broken"], BrokenPluginCommand) + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "version"]) + assert result.exit_code == 0 diff --git a/tests/conftest.py b/tests/conftest.py index c89f629..06a890a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,11 @@ import os import pytest +from mvt.common.cli_plugins import ( + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, +) from mvt.common.indicators import Indicators from .artifacts.generate_stix import generate_test_stix_file @@ -58,3 +63,33 @@ def indicators_factory(indicator_file): return ind return f + + +@pytest.fixture() +def restore_cli_commands(monkeypatch): + """Keep the external commands a test registers out of the next test. + + Each CLI group is a module-level object shared by every test, so a test + registering plugin or environment commands on one has to put it back. The + groups are imported here rather than at the top of the file, so that + collecting the tests does not import three CLIs for the sake of one + fixture. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as neutral_cli + from mvt.ios.cli import cli as ios_cli + + groups = (neutral_cli, ios_cli, android_cli) + for variable in ( + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ): + monkeypatch.delenv(variable, raising=False) + originals = [dict(group.commands) for group in groups] + yield + for group, commands in zip(groups, originals): + group.commands.clear() + group.commands.update(commands) + if hasattr(group, "_mvt_external_command_sources"): + delattr(group, "_mvt_external_command_sources") diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7a5cb51 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,49 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from click.testing import CliRunner + +from mvt.cli import cli +from mvt.common.updates import IndicatorsUpdates +from mvt.common.version import MVT_VERSION + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class TestMvtCommand: + def test_running_mvt_alone_shows_the_logo_and_the_commands(self): + result = CliRunner().invoke(cli, OFFLINE) + + assert result.exit_code == 0 + logo_at = result.output.index("Mobile Verification Toolkit") + usage_at = result.output.index("Usage:") + assert logo_at < usage_at + assert "mvt-ios" in result.output and "mvt-android" in result.output + + def test_help_reminds_where_the_analysis_runs(self): + result = CliRunner().invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "mvt-ios" in result.output + assert "mvt-android" in result.output + assert "check-*" in result.output + + def test_version_prints_the_installed_version(self): + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert f"Version: {MVT_VERSION}" in result.output + + def test_download_iocs_updates_the_indicators(self, monkeypatch): + updates = [] + monkeypatch.setattr( + IndicatorsUpdates, "update", lambda self: updates.append(self) + ) + + result = CliRunner().invoke(cli, [*OFFLINE, "download-iocs"]) + + assert result.exit_code == 0 + assert len(updates) == 1 diff --git a/tests/test_cli_entry_points.py b/tests/test_cli_entry_points.py index f5834ac..1b51413 100644 --- a/tests/test_cli_entry_points.py +++ b/tests/test_cli_entry_points.py @@ -10,10 +10,20 @@ import click import pytest import mvt.android +import mvt.cli import mvt.ios from mvt.android.cli import cli as android_cli from mvt.android.cli import main as android_main -from mvt.common.cli_plugins import ANDROID_CLI_PLUGIN_GROUP, IOS_CLI_PLUGIN_GROUP +from mvt.cli import cli as mvt_cli +from mvt.cli import main as mvt_main +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, +) from mvt.ios.cli import cli as ios_cli from mvt.ios.cli import main as ios_main @@ -39,23 +49,32 @@ def cli(): """ PROGRAMS = { - "mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP), - "mvt-android": (mvt.android, android_cli, ANDROID_CLI_PLUGIN_GROUP), + "mvt": (mvt.cli, mvt_cli, NEUTRAL_CLI_PLUGIN_GROUP, MVT_CUSTOM_COMMANDS_ENV), + "mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP, MVT_IOS_CUSTOM_COMMANDS_ENV), + "mvt-android": ( + mvt.android, + android_cli, + ANDROID_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ), } +CASE_SUMMARY_COMMAND = """ +import click -@pytest.fixture -def restore_cli_commands(): - """Undo the plugin registration main() performs on the shared CLI groups.""" - originals = { - program: dict(group.commands) for program, (_, group, _) in PROGRAMS.items() - } - yield - for program, (_, group, _) in PROGRAMS.items(): - group.commands.clear() - group.commands.update(originals[program]) - if hasattr(group, "_mvt_external_command_sources"): - delattr(group, "_mvt_external_command_sources") + +@click.command("case-summary") +def cli(): + click.echo("case summary ran") +""" + +# The entry-point group of another program, for each program: no group may add +# its commands to a CLI other than its own. +OTHER_PROGRAMS_GROUP = { + "mvt": IOS_CLI_PLUGIN_GROUP, + "mvt-ios": NEUTRAL_CLI_PLUGIN_GROUP, + "mvt-android": NEUTRAL_CLI_PLUGIN_GROUP, +} def _install_fixture_entry_point(monkeypatch, entry_point_group, command): @@ -92,7 +111,7 @@ def _offline_argv(program, *arguments): def test_main_registers_installed_plugins_before_running_the_cli( program, monkeypatch, capsys, restore_cli_commands ): - package, group, entry_point_group = PROGRAMS[program] + package, group, entry_point_group, _ = PROGRAMS[program] @click.command() def fixture_command(): @@ -113,7 +132,7 @@ def test_main_registers_installed_plugins_before_running_the_cli( def test_main_completes_plugin_command_names( program, monkeypatch, capsys, restore_cli_commands ): - package, _, entry_point_group = PROGRAMS[program] + package, _, entry_point_group, _ = PROGRAMS[program] @click.command() def fixture_command(): @@ -136,17 +155,9 @@ def test_main_completes_plugin_command_names( def test_main_still_loads_commands_from_a_file( program, monkeypatch, capsys, tmp_path, restore_cli_commands ): - package, _, entry_point_group = PROGRAMS[program] + package, _, entry_point_group, _ = PROGRAMS[program] command_path = tmp_path / "case_summary.py" - command_path.write_text( - "import click\n" - "\n" - "\n" - '@click.command("case-summary")\n' - "def cli():\n" - ' click.echo("case summary ran")\n', - encoding="utf-8", - ) + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") _install_fixture_entry_point( monkeypatch, entry_point_group, click.Command("unused") ) @@ -163,12 +174,66 @@ def test_main_still_loads_commands_from_a_file( assert "case summary ran" in capsys.readouterr().out +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_loads_commands_from_the_environment_variable( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + # Each CLI reads its own variable, so a main() reading another CLI's would + # go unnoticed without this. + package, _, _, environment_variable = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") + monkeypatch.setenv(environment_variable, str(command_path)) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "case-summary")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_ignores_the_entry_point_groups_of_the_other_programs( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, _, _ = PROGRAMS[program] + _install_fixture_entry_point( + monkeypatch, + OTHER_PROGRAMS_GROUP[program], + click.Command(FIXTURE_COMMAND_NAME), + ) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "--help")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert FIXTURE_COMMAND_NAME not in group.commands + assert FIXTURE_COMMAND_NAME not in capsys.readouterr().out + + def test_the_console_script_targets_are_importable(): - # [project.scripts] points at these, so they must stay on the packages. + # [project.scripts] points at these, so they must stay where they are. + assert mvt.cli.main is mvt_main assert mvt.ios.main is ios_main assert mvt.android.main is android_main +def test_importing_mvt_does_not_import_a_cli(tmp_path): + # The mvt package deliberately re-exports nothing of mvt.cli, so that + # importing MVT stays cheap and free of side effects. + result = run_isolated_python( + "import sys\n" + "import mvt\n" + "print('imported a cli' if 'mvt.cli' in sys.modules else 'imported mvt')\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "imported mvt" + + def test_importing_mvt_does_not_run_installed_plugins(tmp_path): site_path = write_cli_plugin_distribution( tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE diff --git a/tests/test_completion.py b/tests/test_completion.py index 48c0177..08ad3ce 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -6,56 +6,60 @@ from click.testing import CliRunner from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli from mvt.ios.cli import cli as ios_cli class TestCompletionCommand: def test_completion_prints_instructions_by_default(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion"]) + result = runner.invoke(mvt_cli, ["completion"]) assert result.exit_code == 0 - assert "Shell completion for mvt-ios" in result.output - assert "mvt-ios completion bash > ~/.mvt-ios-complete.bash" in result.output + assert "Shell completion for mvt, mvt-ios and mvt-android" in result.output + assert "mvt completion bash > ~/.mvt-complete.bash" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_bash_script(self): + def test_completion_bash_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash"]) + result = runner.invoke(mvt_cli, ["completion", "bash"]) assert result.exit_code == 0 + assert "_MVT_COMPLETE=bash_complete" in result.output assert "_MVT_IOS_COMPLETE=bash_complete" in result.output + assert "_MVT_ANDROID_COMPLETE=bash_complete" in result.output assert "complete -o nosort" in result.output - assert "mvt-ios" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_fish_script(self): + def test_completion_fish_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish"]) + result = runner.invoke(mvt_cli, ["completion", "fish"]) assert result.exit_code == 0 - assert "_MVT_ANDROID_COMPLETE=fish_complete" in result.output + assert "complete --no-files --command mvt-ios" in result.output assert "complete --no-files --command mvt-android" in result.output + assert "complete --no-files --command mvt " in result.output assert "Mobile Verification Toolkit" not in result.output def test_completion_install_updates_bashrc_once(self, tmp_path, monkeypatch): monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 - script_path = tmp_path / ".mvt-ios-complete.bash" + script_path = tmp_path / ".mvt-complete.bash" bashrc_path = tmp_path / ".bashrc" assert script_path.exists() - assert "_MVT_IOS_COMPLETE=bash_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=bash_complete" in script + assert "_MVT_IOS_COMPLETE=bash_complete" in script + assert "_MVT_ANDROID_COMPLETE=bash_complete" in script bashrc = bashrc_path.read_text(encoding="utf-8") assert "[ -f" in bashrc - assert ".mvt-ios-complete.bash" in bashrc + assert ".mvt-complete.bash" in bashrc - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 assert bashrc_path.read_text(encoding="utf-8") == bashrc @@ -65,14 +69,30 @@ class TestCompletionCommand: monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "fish", "--install"]) assert result.exit_code == 0 - script_path = ( - tmp_path / ".config" / "fish" / "completions" / "mvt-android.fish" - ) + script_path = tmp_path / ".config" / "fish" / "conf.d" / "mvt-completion.fish" assert script_path.exists() - assert "_MVT_ANDROID_COMPLETE=fish_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=fish_complete" in script + assert "_MVT_IOS_COMPLETE=fish_complete" in script + assert "_MVT_ANDROID_COMPLETE=fish_complete" in script assert not (tmp_path / ".fishrc").exists() + assert not (tmp_path / ".bashrc").exists() + assert not (tmp_path / ".zshrc").exists() + + def test_completion_install_without_shell_is_a_usage_error(self): + runner = CliRunner() + result = runner.invoke(mvt_cli, ["completion", "--install"]) + + assert result.exit_code == 2 + assert "A shell is required when using --install." in result.output + + def test_completion_is_not_a_command_of_the_platform_clis(self): + runner = CliRunner() + + assert "completion" not in ios_cli.commands + assert "completion" not in android_cli.commands + assert runner.invoke(ios_cli, ["completion"]).exit_code == 2 + assert runner.invoke(android_cli, ["completion"]).exit_code == 2 From dcfd500112399f77df71719d3daa0ba880cc301c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:15 +0200 Subject: [PATCH 33/43] Add plugin update checking and a plugins command (#898) * Add plugin update checking Report available updates to the installed MVT plugin packages in the startup banner, for plugins installed from a package index and for plugins installed directly from a repository. Repository installs pinned to a commit or a tag are never reported as outdated. MVT only prints the command which upgrades a plugin. Installing the update stays a deliberate choice of the analyst. The check runs at most once every twelve hours, and in between prints the findings of the latest check which still apply to what is installed. Nothing about the check can interrupt a running command: the parts of the suggested command come from package metadata and are quoted for the shell, the repository query refuses to prompt for credentials and never passes metadata as a git option, and a corrupt or stale cache is discarded rather than trusted. * Add a plugins command to list installed plugins and check updates Add a "plugins" command to the platform-neutral mvt command. "plugins list" shows every installed plugin package with its version, where it was installed from, how many forensic modules it contributes and which commands it adds. "plugins check-updates" checks for updates immediately, without waiting for the automatic check, and prints the command which upgrades a plugin instead of installing anything. It lives on mvt only. The packages it lists extend mvt-ios and mvt-android too, but auditing them is not the job of a command which analyses one platform, and the two platform CLIs should not carry commands which are not about an acquisition. The command is registered as a built-in, before any external command, so that an installed package cannot replace this audit surface. --- README.md | 2 +- docs/development/custom_commands.md | 9 +- docs/development/index.md | 3 + docs/development/plugins.md | 116 +++++ mkdocs.yml | 1 + src/mvt/cli.py | 6 + src/mvt/common/cmd_plugins.py | 205 ++++++++ src/mvt/common/help.py | 5 + src/mvt/common/logo.py | 181 ++++--- src/mvt/common/module_loader.py | 29 +- src/mvt/common/updates.py | 419 +++++++++++++++- tests/common/test_cmd_plugins.py | 236 +++++++++ tests/common/test_plugin_updates.py | 711 ++++++++++++++++++++++++++++ 13 files changed, 1849 insertions(+), 74 deletions(-) create mode 100644 docs/development/plugins.md create mode 100644 src/mvt/common/cmd_plugins.py create mode 100644 tests/common/test_cmd_plugins.py create mode 100644 tests/common/test_plugin_updates.py diff --git a/README.md b/README.md index 5f00600..54cccca 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ For alternative installation options and known issues, please refer to the [docu ## Usage -MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) ### Shell completion diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md index 19cdb85..71ac623 100644 --- a/docs/development/custom_commands.md +++ b/docs/development/custom_commands.md @@ -60,7 +60,8 @@ pipx inject mvt my-mvt-plugin ``` When MVT is installed in an active virtual environment, install the plugin with -`pip` in that environment. +`pip` in that environment. `mvt plugins list` shows the installed packages and +the commands they add, see [Managing Plugins](plugins.md). Command packages that need their own settings, such as an API key, should store them in a namespaced [plugin configuration file](plugin_configuration.md) @@ -69,9 +70,9 @@ rather than in MVT's own `config.yaml`. ### Commands on `mvt` The `mvt` command hosts what belongs to neither platform: `version`, -`completion` and `download-iocs`. A plugin command which is not about the -acquisition of one platform, such as one which configures the plugin or -synchronizes the indicators it uses, belongs there too, in the +`completion`, `plugins` and `download-iocs`. A plugin command which is not +about the acquisition of one platform, such as one which configures the plugin +or synchronizes the indicators it uses, belongs there too, in the `mvt.cli_plugins` group: ```toml diff --git a/docs/development/index.md b/docs/development/index.md index c2b8464..692a2ab 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -214,6 +214,9 @@ module came from: - When a command runs with an `--output` folder, the `command.log` file records one line per module source with the source's version or hash and the list of modules loaded from it. +- `mvt plugins list` lists the installed packages, where each of them + was installed from and how many modules it contributes, see + [Managing Plugins](plugins.md). ## Profiling diff --git a/docs/development/plugins.md b/docs/development/plugins.md new file mode 100644 index 0000000..0085975 --- /dev/null +++ b/docs/development/plugins.md @@ -0,0 +1,116 @@ +# Managing Plugins + +Plugin packages extend MVT with additional +[forensic modules](index.md#custom-modules) and +[CLI commands](custom_commands.md). Because installed packages load +automatically, `mvt plugins` audits what is installed and checks whether +updates are available. The command lives on `mvt` only, although the packages +it lists extend `mvt-ios` and `mvt-android` too. + +## List Installed Plugins + +```bash +mvt plugins list +``` + +``` + Installed MVT plugins +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓ +┃ Name ┃ Version ┃ Origin ┃ Modules ┃ Commands ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩ +│ mvt-plugin-example │ 1.2.0 │ pypi │ 4 │ summarize │ +│ mvt-plugin-research │ 0.1.0 │ git+3f9a1c7d │ 2 │ - │ +│ mvt-plugin-local │ 0.0.1 │ local │ 1 │ triage │ +└─────────────────────┴─────────┴──────────────┴─────────┴───────────┘ +``` + +The origin records where each package was installed from: `pypi` for a package +installed from a package index, `git+` for a package installed directly +from a repository, and `local` for a package installed from a local folder or +archive rather than from an index, including an editable development install. +The last two columns show how many forensic modules the package contributes and +which CLI commands it adds. + +A plugin whose modules cannot be imported is listed with `error` in the +`Modules` column rather than breaking the listing. + +## Check for Updates + +```bash +mvt plugins check-updates +``` + +``` +Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 + Upgrade with: pip install -U mvt-plugin-example + +MVT does not install plugin updates. Run the command above when you decide to +upgrade. +``` + +Packages installed from a package index are compared against the latest release +published for them. A package which was never published, for example a plugin +distributed only within an organization, is skipped silently. + +!!! note + + Packages shown with the `pypi` origin are compared against + [PyPI](https://pypi.org), whichever index they were installed from. A + plugin installed from a private index under a name which also exists on + PyPI is therefore compared against the unrelated public package of that + name. Give plugins published to a private index a name which is not taken + on PyPI, and treat an unexpected update suggestion as a reason to check + where the package would come from. + +!!! warning + + MVT never installs or upgrades a plugin itself, it only prints the command + which does. Upgrading a plugin in the middle of an investigation changes + the modules producing the results, and a plugin runs as trusted code inside + the MVT process, so pulling in a new version is a decision for the analyst + to make deliberately and not a side effect of running a check. + +## Automatic Update Checks + +MVT also reports available plugin updates in the banner printed when a command +starts: + +``` + MVT - Mobile Verification Toolkit + + https://mvt.re + Version: 2026.7.29 + + Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 (pip install -U mvt-plugin-example) +``` + +This check runs at most once every 12 hours. In between checks MVT prints the +findings of the latest check without contacting anything, so a plugin update +stays visible without a lookup on every command. The +`mvt plugins check-updates` command checks immediately, regardless of when the +last check happened. + +The automatic check is skipped when the `--disable-update-check` option is +used, when `NETWORK_ACCESS_ALLOWED` is disabled in the MVT configuration, and +when no plugins are installed. + +## Plugins Installed From a Repository + +A plugin installed with `pip install "mvt-plugin-example @ git+"` is +checked by asking the remote repository which commit the installed revision +points at now. MVT runs git and ssh in batch mode, so a repository which needs +credentials MVT does not already have fails the check instead of prompting for +them. The check is skipped silently when git is not available, when the +repository cannot be reached, and when access to it is denied. + +How the plugin was installed decides what an update means: + +- A plugin installed from a branch is reported as outdated when the branch has + moved past the installed commit. +- A plugin installed from a specific commit or a tag is pinned. It is never + reported as outdated, however far the branch it came from moves on. + +Pinning a plugin to a commit or a tag is therefore the way to keep the modules +used across an investigation stable. diff --git a/mkdocs.yml b/mkdocs.yml index 6250cdc..66def17 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -53,4 +53,5 @@ nav: - Development Instructions: "development/index.md" - Custom CLI Commands: "development/custom_commands.md" - Plugin Configuration: "development/plugin_configuration.md" + - Managing Plugins: "development/plugins.md" - License: "license.md" diff --git a/src/mvt/cli.py b/src/mvt/cli.py index a2e48dd..3ba6602 100644 --- a/src/mvt/cli.py +++ b/src/mvt/cli.py @@ -11,6 +11,7 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) +from mvt.common.cmd_plugins import plugins from mvt.common.completion import completion from mvt.common.help import ( HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, @@ -84,6 +85,11 @@ def version(): return +# The plugins command is registered as a built-in command, before any external +# command, so that an installed package can never replace it. +cli.add_command(plugins) + + # ============================================================================== # Entry point of the mvt console script # ============================================================================== diff --git a/src/mvt/common/cmd_plugins.py b/src/mvt/common/cmd_plugins.py new file mode 100644 index 0000000..21619eb --- /dev/null +++ b/src/mvt/common/cmd_plugins.py @@ -0,0 +1,205 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import importlib.metadata +import logging +from typing import Optional + +import click +from rich.console import Console +from rich.table import Table + +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from .config import settings +from .help import ( + HELP_MSG_PLUGINS, + HELP_MSG_PLUGINS_CHECK_UPDATES, + HELP_MSG_PLUGINS_LIST, +) +from .module import MVTModule +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url +from .updates import ( + SHORT_COMMIT_LENGTH, + PluginUpdates, + installed_plugin_distributions, +) + +log = logging.getLogger(__name__) + +CLI_PLUGIN_GROUPS = ( + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +def _entry_points(group: str) -> list[importlib.metadata.EntryPoint]: + try: + return list(importlib.metadata.entry_points(group=group)) + except Exception as exc: + log.warning("Unable to discover the entry points in group %s: %s", group, exc) + return [] + + +def _entry_point_distribution( + entry_point: importlib.metadata.EntryPoint, +) -> Optional[str]: + dist = getattr(entry_point, "dist", None) + if dist is None: + return None + try: + return dist.name + except Exception: + return None + + +def _distribution_version(dist: importlib.metadata.Distribution) -> str: + try: + return dist.version or "unknown" + except Exception: + return "unknown" + + +def _distribution_origin(dist: importlib.metadata.Distribution) -> str: + """Describe where a plugin package was installed from.""" + direct_url = distribution_direct_url(dist) + if direct_url is None: + return "pypi" + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + commit = vcs_info.get("commit_id") or "" + if commit: + return f"git+{commit[:SHORT_COMMIT_LENGTH]}" + return "git" + + return "local" + + +def _contributed_modules( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + """Count the forensic modules a plugin package contributes. + + Entry points are resolved the way MVT resolves them when it loads + modules, but a broken entry point is reported instead of raising: listing + the installed plugins must work even when one of them is faulty. + """ + count = 0 + broken = False + + for entry_point in entry_points: + if _entry_point_distribution(entry_point) != distribution: + continue + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + count += sum( + 1 + for module in loaded + if isinstance(module, type) and issubclass(module, MVTModule) + ) + except (Exception, SystemExit) as exc: + log.debug( + "Unable to load the modules of entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + broken = True + + if broken: + return f"{count} (error)" if count else "error" + + return str(count) + + +def _contributed_commands( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + names = { + entry_point.name + for entry_point in entry_points + if _entry_point_distribution(entry_point) == distribution + } + + return ", ".join(sorted(names)) if names else "-" + + +@click.group("plugins", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS) +def plugins() -> None: + pass + + +@plugins.command("list", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS_LIST) +def list_plugins() -> None: + distributions = installed_plugin_distributions() + if not distributions: + click.echo("No MVT plugins are installed.") + return + + module_entry_points = _entry_points(MODULES_ENTRY_POINT_GROUP) + command_entry_points = [] + for group in CLI_PLUGIN_GROUPS: + command_entry_points.extend(_entry_points(group)) + + table = Table(title="Installed MVT plugins") + table.add_column("Name", style="bold") + table.add_column("Version") + table.add_column("Origin") + table.add_column("Modules", justify="right") + table.add_column("Commands") + + for dist in distributions: + name = dist.name + table.add_row( + name, + _distribution_version(dist), + _distribution_origin(dist), + _contributed_modules(module_entry_points, name), + _contributed_commands(command_entry_points, name), + ) + + Console().print(table) + + +@plugins.command( + "check-updates", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_PLUGINS_CHECK_UPDATES, + short_help="Check the installed plugins for updates", +) +def check_plugin_updates() -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + click.echo( + "Network access is disabled, cannot check for plugin updates. " + "Enable NETWORK_ACCESS_ALLOWED in the MVT configuration to check." + ) + return + + if not installed_plugin_distributions(): + click.echo("No MVT plugins are installed.") + return + + findings = PluginUpdates().check() + if not findings: + click.echo("All plugins are up to date.") + return + + click.echo("Plugin updates available:") + for finding in findings: + click.echo(f" {finding['name']} {finding['installed']} → {finding['latest']}") + click.echo(f" Upgrade with: {finding['upgrade_command']}") + + click.echo( + "\nMVT does not install plugin updates. Run the command above when you " + "decide to upgrade." + ) diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 4eccef1..13575e5 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -24,6 +24,11 @@ HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check" HELP_MSG_COMPLETION = ( "Generate or install shell completion for mvt, mvt-ios and mvt-android" ) +HELP_MSG_PLUGINS = "Inspect the installed MVT plugin packages" +HELP_MSG_PLUGINS_LIST = "List the installed plugins and what they contribute to MVT" +HELP_MSG_PLUGINS_CHECK_UPDATES = ( + "Check the installed plugins for updates without installing them" +) # IOS Specific HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup" diff --git a/src/mvt/common/logo.py b/src/mvt/common/logo.py index 862479f..96836a9 100644 --- a/src/mvt/common/logo.py +++ b/src/mvt/common/logo.py @@ -8,10 +8,125 @@ import logging import requests from rich import print as rich_print -from .updates import IndicatorsUpdates, MVTUpdates +from .config import settings +from .updates import ( + IndicatorsUpdates, + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) from .version import MVT_VERSION +def _check_version_updates(log: logging.Logger) -> None: + try: + mvt_updates = MVTUpdates() + latest_version = mvt_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for MVT updates.[/bold] " + "You may be working offline. Please update MVT regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT version: %s", e) + else: + if latest_version: + rich_print( + f"\t[bold]Version {latest_version} is available! " + "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" + ) + + +def _check_indicator_updates(log: logging.Logger) -> None: + ioc_updates = IndicatorsUpdates() + + # Before proceeding, we check if we have downloaded an indicators index. + # If not, there's no point in proceeding with the updates check. + if ioc_updates.get_latest_update() == 0: + rich_print( + "\t[bold]You have not yet downloaded any indicators, check " + "the `download-iocs` command![/bold]" + ) + return + + # We only perform this check at a fixed frequency, in order to not + # overburden the user with too many lookups if the command is being run + # multiple times. + should_check, hours = ioc_updates.should_check() + if not should_check: + rich_print( + f"\tIndicators updates checked recently, next automatic check " + f"in {int(hours)} hours" + ) + return + + try: + ioc_to_update = ioc_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for indicator updates.[/bold] " + "You may be working offline. Please update MVT indicators regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT indicators: %s", e) + else: + if ioc_to_update: + rich_print( + "\t[bold]There are updates to your indicators files! " + "Run the `download-iocs` command to update![/bold]" + ) + else: + rich_print("\tYour indicators files seem to be up to date.") + + +def _print_plugin_updates(findings: list) -> None: + if not findings: + return + + rich_print("\t[bold]Plugin updates available:[/bold]") + for finding in findings: + rich_print( + f"\t {finding['name']} {finding['installed']} → " + f"{finding['latest']} ({finding['upgrade_command']})" + ) + + +def _check_plugin_updates(log: logging.Logger) -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + return + + # This runs on every command, so nothing here, including reading back what + # the latest check stored, may ever interrupt MVT. + try: + distributions = installed_plugin_distributions() + + # There is nothing to check when MVT was not extended with any plugin. + if not distributions: + return + + plugin_updates = PluginUpdates() + + # We only perform this check at a fixed frequency, in order to not + # overburden the user (and the plugin repositories) with too many + # lookups. In between checks we print the findings of the latest one, + # leaving out those which no longer apply to what is installed. + should_check, _ = plugin_updates.should_check() + if not should_check: + _print_plugin_updates(plugin_updates.current_findings(distributions)) + return + + findings = plugin_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for plugin updates.[/bold] " + "You may be working offline. Please update your MVT plugins regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check MVT plugin updates: %s", e) + else: + _print_plugin_updates(findings) + + def check_updates( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: @@ -19,68 +134,16 @@ def check_updates( # First we check for MVT version updates. if not disable_version_check: - try: - mvt_updates = MVTUpdates() - latest_version = mvt_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t[bold]Note: Could not check for MVT updates.[/bold] " - "You may be working offline. Please update MVT regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT version: %s", e - ) - else: - if latest_version: - rich_print( - f"\t[bold]Version {latest_version} is available! " - "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" - ) + _check_version_updates(log) # Then we check for indicators files updates. if not disable_indicator_check: - ioc_updates = IndicatorsUpdates() + _check_indicator_updates(log) - # Before proceeding, we check if we have downloaded an indicators index. - # If not, there's no point in proceeding with the updates check. - if ioc_updates.get_latest_update() == 0: - rich_print( - "\t[bold]You have not yet downloaded any indicators, check " - "the `download-iocs` command![/bold]" - ) - return - - # We only perform this check at a fixed frequency, in order to not - # overburden the user with too many lookups if the command is being run - # multiple times. - should_check, hours = ioc_updates.should_check() - if not should_check: - rich_print( - f"\tIndicators updates checked recently, next automatic check " - f"in {int(hours)} hours" - ) - return - - try: - ioc_to_update = ioc_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t[bold]Note: Could not check for indicator updates.[/bold] " - "You may be working offline. Please update MVT indicators regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT indicators: %s", e - ) - else: - if ioc_to_update: - rich_print( - "\t[bold]There are updates to your indicators files! " - "Run the `download-iocs` command to update![/bold]" - ) - else: - rich_print("\tYour indicators files seem to be up to date.") + # Finally we check for updates to the installed plugin packages. MVT never + # installs an update itself, it only reports the command which does. + if not disable_version_check: + _check_plugin_updates(log) def logo( diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index 956da4c..37f447d 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -179,21 +179,36 @@ def _module_key(module_class: type[MVTModule]) -> tuple[str, str]: return (source, module_class.__qualname__) +def distribution_direct_url(dist: importlib.metadata.Distribution) -> Optional[dict]: + """Return the PEP 610 direct URL metadata of a distribution, if recorded. + + Packages installed from an index have no direct URL metadata, while + packages installed directly from a repository or from a local folder + record where they were installed from in ``direct_url.json``. + """ + try: + direct_url_text = dist.read_text("direct_url.json") + if not direct_url_text: + return None + direct_url = json.loads(direct_url_text) + return direct_url if isinstance(direct_url, dict) else None + except Exception: + return None + + def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]: """Return the VCS commit a distribution was installed from, if recorded. Packages installed directly from a repository (``pip install git+...``) record the commit in ``direct_url.json`` (PEP 610). """ - try: - direct_url_text = dist.read_text("direct_url.json") - if not direct_url_text: - return None - commit = json.loads(direct_url_text).get("vcs_info", {}).get("commit_id") - return commit if isinstance(commit, str) else None - except Exception: + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): return None + commit = vcs_info.get("commit_id") + return commit if isinstance(commit, str) else None + def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin: name = entry_point.name diff --git a/src/mvt/common/updates.py b/src/mvt/common/updates.py index 001a5c2..71e97cc 100644 --- a/src/mvt/common/updates.py +++ b/src/mvt/common/updates.py @@ -3,8 +3,13 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import importlib.metadata +import json import logging import os +import re +import shlex +import subprocess from datetime import datetime from typing import Optional, Tuple @@ -12,14 +17,35 @@ import requests import yaml from packaging import version +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) from .config import settings from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url from .version import MVT_VERSION log = logging.getLogger(__name__) # In hours. INDICATORS_CHECK_FREQUENCY = 12 +PLUGINS_CHECK_FREQUENCY = 12 + +# The entry-point groups a package can use to extend MVT. +PLUGIN_ENTRY_POINT_GROUPS = ( + MODULES_ENTRY_POINT_GROUP, + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +SHORT_COMMIT_LENGTH = 8 +# The keys every cached finding has to carry to be printed. +FINDING_KEYS = ("name", "installed", "latest", "upgrade_command") +# Options which stop ssh from waiting for an answer nobody is there to give. +_SSH_BATCH_OPTIONS = ("-o", "BatchMode=yes", "-o", "ConnectTimeout=10") +_COMMIT_PATTERN = re.compile(r"\A[0-9a-f]{7,40}\Z") class MVTUpdates: @@ -38,6 +64,395 @@ class MVTUpdates: return "" +def installed_plugin_distributions() -> list[importlib.metadata.Distribution]: + """Return the installed distributions which extend MVT. + + A plugin package is any distribution registering at least one entry point + in the module or CLI command groups. Distributions are returned once each, + sorted by name. MVT itself is not a plugin and is never returned. + """ + distributions: dict[str, importlib.metadata.Distribution] = {} + + for group in PLUGIN_ENTRY_POINT_GROUPS: + try: + entry_points = importlib.metadata.entry_points(group=group) + except Exception as exc: + log.warning( + "Unable to discover installed plugin packages in entry-point " + "group %s: %s", + group, + exc, + ) + continue + + for entry_point in entry_points: + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is None: + continue + try: + name = dist.name + except Exception: + continue + if not name or name == "mvt": + continue + distributions.setdefault(name, dist) + + return [distributions[name] for name in sorted(distributions)] + + +def _is_usable_finding(finding: object) -> bool: + """Check that a cached finding carries everything needed to print it.""" + if not isinstance(finding, dict): + return False + + return all( + isinstance(finding.get(key), str) and finding.get(key) for key in FINDING_KEYS + ) + + +def _installed_revision( + dist: importlib.metadata.Distribution, origin: object +) -> Optional[str]: + """Return what a plugin's installed revision is right now. + + This is the value a finding recorded as installed when it was made, so a + finding can be compared against the current state of the installation + without looking anything up remotely. + """ + try: + if origin != "git": + return dist.version + + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): + return None + + commit = vcs_info.get("commit_id") or "" + return commit[:SHORT_COMMIT_LENGTH] or None + except Exception as e: + log.debug("Failed to read the installed revision of a plugin: %s", e) + return None + + +def _batch_mode_ssh_command(ssh_command: str) -> str: + """Return an ssh command line which cannot stop to ask a question. + + ssh keeps the first value it is given for a keyword, so the batch mode + options are inserted right after the ssh program, ahead of whatever the + analyst configured. Their remaining options, such as which key to use, + still apply. + """ + try: + arguments = shlex.split(ssh_command.strip()) + except ValueError: + arguments = [] + + if not arguments: + arguments = ["ssh"] + + return shlex.join([arguments[0], *_SSH_BATCH_OPTIONS, *arguments[1:]]) + + +def _revision_pins_commit(revision: str, commit: str) -> bool: + """Check whether a requested revision pins the installed commit.""" + candidate = revision.lower() + if not _COMMIT_PATTERN.match(candidate): + return False + + return commit.lower().startswith(candidate) + + +class PluginUpdates: + """Check for updates to the installed MVT plugin packages. + + MVT never installs or upgrades a plugin package itself. It only reports + the command which upgrades a plugin, leaving the analyst to decide when to + run it. + """ + + @property + def latest_check_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "latest_plugins_check") + + @property + def findings_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "plugin_updates.json") + + def _create_data_folder(self) -> None: + if not os.path.exists(MVT_DATA_FOLDER): + os.makedirs(MVT_DATA_FOLDER) + + def get_latest_check(self) -> int: + if not os.path.exists(self.latest_check_path): + return 0 + + # A corrupt or truncated timestamp only means the next check happens + # sooner. It must never stop MVT from running. + try: + with open(self.latest_check_path, "r", encoding="utf-8") as handle: + data = handle.read().strip() + if data: + return int(data) + except (OSError, ValueError) as e: + log.debug("Failed to read the time of the latest plugin check: %s", e) + + return 0 + + def set_latest_check(self) -> None: + self._create_data_folder() + timestamp = int(datetime.now().timestamp()) + with open(self.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(timestamp)) + + def get_findings(self) -> list[dict]: + """ + Return the findings of the latest check, without checking again. + Returns an empty list if no check was ever performed. + """ + if not os.path.exists(self.findings_path): + return [] + + try: + with open(self.findings_path, "r", encoding="utf-8") as handle: + findings = json.load(handle) + except Exception as e: + log.debug("Failed to read the cached plugin updates: %s", e) + return [] + + if not isinstance(findings, list): + return [] + + # Anything which does not look like a finding is dropped rather than + # trusted: the cache is only a convenience. + return [finding for finding in findings if _is_usable_finding(finding)] + + def current_findings( + self, distributions: Optional[list[importlib.metadata.Distribution]] = None + ) -> list[dict]: + """ + Return the cached findings which still apply to what is installed. + Findings about a plugin which was upgraded or removed since the latest + check are dropped, so an update is never reported twice. + """ + if distributions is None: + distributions = installed_plugin_distributions() + + installed = {} + for dist in distributions: + try: + installed[dist.name] = dist + except Exception: + continue + + current = [] + for finding in self.get_findings(): + plugin = installed.get(finding["name"]) + if plugin is None: + continue + if ( + _installed_revision(plugin, finding.get("origin")) + != finding["installed"] + ): + continue + current.append(finding) + + return current + + def set_findings(self, findings: list[dict]) -> None: + self._create_data_folder() + with open(self.findings_path, "w", encoding="utf-8") as handle: + json.dump(findings, handle) + + def should_check(self) -> Tuple[bool, int]: + """ + Compare time of the latest plugins check with current time. + Returns bool and number of hours since the last check. + """ + now = datetime.now() + latest_check_ts = self.get_latest_check() + latest_check_dt = datetime.fromtimestamp(latest_check_ts) + + diff = now - latest_check_dt + diff_hours = divmod(diff.total_seconds(), 3600)[0] + + if diff_hours >= PLUGINS_CHECK_FREQUENCY: + return True, 0 + + return False, int(PLUGINS_CHECK_FREQUENCY - diff_hours) + + def _check_index_plugin(self, name: str, installed: str) -> Optional[dict]: + """Check a plugin installed from a package index for a newer release.""" + url = f"https://pypi.org/pypi/{name}/json" + try: + res = requests.get(url, timeout=settings.NETWORK_TIMEOUT) + except requests.exceptions.RequestException as e: + log.debug("Failed to check for updates to plugin %s: %s", name, e) + return None + + # Plugins which were never published to a public index are expected, + # and there is nothing to compare their version against. + if res.status_code == 404: + return None + + if res.status_code != 200: + log.debug( + "Failed to check for updates to plugin %s (error %d)", + name, + res.status_code, + ) + return None + + try: + latest = res.json().get("info", {}).get("version", "") + if not latest or version.parse(latest) <= version.parse(installed): + return None + except Exception as e: + log.debug("Failed to compare the versions of plugin %s: %s", name, e) + return None + + return { + "name": name, + "installed": installed, + "latest": latest, + "origin": "pypi", + # The name comes from package metadata, so the command MVT + # suggests is quoted rather than assumed to be shell-safe. + "upgrade_command": f"pip install -U {shlex.quote(name)}", + } + + def _git_ls_remote(self, url: str, revision: str) -> list[Tuple[str, str]]: + """Return the remote references matching a revision, if git allows it.""" + # Neither value is trusted: they are read from the metadata of an + # installed package and must not turn into git options. + if url.startswith("-") or revision.startswith("-"): + log.debug("Skipping the update check for the invalid repository %s", url) + return [] + + environment = dict(os.environ) + # Never prompt the analyst for repository credentials. git handles its + # own prompts, while ssh reads the terminal directly and only batch + # mode makes it fail instead of asking. + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["GIT_SSH_COMMAND"] = _batch_mode_ssh_command( + environment.get("GIT_SSH_COMMAND", "") + ) + + try: + process = subprocess.run( + ["git", "ls-remote", url, revision], + capture_output=True, + stdin=subprocess.DEVNULL, + text=True, + env=environment, + timeout=settings.NETWORK_TIMEOUT, + check=False, + ) + except FileNotFoundError: + log.debug("Could not find git, skipping the update check for %s", url) + return [] + except (subprocess.SubprocessError, OSError) as e: + log.debug("Failed to query the repository %s: %s", url, e) + return [] + + if process.returncode != 0: + log.debug( + "Failed to query the repository %s (error %d): %s", + url, + process.returncode, + (process.stderr or "").strip(), + ) + return [] + + references = [] + for line in (process.stdout or "").splitlines(): + commit, _, reference = line.partition("\t") + if commit.strip() and reference.strip(): + references.append((commit.strip(), reference.strip())) + + return references + + def _check_repository_plugin( + self, name: str, direct_url: dict, vcs_info: dict + ) -> Optional[dict]: + """Check a plugin installed from a repository for a newer commit.""" + url = direct_url.get("url") or "" + installed = vcs_info.get("commit_id") or "" + revision = vcs_info.get("requested_revision") or "" + if not url or not installed: + return None + + # A plugin installed from a commit is pinned and never goes out of + # date, no matter what the branch it came from does next. + if revision and _revision_pins_commit(revision, installed): + return None + + latest = "" + wanted_reference = f"refs/heads/{revision}" if revision else "HEAD" + for commit, reference in self._git_ls_remote(url, revision or "HEAD"): + # Tags are pinned installs too. + if reference.startswith("refs/tags/"): + return None + if reference == wanted_reference: + latest = commit + + if not latest or latest == installed: + return None + + requirement = f"{name} @ git+{url}" + if revision: + requirement += f"@{revision}" + + return { + "name": name, + "installed": installed[:SHORT_COMMIT_LENGTH], + "latest": latest[:SHORT_COMMIT_LENGTH], + "origin": "git", + # A repository URL and a branch name can both hold characters a + # shell would act on, so the requirement is quoted for the shell + # the analyst is going to paste the command into. + "upgrade_command": f"pip install -U {shlex.quote(requirement)}", + } + + def _check_distribution( + self, dist: importlib.metadata.Distribution + ) -> Optional[dict]: + try: + name = dist.name + installed = dist.version + except Exception as e: + log.debug("Failed to read the metadata of an installed plugin: %s", e) + return None + + direct_url = distribution_direct_url(dist) + if direct_url is None: + return self._check_index_plugin(name, installed) + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + return self._check_repository_plugin(name, direct_url, vcs_info) + + # Plugins installed from a local folder, including editable installs, + # are maintained by the analyst and have nothing to check against. + return None + + def check(self) -> list[dict]: + """ + Check every installed plugin package for an available update. + Returns one entry per plugin which can be upgraded. + """ + findings = [] + for dist in installed_plugin_distributions(): + finding = self._check_distribution(dist) + if finding: + findings.append(finding) + + self.set_findings(findings) + self.set_latest_check() + + return findings + + class IndicatorsUpdates: def __init__(self) -> None: self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}" @@ -180,9 +595,7 @@ class IndicatorsUpdates: def _get_remote_file_latest_commit( self, owner: str, repo: str, branch: str, path: str ) -> int: - file_commit_url = ( - f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" - ) + file_commit_url = f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" try: res = requests.get(file_commit_url, timeout=5) except requests.exceptions.RequestException as e: diff --git a/tests/common/test_cmd_plugins.py b/tests/common/test_cmd_plugins.py new file mode 100644 index 0000000..adf9322 --- /dev/null +++ b/tests/common/test_cmd_plugins.py @@ -0,0 +1,236 @@ +import json +from types import SimpleNamespace + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.cmd_plugins import plugins +from mvt.common.module import MVTModule +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import PluginUpdates +from mvt.ios.cli import cli as ios_cli + + +class ExampleModule(MVTModule): + pass + + +class AnotherModule(MVTModule): + pass + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +def _entry_point(name, distribution, modules=None, exception=None): + def load(): + if exception is not None: + raise exception + return modules + + return SimpleNamespace( + name=name, value="example_plugin:modules", dist=distribution, load=load + ) + + +def _run(command, arguments): + # Keep rich from wrapping the table while its content is being asserted. + return CliRunner().invoke(command, arguments, env={"COLUMNS": "200"}) + + +def _table_rows(output): + """Return the content of the table rows, without the header and the box.""" + return [ + [cell.strip() for cell in line.strip().strip("│").split("│")] + for line in output.splitlines() + if "│" in line + ] + + +def _table_header(output): + for line in output.splitlines(): + if "┃" in line: + return [cell.strip() for cell in line.strip().strip("┃").split("┃")] + return [] + + +def _install(monkeypatch, distributions, entry_points): + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: distributions, + ) + monkeypatch.setattr( + "mvt.common.cmd_plugins.importlib.metadata.entry_points", + lambda *, group: entry_points.get(group, []), + ) + + +def test_plugins_is_a_builtin_command_of_the_mvt_cli_only(): + assert mvt_cli.commands["plugins"] is plugins + assert "plugins" not in ios_cli.commands + assert "plugins" not in android_cli.commands + + +def test_list_shows_what_every_plugin_contributes(monkeypatch): + index_plugin = FakeDistribution("example-plugin", version="1.2.0") + repository_plugin = FakeDistribution( + "repository-plugin", + version="0.1.0", + direct_url={ + "url": "https://example.org/plugin.git", + "vcs_info": {"vcs": "git", "commit_id": "b" * 40}, + }, + ) + local_plugin = FakeDistribution( + "local-plugin", + direct_url={"url": "file:///plugins", "dir_info": {"editable": True}}, + ) + _install( + monkeypatch, + [index_plugin, local_plugin, repository_plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "example", index_plugin, modules=[ExampleModule, AnotherModule] + ), + _entry_point("local", local_plugin, modules=lambda: [ExampleModule]), + ], + IOS_CLI_PLUGIN_GROUP: [_entry_point("summarize", repository_plugin)], + ANDROID_CLI_PLUGIN_GROUP: [_entry_point("triage", local_plugin)], + NEUTRAL_CLI_PLUGIN_GROUP: [_entry_point("report", repository_plugin)], + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + # Plugins are listed by name, with the modules and the commands each of + # them contributes. + assert _table_header(result.output) == [ + "Name", + "Version", + "Origin", + "Modules", + "Commands", + ] + assert _table_rows(result.output) == [ + ["example-plugin", "1.2.0", "pypi", "2", "-"], + ["local-plugin", "1.0.0", "local", "1", "triage"], + ["repository-plugin", "0.1.0", "git+bbbbbbbb", "0", "report, summarize"], + ] + + +def test_list_reports_a_broken_module_entry_point(monkeypatch): + plugin = FakeDistribution("broken-plugin") + _install( + monkeypatch, + [plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "broken", plugin, exception=ImportError("missing dependency") + ) + ] + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert _table_rows(result.output) == [ + ["broken-plugin", "1.0.0", "pypi", "error", "-"] + ] + + +def test_list_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert result.output == "No MVT plugins are installed.\n" + + +def test_check_updates_prints_the_findings_and_ignores_the_throttle(monkeypatch): + findings = [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: findings) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("an explicit check must not be throttled"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Plugin updates available:" in result.output + assert "example-plugin 1.0.0 → 1.2.0" in result.output + assert "Upgrade with: pip install -U example-plugin" in result.output + assert "MVT does not install plugin updates." in result.output + + +def test_check_updates_without_available_updates(monkeypatch): + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: []) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "All plugins are up to date." in result.output + + +def test_check_updates_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without plugins"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "No MVT plugins are installed." in result.output + + +def test_check_updates_without_network_access(monkeypatch): + monkeypatch.setattr("mvt.common.cmd_plugins.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without network access"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Network access is disabled" in result.output diff --git a/tests/common/test_plugin_updates.py b/tests/common/test_plugin_updates.py new file mode 100644 index 0000000..9c45cea --- /dev/null +++ b/tests/common/test_plugin_updates.py @@ -0,0 +1,711 @@ +import json +import shlex +from datetime import datetime, timedelta +from types import SimpleNamespace + +import pytest + +from mvt.common import logo +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import ( + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) + + +REPOSITORY_URL = "https://example.org/plugin.git" +INSTALLED_COMMIT = "a" * 40 +REMOTE_COMMIT = "b" * 40 + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self.payload = payload or {} + + def json(self): + return self.payload + + +def _entry_point(name, distribution, value="plugin:modules"): + return SimpleNamespace(name=name, value=value, dist=distribution) + + +def _git_distribution(requested_revision=None, commit=INSTALLED_COMMIT): + vcs_info = {"vcs": "git", "commit_id": commit} + if requested_revision: + vcs_info["requested_revision"] = requested_revision + + return FakeDistribution( + "example-plugin", + direct_url={"url": REPOSITORY_URL, "vcs_info": vcs_info}, + ) + + +def _fake_git(stdout="", returncode=0, exception=None, calls=None): + def run(command, **kwargs): + if calls is not None: + calls.append((command, kwargs)) + if exception is not None: + raise exception + return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") + + return run + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "mvt-data" + monkeypatch.setattr("mvt.common.updates.MVT_DATA_FOLDER", str(folder)) + return folder + + +@pytest.fixture +def one_plugin(monkeypatch): + def install(distribution): + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [distribution], + ) + return distribution + + return install + + +def test_installed_plugin_distributions_covers_every_plugin_group(monkeypatch): + zeta = FakeDistribution("zeta-plugin") + alpha = FakeDistribution("alpha-plugin") + neutral = FakeDistribution("neutral-plugin") + + def entry_points(*, group): + if group == MODULES_ENTRY_POINT_GROUP: + return [_entry_point("zeta", zeta), _entry_point("alpha", alpha)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("zeta-ios", zeta)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [_entry_point("alpha-android", alpha)] + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared", neutral)] + return [] + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == [ + "alpha-plugin", + "neutral-plugin", + "zeta-plugin", + ] + + +def test_installed_plugin_distributions_skips_mvt_and_orphan_entry_points(monkeypatch): + entry_points = [ + _entry_point("builtin", FakeDistribution("mvt")), + SimpleNamespace(name="orphan", value="plugin:modules", dist=None), + _entry_point("plugin", FakeDistribution("example-plugin")), + ] + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", + lambda **kwargs: entry_points, + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == ["example-plugin"] + + +def test_installed_plugin_distributions_survives_broken_metadata(monkeypatch, caplog): + def entry_points(*, group): + raise RuntimeError("invalid package metadata") + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + assert installed_plugin_distributions() == [] + assert "Unable to discover installed plugin packages" in caplog.text + + +def test_index_plugin_update_is_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + + +def test_index_plugin_queries_the_package_index_with_the_configured_timeout( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin")) + requests_made = [] + + def get(url, **kwargs): + requests_made.append((url, kwargs)) + return FakeResponse(payload={"info": {"version": "1.0.0"}}) + + monkeypatch.setattr("mvt.common.updates.requests.get", get) + monkeypatch.setattr("mvt.common.updates.settings.NETWORK_TIMEOUT", 3) + + PluginUpdates().check() + + assert requests_made == [ + ("https://pypi.org/pypi/example-plugin/json", {"timeout": 3}) + ] + + +def test_up_to_date_index_plugin_is_not_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.2.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + assert PluginUpdates().check() == [] + + +def test_unpublished_plugin_is_skipped_silently(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("private-plugin")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(status_code=404), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_following_a_branch_is_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.delenv("GIT_SSH_COMMAND", raising=False) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@main'" + ), + } + ] + command, options = calls[0] + assert command == ["git", "ls-remote", REPOSITORY_URL, "main"] + assert options["env"]["GIT_TERMINAL_PROMPT"] == "0" + # ssh asks the terminal for a passphrase or a host key unless it is told + # not to, which git itself cannot prevent. + assert options["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10" + ) + + +def test_batch_mode_options_come_before_the_configured_ssh_options( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.setenv("GIT_SSH_COMMAND", "ssh -o BatchMode=no -i /home/analyst/key") + + PluginUpdates().check() + + # ssh uses the first value it is given for a keyword, so an analyst asking + # for prompts cannot bring them back, while their other options still + # apply. + assert calls[0][1]["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10 -o BatchMode=no -i /home/analyst/key" + ) + + +def test_repository_plugin_without_a_revision_follows_the_default_branch( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution()) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\tHEAD\n", calls=calls), + ) + + findings = PluginUpdates().check() + + assert calls[0][0] == ["git", "ls-remote", REPOSITORY_URL, "HEAD"] + assert findings[0]["upgrade_command"] == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}'" + ) + + +def test_hostile_revision_cannot_inject_into_the_upgrade_command( + monkeypatch, data_folder, one_plugin +): + revision = "main$(id)`id`;id" + one_plugin(_git_distribution(requested_revision=revision)) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/{revision}\n"), + ) + + upgrade_command = PluginUpdates().check()[0]["upgrade_command"] + + # Single quotes are the only quoting a shell does not expand anything in. + assert upgrade_command == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@{revision}'" + ) + assert shlex.split(upgrade_command) == [ + "pip", + "install", + "-U", + f"example-plugin @ git+{REPOSITORY_URL}@{revision}", + ] + + +def test_repository_plugin_with_an_option_like_url_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "--upload-pack=touch /tmp/mvt", + "vcs_info": {"vcs": "git", "commit_id": INSTALLED_COMMIT}, + }, + ) + ) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_with_an_option_like_revision_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="--upload-pack=touch /tmp/mvt")) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_at_the_latest_commit_is_not_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{INSTALLED_COMMIT}\trefs/heads/main\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT)) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_short_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT[:10])) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_tag_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="v1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/tags/v1.0.0\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_without_git(monkeypatch, data_folder, one_plugin): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(exception=FileNotFoundError("git")), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_when_git_fails( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout="", returncode=128), + ) + + assert PluginUpdates().check() == [] + + +def test_local_plugin_install_is_skipped(monkeypatch, data_folder, one_plugin): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "file:///home/analyst/example-plugin", + "dir_info": {"editable": True}, + }, + ) + ) + + def fail(*args, **kwargs): + raise AssertionError("a local plugin install must not be checked") + + monkeypatch.setattr("mvt.common.updates.requests.get", fail) + monkeypatch.setattr("mvt.common.updates.subprocess.run", fail) + + assert PluginUpdates().check() == [] + + +def test_check_stores_the_findings_and_the_check_timestamp( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + plugin_updates = PluginUpdates() + + findings = plugin_updates.check() + + assert json.loads((data_folder / "plugin_updates.json").read_text()) == findings + assert (data_folder / "latest_plugins_check").read_text().isdigit() + assert PluginUpdates().get_findings() == findings + + +def test_findings_are_empty_before_the_first_check(data_folder): + assert PluginUpdates().get_findings() == [] + + +def test_malformed_cached_findings_are_dropped(data_folder): + plugin_updates = PluginUpdates() + usable = { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps( + [ + {"oops": 1}, + "not a finding", + {"name": "half-plugin", "installed": "1.0.0"}, + {**usable, "latest": None}, + usable, + ] + ), + encoding="utf-8", + ) + + assert plugin_updates.get_findings() == [usable] + + +def test_corrupt_cached_findings_are_ignored(data_folder): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text("{ not json", encoding="utf-8") + + assert PluginUpdates().get_findings() == [] + + +def test_cached_findings_of_upgraded_and_removed_plugins_are_dropped( + monkeypatch, data_folder +): + findings = [ + { + "name": "upgraded-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U upgraded-plugin", + }, + { + "name": "removed-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U removed-plugin", + }, + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + }, + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [ + # The analyst upgraded this plugin since the latest check. + FakeDistribution("upgraded-plugin", version="1.2.0"), + FakeDistribution("example-plugin", version="1.0.0"), + ], + ) + + assert plugin_updates.current_findings() == [findings[2]] + + +def test_cached_findings_of_updated_repository_plugins_are_dropped(data_folder): + findings = [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": "pip install -U example-plugin", + } + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + + assert plugin_updates.current_findings([_git_distribution()]) == findings + assert ( + plugin_updates.current_findings([_git_distribution(commit=REMOTE_COMMIT)]) == [] + ) + + +def test_corrupt_check_timestamp_does_not_raise(data_folder): + plugin_updates = PluginUpdates() + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "latest_plugins_check").write_text("truncated", encoding="utf-8") + + assert plugin_updates.get_latest_check() == 0 + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_is_throttled_for_twelve_hours(data_folder): + plugin_updates = PluginUpdates() + plugin_updates.set_findings([]) + + recent = datetime.now() - timedelta(hours=4) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(recent.timestamp()))) + + should_check, hours = plugin_updates.should_check() + assert not should_check + assert hours == 8 + + old = datetime.now() - timedelta(hours=13) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(old.timestamp()))) + + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_without_a_previous_check(data_folder): + assert PluginUpdates().should_check() == (True, 0) + + +@pytest.fixture +def no_version_check(monkeypatch): + monkeypatch.setattr(MVTUpdates, "check", lambda self: "") + # Keep rich from wrapping the plugin lines while they are being asserted. + monkeypatch.setenv("COLUMNS", "200") + + +@pytest.fixture +def throttled_cache(monkeypatch, data_folder): + """Fill the findings cache and put the check inside its throttle window.""" + + def fill(findings, distributions): + PluginUpdates().set_findings(findings) + monkeypatch.setattr( + logo, "installed_plugin_distributions", lambda: distributions + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("the check must be throttled"), + ) + + return fill + + +def test_logo_prints_the_cached_plugin_updates( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + [FakeDistribution("example-plugin", version="1.0.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + output = capsys.readouterr().out + assert "Plugin updates available:" in output + assert "example-plugin 1.0.0 → 1.2.0 (pip install -U example-plugin)" in output + + +def test_logo_does_not_print_a_cached_update_of_an_upgraded_plugin( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + # The analyst already upgraded the plugin the cached finding is about. + [FakeDistribution("example-plugin", version="1.2.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_prints_nothing_when_throttled_without_findings( + capsys, no_version_check, throttled_cache +): + throttled_cache([], [FakeDistribution("example-plugin")]) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_survives_a_corrupt_plugin_cache( + monkeypatch, capsys, data_folder, no_version_check +): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps([{"oops": 1}]), encoding="utf-8" + ) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: [FakeDistribution("example-plugin")], + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_plugins( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr(logo, "installed_plugin_distributions", list) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("plugins must not be checked without plugins"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_network_access( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr("mvt.common.logo.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_when_update_checks_are_disabled( + monkeypatch, capsys +): + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be checked with --disable-update-check"), + ) + + logo.check_updates(disable_version_check=True, disable_indicator_check=True) + + assert capsys.readouterr().out == "" From 91da90174166a84f7e8f6682385abe1904e89093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:16 +0200 Subject: [PATCH 34/43] Find the console log handler by type when changing verbosity (#899) * Find the console log handler by type when changing verbosity set_verbose_logging() adjusted the first handler on the "mvt" logger, whichever handler that happened to be. Anything else attaching a handler to that logger - an embedding application, a plugin, a test harness - had it mistaken for the console and raised or lowered behind its back, and the same slot can hold the file handler a command attaches to its output folder, whose level a --verbose flag should never decide: command.log records the whole run either way. Walk the handlers instead and adjust only MVT's own console handler, found by its MVTLogHandler type. Every other handler on the logger is left alone. Behaviour is otherwise unchanged. * Add a --verbose option to the mvt, mvt-ios and mvt-android commands Verbosity was an option of each module-running command, so "mvt-ios --verbose check-backup" was a usage error, a plugin command had to define a flag of its own, and there was no way to get debug output from mvt at all. The option now sits on the three commands themselves and sets the level of MVT's console handler for the run, through set_verbose_logging(), before any command runs. Plugin commands registered on any of the three CLIs get the option for free and need none of their own. The per-command --verbose of the check-* commands is kept for backward compatibility. It only ever raises the level, so the CLI's choice is never undone by a command's default, and its help text says it is kept for compatibility. It is to be removed in a later release. --- README.md | 2 + docs/android/intrusion_logs.md | 2 +- src/mvt/android/cli.py | 27 ++++++--- src/mvt/cli.py | 8 ++- src/mvt/common/help.py | 4 ++ src/mvt/common/utils.py | 16 +++-- src/mvt/ios/cli.py | 23 ++++--- tests/common/test_utils.py | 27 +++++++++ tests/test_cli_verbose.py | 107 +++++++++++++++++++++++++++++++++ 9 files changed, 192 insertions(+), 24 deletions(-) create mode 100644 tests/test_cli_verbose.py diff --git a/README.md b/README.md index 54cccca..98e90e3 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ For alternative installation options and known issues, please refer to the [docu MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +Pass `--verbose` to any of the three commands, before the command name (`mvt-ios --verbose check-backup ...`), for debug output. The `--verbose` option the `check-*` commands accept after their name still works but is kept for compatibility only and will be removed in a future release. + ### Shell completion MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`: diff --git a/docs/android/intrusion_logs.md b/docs/android/intrusion_logs.md index da8fc83..16c6bf1 100644 --- a/docs/android/intrusion_logs.md +++ b/docs/android/intrusion_logs.md @@ -58,7 +58,7 @@ mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-l | `-l, --list-modules` | List the available intrusion-log modules and exit. | | `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. | | `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. | -| `-v, --verbose` | Verbose logging. | +| `-v, --verbose` | Verbose logging. Kept for compatibility and to be removed in a future release: pass `--verbose` to `mvt-android` itself instead. | ## Modules diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index 4716168..c32a7b2 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -36,6 +36,7 @@ from mvt.common.help import ( HELP_MSG_OUTPUT, HELP_MSG_STIX2, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_VERSION, HELP_MSG_VIRUS_TOTAL, ) @@ -70,6 +71,11 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): try: return load_custom_modules(load_module) @@ -90,11 +96,14 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -145,7 +154,7 @@ def check_adb(ctx): default=[], help=HELP_MSG_LOAD_MODULE, ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BUGREPORT_PATH", type=click.Path(exists=True)) @click.pass_context def check_bugreport( @@ -158,7 +167,7 @@ def check_bugreport( verbose, bugreport_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. cmd = CmdAndroidCheckBugreport( @@ -213,7 +222,7 @@ def check_bugreport( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -227,7 +236,7 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as backups are generally small. @@ -287,7 +296,7 @@ def check_backup( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("ANDROIDQF_PATH", type=click.Path(exists=True)) @click.pass_context def check_androidqf( @@ -305,7 +314,7 @@ def check_androidqf( verbose, androidqf_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdAndroidCheckAndroidQF( @@ -373,7 +382,7 @@ def check_androidqf( "time instead of UTC." ), ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("LOGS_PATH", type=click.Path(exists=True)) @click.pass_context def check_intrusion_logs( @@ -387,7 +396,7 @@ def check_intrusion_logs( verbose, logs_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) module_options = {} diff --git a/src/mvt/cli.py b/src/mvt/cli.py index 3ba6602..40a86fe 100644 --- a/src/mvt/cli.py +++ b/src/mvt/cli.py @@ -17,11 +17,12 @@ from mvt.common.help import ( HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_STIX2, + HELP_MSG_VERBOSE, HELP_MSG_VERSION, ) from mvt.common.logo import logo from mvt.common.updates import IndicatorsUpdates -from mvt.common.utils import init_logging +from mvt.common.utils import init_logging, set_verbose_logging init_logging() @@ -41,8 +42,9 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): """Mobile Verification Toolkit. mvt-ios and mvt-android run the forensic analysis of an acquisition: each @@ -53,6 +55,8 @@ def cli(ctx, disable_update_check, disable_indicator_update_check): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) if ctx.invoked_subcommand != "completion": logo( disable_version_check=disable_update_check, diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 13575e5..8b70f20 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -17,6 +17,10 @@ HELP_MSG_LOAD_MODULE = ( HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing" HELP_MSG_HASHES = "Generate hashes of all the files analyzed" HELP_MSG_VERBOSE = "Verbose mode" +HELP_MSG_VERBOSE_COMMAND = ( + "Verbose mode (kept for compatibility, pass --verbose before the command " + "name instead)" +) HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators" HELP_MSG_STIX2 = "Download public STIX2 indicators" HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check" diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index ad3b394..b71c4ab 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -256,12 +256,18 @@ def init_logging(verbose: bool = False): def set_verbose_logging(verbose: bool = False): + """Raise or lower the verbosity of MVT's console output. + + Only MVT's own console handler is adjusted, wherever it sits in the list. + The file handler a command attaches to its output folder keeps recording + everything, so the command.log of a run does not depend on how the run was + invoked, and a handler attached to the "mvt" logger by anything else is + left alone. + """ log = logging.getLogger("mvt") - handler = log.handlers[0] - if verbose: - handler.setLevel(logging.DEBUG) - else: - handler.setLevel(logging.INFO) + for handler in log.handlers: + if isinstance(handler, MVTLogHandler): + handler.setLevel(logging.DEBUG if verbose else logging.INFO) def exec_or_profile(module, globals, locals): diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 6f633ea..35acec1 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -38,6 +38,7 @@ from mvt.common.help import ( HELP_MSG_LOAD_MODULE, HELP_MSG_MODULE, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_CHECK_FS, HELP_MSG_CHECK_IOCS, HELP_MSG_STIX2, @@ -74,6 +75,11 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): try: return load_custom_modules(load_module) @@ -94,11 +100,14 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -254,7 +263,7 @@ def extract_key(password, key_file, backup_path): help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -269,7 +278,7 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -323,7 +332,7 @@ def check_backup( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("DUMP_PATH", type=click.Path(exists=True)) @click.pass_context def check_fs( @@ -338,7 +347,7 @@ def check_fs( verbose, dump_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -392,7 +401,7 @@ def check_fs( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True)) @click.pass_context def check_sysdiagnose( @@ -406,7 +415,7 @@ def check_sysdiagnose( verbose, sysdiagnose_path, ): - set_verbose_logging(verbose) + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckSysdiagnose( target_path=sysdiagnose_path, diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index b8791e1..6bef147 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -18,6 +18,7 @@ from mvt.common.utils import ( generate_hashes_from_path, get_sha256_from_file_path, init_logging, + set_verbose_logging, ) from ..utils import get_artifact_folder @@ -122,3 +123,29 @@ class TestInitLogging: sum(isinstance(handler, MVTLogHandler) for handler in log.handlers) == handler_count ) + + def test_verbose_logging_finds_the_console_handler_among_others(self): + # Something else may have attached a handler to the "mvt" logger + # before MVT did, so the console handler is not always the first. + log = logging.getLogger("mvt") + init_logging() + foreign_handler = logging.NullHandler() + foreign_handler.setLevel(logging.CRITICAL) + log.handlers.insert(0, foreign_handler) + + try: + set_verbose_logging(True) + console_handlers = [ + handler + for handler in log.handlers + if isinstance(handler, MVTLogHandler) + ] + assert console_handlers + assert all(handler.level == logging.DEBUG for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + + set_verbose_logging(False) + assert all(handler.level == logging.INFO for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + finally: + log.handlers.remove(foreign_handler) diff --git a/tests/test_cli_verbose.py b/tests/test_cli_verbose.py new file mode 100644 index 0000000..d8826ff --- /dev/null +++ b/tests/test_cli_verbose.py @@ -0,0 +1,107 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.log import MVTLogHandler +from mvt.common.utils import set_verbose_logging +from mvt.ios.cli import cli as ios_cli + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + +PROGRAMS = {"mvt": mvt_cli, "mvt-ios": ios_cli, "mvt-android": android_cli} + + +@pytest.fixture(autouse=True) +def _reset_console_level(): + """Leave the console handler at its default level after every test.""" + yield + set_verbose_logging(False) + + +def _console_level(): + """Return the level of MVT's own console log handler.""" + for handler in logging.getLogger("mvt").handlers: + if isinstance(handler, MVTLogHandler): + return handler.level + raise AssertionError("MVT has no console log handler") + + +class TestVerboseOnTheCommands: + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_verbose_before_the_command_name_turns_on_debug(self, program): + cli = PROGRAMS[program] + + result = CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_a_run_without_verbose_goes_back_to_info(self, program): + cli = PROGRAMS[program] + CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.INFO + + def test_mvt_verbose_without_a_command_prints_the_help(self): + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "--verbose"]) + + assert result.exit_code == 0 + assert "Usage:" in result.output + assert _console_level() == logging.DEBUG + + +class TestVerboseOnTheCheckCommands: + def test_ios_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "--verbose", "check-backup", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_ios_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "check-backup", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "--verbose", "check-bugreport", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "check-bugreport", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_the_command_option_says_it_is_kept_for_compatibility(self): + result = CliRunner().invoke(ios_cli, [*OFFLINE, "check-backup", "--help"]) + + assert result.exit_code == 0 + assert "kept for compatibility" in result.output From 85adb02eb994215cf72547713fca0bcef4b2a5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:16 +0200 Subject: [PATCH 35/43] Share the check-iocs module lists between the CLI and the code (#900) * Share the check-iocs module lists between the CLI and the code check-iocs re-checks the results a previous run stored, so its module list is every module of the platform that could have written one. Each platform's CLI composed that list inline, concatenating the families by hand, so the list existed only inside the click callback: anything else needing to know what check-iocs runs had to build its own copy, and the two could drift apart without a test noticing. Give each platform a command_modules.py holding the one list, and have its CLI assign it. The modules check-iocs runs are unchanged, and a test pins each list to the families it is composed of. * Pin that check-iocs re-checks the results of custom modules check-iocs takes its custom modules from load_custom_modules() like every check-* command and matches result files to modules by slug, so a plugin module's stored results are re-checked whenever it declares the check-iocs pair of its platform; nothing asserted it. --- src/mvt/android/cli.py | 9 +- src/mvt/android/command_modules.py | 23 +++++ src/mvt/ios/cli.py | 6 +- src/mvt/ios/command_modules.py | 22 +++++ tests/common/test_cmd_check_iocs.py | 139 +++++++++++++++++++++++++++ tests/common/test_command_modules.py | 27 ++++++ 6 files changed, 215 insertions(+), 11 deletions(-) create mode 100644 src/mvt/android/command_modules.py create mode 100644 src/mvt/ios/command_modules.py create mode 100644 tests/common/test_cmd_check_iocs.py create mode 100644 tests/common/test_command_modules.py diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index c32a7b2..92b2fe9 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -49,11 +49,8 @@ from .cmd_check_androidqf import CmdAndroidCheckAndroidQF from .cmd_check_backup import CmdAndroidCheckBackup from .cmd_check_bugreport import CmdAndroidCheckBugreport from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from .modules.intrusion_logs import INTRUSION_LOGS_MODULES -from .modules.androidqf import ANDROIDQF_MODULES -from .modules.backup import BACKUP_MODULES +from .command_modules import ANDROID_CHECK_IOCS_MODULES from .modules.backup.helpers import cli_load_android_backup_password -from .modules.bugreport import BUGREPORT_MODULES init_logging() log = logging.getLogger("mvt") @@ -459,9 +456,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="android", ) - cmd.modules = ( - BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES - ) + cmd.modules = ANDROID_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() diff --git a/src/mvt/android/command_modules.py b/src/mvt/android/command_modules.py new file mode 100644 index 0000000..9b6b56e --- /dev/null +++ b/src/mvt/android/command_modules.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-android command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.androidqf import ANDROIDQF_MODULES +from .modules.backup import BACKUP_MODULES +from .modules.bugreport import BUGREPORT_MODULES +from .modules.intrusion_logs import INTRUSION_LOGS_MODULES + +ANDROID_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES +) diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 35acec1..8251fd9 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -53,9 +53,7 @@ from .cmd_check_backup import CmdIOSCheckBackup from .cmd_check_fs import CmdIOSCheckFS from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose from .decrypt import DecryptBackup -from .modules.backup import BACKUP_MODULES -from .modules.fs import FS_MODULES -from .modules.mixed import MIXED_MODULES +from .command_modules import IOS_CHECK_IOCS_MODULES init_logging() log = logging.getLogger("mvt") @@ -479,7 +477,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="ios", ) - cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES + cmd.modules = IOS_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() diff --git a/src/mvt/ios/command_modules.py b/src/mvt/ios/command_modules.py new file mode 100644 index 0000000..26fa1d6 --- /dev/null +++ b/src/mvt/ios/command_modules.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-ios command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.backup import BACKUP_MODULES +from .modules.fs import FS_MODULES +from .modules.mixed import MIXED_MODULES + +IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + FS_MODULES + MIXED_MODULES +) diff --git a/tests/common/test_cmd_check_iocs.py b/tests/common/test_cmd_check_iocs.py new file mode 100644 index 0000000..1bdd232 --- /dev/null +++ b/tests/common/test_cmd_check_iocs.py @@ -0,0 +1,139 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import json +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.common.cmd_check_iocs import CmdCheckIOCS +from mvt.common.module import MVTModule +from mvt.ios.cli import cli as ios_cli +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class CustomResultsModule(MVTModule): + """A custom module which declares the check-iocs pair of both platforms.""" + + slug = "custom_results" + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-iocs"), + ("android", "check-iocs"), + ) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BackupOnlyModule(MVTModule): + """A custom module which does not declare check-iocs.""" + + slug = "backup_only" + supported_commands = (("ios", "check-backup"),) + + def check_indicators(self) -> None: + raise AssertionError("must not be re-checked") + + +@pytest.mark.parametrize( + "platform, builtin_modules", + [("ios", IOS_CHECK_IOCS_MODULES), ("android", ANDROID_CHECK_IOCS_MODULES)], +) +def test_check_iocs_rechecks_the_stored_results_of_custom_modules( + platform, builtin_modules, tmp_path, caplog +): + # check-iocs matches every .json in the results folder to the module + # with that slug, custom modules included, and runs its check_indicators() + # again over the stored results. + results = [{"domain": "example.org"}] + (tmp_path / "custom_results.json").write_text(json.dumps(results)) + (tmp_path / "backup_only.json").write_text(json.dumps(results)) + CustomResultsModule.checked.clear() + + cmd = CmdCheckIOCS( + target_path=str(tmp_path), + custom_modules=[CustomResultsModule, BackupOnlyModule], + platform=platform, + ) + cmd.modules = builtin_modules + + with caplog.at_level(logging.INFO): + cmd.run() + + assert CustomResultsModule.checked == [results] + assert ( + 'Loading results from "custom_results.json" with module CustomResultsModule' + in caplog.text + ) + # A module declaring only check-backup is not part of check-iocs. + assert "backup_only.json" not in caplog.text + + +def test_check_iocs_lists_custom_modules_declaring_the_command(caplog): + cmd = CmdCheckIOCS( + custom_modules=[CustomResultsModule, BackupOnlyModule], + platform="ios", + ) + cmd.modules = IOS_CHECK_IOCS_MODULES + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + assert "CustomResultsModule" in caplog.text + assert "BackupOnlyModule" not in caplog.text + + +LOADED_MODULE = ''' +from mvt.common.module import MVTModule + + +class LoadedResultsModule(MVTModule): + """A module loaded from a file with --load-module.""" + + slug = "loaded_results" + supported_commands = (("ios", "check-iocs"), ("android", "check-iocs")) + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.log.warning("loaded module checked %d results", len(self.results)) +''' + + +@pytest.mark.parametrize("cli", [ios_cli, android_cli], ids=["mvt-ios", "mvt-android"]) +def test_check_iocs_loads_custom_modules_from_a_file_on_each_cli(cli, tmp_path, caplog): + module_path = tmp_path / "loaded_module.py" + module_path.write_text(LOADED_MODULE) + results_folder = tmp_path / "results" + results_folder.mkdir() + (results_folder / "loaded_results.json").write_text(json.dumps([{"a": 1}])) + + with caplog.at_level(logging.INFO): + result = CliRunner().invoke( + cli, + [ + *OFFLINE, + "check-iocs", + "--load-module", + str(module_path), + str(results_folder), + ], + ) + + assert result.exit_code == 0, result.output + assert "loaded module checked 1 results" in caplog.text diff --git a/tests/common/test_command_modules.py b/tests/common/test_command_modules.py new file mode 100644 index 0000000..8f79ce9 --- /dev/null +++ b/tests/common/test_command_modules.py @@ -0,0 +1,27 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.android.modules.androidqf import ANDROIDQF_MODULES +from mvt.android.modules.backup import BACKUP_MODULES as ANDROID_BACKUP_MODULES +from mvt.android.modules.bugreport import BUGREPORT_MODULES +from mvt.android.modules.intrusion_logs import INTRUSION_LOGS_MODULES +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES +from mvt.ios.modules.fs import FS_MODULES +from mvt.ios.modules.mixed import MIXED_MODULES + + +def test_the_check_iocs_lists_are_the_families_of_their_platform(): + # The CLI reads these same lists, so nothing composing one elsewhere can + # drift from what the command runs. This pins what the lists are composed + # of. + assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES + assert ANDROID_CHECK_IOCS_MODULES == ( + ANDROID_BACKUP_MODULES + + BUGREPORT_MODULES + + ANDROIDQF_MODULES + + INTRUSION_LOGS_MODULES + ) From 0b5b3f2d7c41fd84139a694fb4e973b779640400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:16 +0200 Subject: [PATCH 36/43] Add the mvt.plugin import surface (#901) * Add the mvt.plugin import surface mvt.plugin re-exports the names a plugin needs from MVT under one import path. It holds the module base classes and Command, the alert and result types, the database errors a module raises, the timestamp converters, the plugin settings API, MVT's settings, get_plugin_logger() and MVT_VERSION. The names it exports are kept working on a best-effort basis. Changes to them are announced in the release notes. Anything else in mvt can still be imported, and may change between releases without notice. get_plugin_logger(__name__) returns a logger under mvt.ext for plugin code outside a module class. Its records then reach the console and the command.log file of a run. A file loaded with --load-module or --load-command is named after the file. * Document how to write MVT plugins The custom modules page now leads with plugin packages. Loading module files with --load-module and MVT_CUSTOM_MODULES moves to a section on developing a module locally. A new "Writing a module" section shows a module which subclasses IOSExtraction. It lists each base class, the command pair it serves and the helpers it provides. "Depending on a built-in module" says to import a built-in class from its family package. "Importing from MVT" says what mvt.plugin exports and what importing from it means. The custom commands page shows a Command subclass which lists its own modules. The sysdiagnose and plugin configuration pages import from mvt.plugin. --- docs/development/custom_commands.md | 62 +++++++ docs/development/index.md | 214 +++++++++++++++++------ docs/development/plugin_configuration.md | 22 ++- docs/ios/sysdiagnose.md | 32 ++-- src/mvt/common/cli_plugins.py | 4 +- src/mvt/common/module_loader.py | 57 +++++- src/mvt/plugin.py | 80 +++++++++ tests/common/test_module_loader.py | 31 +++- tests/common/test_plugin.py | 34 ++++ tests/plugin_fixtures.py | 1 + 10 files changed, 448 insertions(+), 89 deletions(-) create mode 100644 src/mvt/plugin.py create mode 100644 tests/common/test_plugin.py diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md index 71ac623..3161cef 100644 --- a/docs/development/custom_commands.md +++ b/docs/development/custom_commands.md @@ -29,6 +29,10 @@ def summarize(path): click.echo(f"Summarizing {path}") ``` +Log through `get_plugin_logger(__name__)` from `mvt.plugin`. Records logged +through `logging.getLogger(__name__)` do not reach `command.log`, and MVT's +console handler does not show them. + Register the object in the package's `pyproject.toml`. The entry-point name is the command users invoke: @@ -125,6 +129,64 @@ export MVT_IOS_CUSTOM_COMMANDS=./ios_commands export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands ``` +## Building a Module-Running Command + +A command which runs forensic modules over an acquisition subclasses `Command`. +`Command` creates the output folder and writes `command.log`. It orders the +modules, resolves their dependencies and runs them. It writes the result files, +`alerts.json` and `info.json`. The subclass sets `platform`, `name` and +`modules`: + +```python +from mvt.plugin import Command, MVTModule, convert_unix_to_iso, get_plugin_logger + +log = get_plugin_logger(__name__) + + +class APKManifest(MVTModule): + supported_commands = (("android", "check-apks"),) + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + +class CmdCheckAPKs(Command): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.platform = "android" + self.name = "check-apks" + self.modules = [APKManifest] +``` + +`platform` and `name` are the pair the modules declare in +`supported_commands`. `modules` lists the module classes the command runs, +imported directly. A command which runs modules of another plugin depends on +that package in `pyproject.toml` and imports them the same way. `init()`, +`module_init(module)` and `finish()` are optional hooks. `run()` calls them +before the run, before each module and after the run. + +Wrap the command in a Click command with an `--output` option. Run it, then +print the alert summary: + +```python +import click + + +@click.command("check-apks") +@click.option("--output", "-o", type=click.Path(exists=False)) +@click.argument("TARGET_PATH", type=click.Path(exists=True)) +def cli(output, target_path): + cmd = CmdCheckAPKs(target_path=target_path, results_path=output) + log.info("Checking APK files at path: %s", target_path) + cmd.run() + cmd.show_alerts_brief() +``` + +The `--verbose` option of `mvt`, `mvt-ios` and `mvt-android` applies to the +command. The command defines no `--verbose` option of its own. The pair a +plugin command adds is not listed anywhere in MVT. Name it in the plugin's +README. + ## Naming and Errors Built-in MVT commands cannot be replaced. External command names must also be diff --git a/docs/development/index.md b/docs/development/index.md index 692a2ab..d1c167f 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -48,51 +48,21 @@ configuration problem: the command logs a warning and runs no modules at all. ## Custom modules -Module-running `check-*` commands can load custom modules from Python files that -are not installed as part of MVT. Load one file with: +MVT's module-running `check-*` commands can run forensic modules which are not +part of MVT. Custom modules are distributed as plugin packages. A Python +package installed next to MVT registers its modules through an entry point. +The modules then load automatically in every command they support, see +[Installed module packages](#installed-module-packages). `mvt plugins list` +shows the installed packages and where each one was installed from. -```bash -mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup -``` +MVT can also load module files by path, with `--load-module` and +`MVT_CUSTOM_MODULES`, see +[Developing modules locally](#developing-modules-locally). This can be used +while writing a module. Use a Python package to distribute one. -You can also load a folder. MVT loads non-hidden top-level `*.py` files in -sorted order and skips `__init__.py`: - -```bash -mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump -``` - -Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This -folder is loaded before any `--load-module` path: - -```bash -MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip -``` - -Custom modules are normal `MVTModule` subclasses: - -```python -from mvt.common.module import MVTModule - - -class ExampleCustomModule(MVTModule): - supported_commands = (("ios", "check-backup"), ("ios", "check-fs")) - slug = "example_custom_module" - - def run(self): - self.results = [{"message": "custom module ran"}] - - def check_indicators(self): - pass - - def serialize(self, result): - return None -``` - -Use `supported_commands` to declare the platform/command pairs a module -supports. Empty `supported_commands` means the module will not run and MVT logs -a warning. This explicit declaration is required for every command. Supported -pairs are: +A custom module declares in `supported_commands` the platform and command +pairs it runs in. A module with empty `supported_commands` does not run and +MVT logs a warning. The nine pairs are: ```python ("ios", "check-backup") @@ -106,13 +76,86 @@ pairs are: ("android", "check-iocs") ``` -Custom modules can depend on existing MVT module classes. Dependencies are -resolved with the same ordering logic as built-in modules, and custom modules -are appended after built-ins before ordering: +`check-iocs` re-checks stored results rather than an acquisition. It matches +every `.json` file in the results folder to the module with that slug. +It then runs that module's `check_indicators()` again. + +### Writing a module + +A module subclasses `MVTModule` or one of the base classes below and +implements `run()`. `check_indicators()` and `serialize()` are optional. The +first matches results against IOCs or detections. The second returns timeline +records. ```python -from mvt.common.module import MVTModule -from mvt.ios.modules.backup.manifest import Manifest +from mvt.plugin import IOSExtraction, convert_unix_to_iso + + +class ExampleCustomModule(IOSExtraction): + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-fs"), + ) + slug = "example_custom_module" + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + def check_indicators(self): + pass + + def serialize(self, result): + return None +``` + +The base classes are: + +- `MVTModule`: the base of every module. It provides `self.results`, + `self.alertstore`, `self.log`, `self.indicators` and + `get_dependency_results()`. Subclass it directly for a module which reads + only the results of other modules. +- `IOSExtraction`: `("ios", "check-backup")` and `("ios", "check-fs")`. Adds + `_find_ios_database()`, which locates a module's database in a backup or in + a filesystem dump and repairs it if it is malformed. Adds + `_get_backup_files_from_manifest()`, `_get_backup_file_from_id()` and + `_get_fs_files_from_patterns()`. Adds `_open_sqlite_db()`, which opens a + database read-only. +- `SysdiagnoseExtraction`: `("ios", "check-sysdiagnose")`. MVT extracts the + archive and calls `from_sysdiagnose_folder()` before `run()`. The module + reads files with `_get_files_by_pattern()` and `_get_file_content()`. + `ips_files` lists the crash reports. See + [Check an iOS Sysdiagnose](../ios/sysdiagnose.md). +- `AndroidQFModule`: `("android", "check-androidqf")`. MVT calls `from_dir()` + or `from_zip()` with the file list of the acquisition. The module reads + files with `_get_files_by_pattern()` and `_get_file_content()`. + `_get_device_timezone()` returns the device timezone. +- `AndroidBackupModule`: `("android", "check-backup")`. MVT calls `from_dir()` + or `from_ab()`. The module reads files with `_get_files_by_pattern()` and + `_get_file_content()`. +- `BugReportModule`: `("android", "check-bugreport")`. MVT calls `from_dir()` + or `from_zip()`. The module reads files with `_get_files_by_pattern()`, + `_get_files_by_patterns()` and `_get_file_content()`. + `_get_dumpstate_file()` returns the dumpstate file, and + `_get_file_modification_time()` the modification time of a file. + +The underscore-named helpers are internal to the base classes. Plugin modules +can call them. Their names and signatures can change between releases. Read +the base class in `src/mvt/ios/modules` or `src/mvt/android/modules` before +relying on one. + +### Depending on a built-in module + +A module which post-processes records generated by one or more built-in MVT +modules must declare the source modules in `dependencies`. It reads their +results with `get_dependency_results()`. Import the class from its family +package: `mvt.ios.modules.backup`, `mvt.ios.modules.fs`, +`mvt.ios.modules.mixed`, `mvt.android.modules.androidqf`, +`mvt.android.modules.backup`, `mvt.android.modules.bugreport` or +`mvt.android.modules.intrusion_logs`. + +```python +from mvt.ios.modules.backup import Manifest +from mvt.plugin import MVTModule class DependentCustomModule(MVTModule): @@ -124,6 +167,30 @@ class DependentCustomModule(MVTModule): self.results = [{"manifest_entries": len(manifest_results)}] ``` +Dependencies are ordered as for the built-in modules, with custom modules +appended after the built-ins, see [Module dependencies](#module-dependencies). +A dependency has to run in every command the module supports. Where it does +not, MVT skips the module with a warning. + +`get_dependency_results()` returns the plain dictionaries the module produced. +They are the same records it writes to `.json`. Typed results per module +are planned. + +### Importing from MVT + +Import from `mvt.plugin` if it has what you need. The names it exports are +kept working on a best-effort basis. A change to one of them is announced in +the release notes. Anything else in `mvt` can be imported too, but may change +between releases without notice. The plugin interface is best effort. + +`mvt.plugin` exports the base classes above and `Command`, `Alert` and +`AlertLevel`, the result types, `DatabaseNotFoundError` and +`DatabaseCorruptedError`, the timestamp converters, the settings API of +[Plugin Configuration](plugin_configuration.md), MVT's own `settings`, +`get_plugin_logger()` and `MVT_VERSION`. `src/mvt/plugin.py` holds the list. +Read MVT's `settings` for values such as `NETWORK_ACCESS_ALLOWED` and +`NETWORK_TIMEOUT`. Plugin values go in the plugin's own settings file. + ## Installed module packages Python packages can register modules so they load automatically in every @@ -133,14 +200,14 @@ the package's `pyproject.toml`: ```toml [project.entry-points."mvt.modules"] -mvt-plugin-amnesty-custom = "mvt_plugin_amnesty_custom:get_modules" +mvt-plugin-example-org = "mvt_plugin_example_org:get_modules" ``` The entry point must resolve to an iterable of `MVTModule` subclasses, or to a callable returning one: ```python -from mvt.common.module import MVTModule +from mvt.plugin import MVTModule class PackagedModule(MVTModule): @@ -154,6 +221,10 @@ def get_modules() -> list[type[MVTModule]]: return [PackagedModule] ``` +`get_modules()` is the package's module list, written by hand. A package which +keeps its modules in separate files imports each class there and lists it. A +module missing from the list does not load. + Installed modules follow the same rules as other custom modules: each module must declare `supported_commands`, and dependencies are resolved with the standard ordering logic. A broken entry point is skipped with a warning and @@ -164,7 +235,7 @@ from sources you trust. For a `pipx` installation of MVT, inject the package into MVT's environment: ```bash -pipx inject mvt mvt-plugin-amnesty-custom +pipx inject mvt mvt-plugin-example-org ``` Module packages that need their own settings, such as an API key, should store @@ -175,9 +246,9 @@ rather than in MVT's own `config.yaml`. Name module packages `mvt-plugin-` (import package `mvt_plugin_`), and include the name of the publishing organization or author so packages from -different groups do not collide: for example, Amnesty International's custom -modules would be distributed as `mvt-plugin-amnesty-custom` with the import -package `mvt_plugin_amnesty_custom`. +different groups do not collide: for example, an organisation's custom modules +would be distributed as `mvt-plugin-example-org` with the import package +`mvt_plugin_example_org`. The prefix makes module packages easy to find on PyPI and keeps their import names from clashing with unrelated Python packages. It is a convention, not a @@ -196,11 +267,42 @@ came from. MVT's own modules log under their dotted path (for example MVT's internal logger tree: - Installed packages log under `mvt.ext.`, with the `mvt_plugin_` - prefix stripped: modules in `mvt_plugin_amnesty_custom` log as - `mvt.ext.amnesty_custom.*`. + prefix stripped: modules in `mvt_plugin_example_org` log as + `mvt.ext.example_org.*`. - Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as `mvt.ext.`. +Outside a module class, for example in a custom command line handler, log +through `get_plugin_logger(__name__)`. It returns a logger in the same +namespace. + +## Developing modules locally + +While a module is being written, load it from its file. `--load-module` takes a +Python file, or a folder of them, on every module-running command, and can be +repeated: + +```bash +mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup +``` + +For a folder, MVT loads its non-hidden top-level `*.py` files in sorted order +and skips `__init__.py`. `MVT_CUSTOM_MODULES` names a folder to load on every +module-running command, before any `--load-module` path: + +```bash +MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip +``` + +Files loaded this way follow the same rules as packaged modules. +`--list-modules` reports them with the SHA-256 hash of the file in place of a +version. An editable install of the package (`pip install -e .`) also works: +the modules load through the entry point, and `mvt plugins list` shows the +package with the `local` origin. + +Loading by path is for development. Move a module into a package once it +works. + ## Auditing loaded modules Because installed module packages load automatically, MVT records where every diff --git a/docs/development/plugin_configuration.md b/docs/development/plugin_configuration.md index e41f4e4..62d35a7 100644 --- a/docs/development/plugin_configuration.md +++ b/docs/development/plugin_configuration.md @@ -24,8 +24,7 @@ configuration: The exact parent folder follows the platform convention used for MVT's `config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use -`mvt.common.plugin_config.plugin_config_path()` instead of building the path by -hand. +`plugin_config_path()` from `mvt.plugin` instead of building the path by hand. Plugin names must be lowercase and may only contain letters, digits and dashes, matching the `mvt-plugin-` package naming convention. MVT creates the @@ -38,9 +37,8 @@ leaves a partially written settings file behind. Everything else a plugin keeps on disk, such as a cache, a downloaded artifact or synchronization state, belongs in the folder returned by the `data_folder()` -class method of the plugin's settings class, or by -`mvt.common.plugin_config.plugin_data_folder()` called with the plugin name if -the plugin has no settings class: +class method of the plugin's settings class, or by `plugin_data_folder()` from +`mvt.plugin`, called with the plugin name if the plugin has no settings class: ``` ~/.local/share/mvt/plugin-data// # Linux @@ -76,7 +74,7 @@ defaults: ```python from typing import Optional -from mvt.common.plugin_config import MVTPluginSettings +from mvt.plugin import MVTPluginSettings class ExamplePluginSettings(MVTPluginSettings): @@ -94,12 +92,15 @@ from datetime import datetime, timezone import click +from mvt.plugin import plugin_env_prefix + def sync(): settings = ExamplePluginSettings.load() if not settings.API_KEY: + prefix = plugin_env_prefix(settings.plugin_name) raise click.ClickException( - "No API key configured. Set MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY or " + f"No API key configured. Set {prefix}API_KEY or " "run 'example-plugin configure'." ) @@ -107,6 +108,9 @@ def sync(): settings.save() ``` +The message builds the variable name with `plugin_env_prefix()`, see +[Environment Variables](#environment-variables). + A missing settings file is not an error: the plugin then runs on the field defaults and on whatever the environment provides. `save()` only persists the values that differ from the defaults, and it never touches MVT's `config.yaml`. @@ -129,6 +133,10 @@ export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=... export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50 ``` +Do not repeat the plugin name in a field name: a plugin named `example-scanner` +with a `SCANNER_API_KEY` field asks the user for +`MVT_PLUGIN_EXAMPLE_SCANNER_SCANNER_API_KEY`. Name the field `API_KEY`. + Settings resolve in this order, from highest to lowest priority: 1. Arguments passed to the settings class directly, such as diff --git a/docs/ios/sysdiagnose.md b/docs/ios/sysdiagnose.md index 6488bc2..d0d84fb 100644 --- a/docs/ios/sysdiagnose.md +++ b/docs/ios/sysdiagnose.md @@ -1,30 +1,32 @@ # Check an iOS Sysdiagnose `mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by -custom MVT modules. MVT does not include built-in sysdiagnose modules. You must -load at least one custom module that explicitly supports this command. +custom MVT modules. MVT does not include built-in sysdiagnose modules. The +command runs the modules of the installed +[plugin packages](../development/index.md#installed-module-packages) which +declare support for it. Install at least one such package first. The command accepts either an extracted sysdiagnose directory or the original gzip-compressed tar archive. ```bash -mvt-ios check-sysdiagnose \ - --load-module ./sysdiagnose_modules.py \ - --output ./results \ +mvt-ios check-sysdiagnose --output ./results \ ./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz ``` Use `--hashes` to include hashes for analyzed files in `info.json`, and -`--list-modules` to display the eligible custom modules without running them. +`--list-modules` to display the eligible modules without running them. ## Writing a custom module -Extend `SysdiagnoseExtraction` to access the archive contents consistently for -both directory and tar inputs. Each module must declare the command explicitly -in `supported_commands`. +Extend `SysdiagnoseExtraction` from `mvt.plugin`, see +[Writing a module](../development/index.md#writing-a-module). The module reads +the archive the same way whether MVT was given a folder or a tar archive. It +declares the command in `supported_commands`. While writing one, +[load it from its file](../development/index.md#developing-modules-locally). ```python -from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction +from mvt.plugin import SysdiagnoseExtraction class ExampleSysdiagnoseModule(SysdiagnoseExtraction): @@ -44,7 +46,9 @@ class ExampleSysdiagnoseModule(SysdiagnoseExtraction): return None ``` -The base class provides `from_sysdiagnose_folder()` and -`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file -reading, and timezone extraction helpers. IPS crash-report metadata is exposed -on `ips_files`. +MVT extracts a tar archive first. It calls `from_sysdiagnose_folder()` on each +module before `run()`. `ips_files` lists the IPS crash reports. + +`_get_files_by_pattern()` and `_get_file_content()` are internal helpers of the +base class. Use them to read the archive. Their names and signatures can change +between releases. See `src/mvt/ios/modules/sysdiagnose/base.py`. diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index 4735222..57f7181 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -15,6 +15,8 @@ from typing import Iterable import click +from .module_loader import CUSTOM_COMMAND_MODULE_PREFIX + IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" # Commands in this group are registered on the platform-neutral mvt command only. @@ -57,7 +59,7 @@ class BrokenPluginCommand(click.Command): def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_command_{path.stem}_{digest}" + return f"{CUSTOM_COMMAND_MODULE_PREFIX}{path.stem}_{digest}" def _iter_command_files(path: Path) -> Iterable[Path]: diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index 37f447d..51f2b80 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -10,6 +10,7 @@ import inspect import json import logging import os +import re import sys from dataclasses import dataclass from functools import lru_cache @@ -26,6 +27,9 @@ EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" _ORIGIN_ATTRIBUTE = "_mvt_module_origin" _PATH_MODULE_PREFIX = "_mvt_custom_module_" +# Shared with cli_plugins, which names a loaded command file this way. +CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" +_LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$") log = logging.getLogger(__name__) @@ -66,6 +70,25 @@ def _module_name_for_path(path: Path) -> str: return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}" +def _is_builtin_logger_name(name: str) -> bool: + return name == "mvt" or name.startswith("mvt.") + + +def _loaded_file_stem(name: str, prefix: str) -> str: + """Recover a loaded file's name from the import name MVT gave it.""" + return _LOADED_FILE_DIGEST.sub("", name[len(prefix) :]) + + +def _external_logger_name(name: str) -> str: + """Return the "mvt.ext" logger name external code logs under.""" + top_level, separator, rest = name.partition(".") + if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len( + PLUGIN_PACKAGE_PREFIX + ): + name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest + return f"{EXTERNAL_LOGGER_NAMESPACE}.{name}" + + def get_module_logger(module_class: type[MVTModule]) -> logging.Logger: """Return the logger a module's records should be emitted through. @@ -80,19 +103,35 @@ def get_module_logger(module_class: type[MVTModule]) -> logging.Logger: "mvt_plugin_" naming convention log under "mvt.ext.". """ name = module_class.__module__ - if name == "mvt" or name.startswith("mvt."): + if _is_builtin_logger_name(name): return logging.getLogger(name) if name.startswith(_PATH_MODULE_PREFIX): - name = Path(get_module_origin(module_class).name).stem - else: - top_level, separator, rest = name.partition(".") - if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len( - PLUGIN_PACKAGE_PREFIX - ): - name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest + file_name = Path(get_module_origin(module_class).name).stem + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{file_name}") - return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{name}") + return logging.getLogger(_external_logger_name(name)) + + +def get_plugin_logger(name: str) -> logging.Logger: + """Return a general logger for use in custom MVT plugins. + + Call it with ``__name__``. The logger sits under "mvt.ext". That is + where get_module_logger() puts module classes. A file loaded with + --load-module or --load-command is named after the file. + """ + if _is_builtin_logger_name(name): + return logging.getLogger(name) + + # A file loaded with --load-module or --load-command is imported under a + # mangled name. Log it under the file it came from. get_module_logger() + # does the same for the module classes such a file defines. + for prefix in (_PATH_MODULE_PREFIX, CUSTOM_COMMAND_MODULE_PREFIX): + if name.startswith(prefix): + stem = _loaded_file_stem(name, prefix) + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{stem}") + + return logging.getLogger(_external_logger_name(name)) def _iter_module_files(path: Path) -> Iterable[Path]: diff --git a/src/mvt/plugin.py b/src/mvt/plugin.py new file mode 100644 index 0000000..9257dc6 --- /dev/null +++ b/src/mvt/plugin.py @@ -0,0 +1,80 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Stable functions and modules which plugins can import. + +Anything else in mvt can be imported too, and may change +between releases without notice. +""" + +from mvt.android.modules.androidqf.base import AndroidQFModule +from mvt.android.modules.backup.base import BackupModule as AndroidBackupModule +from mvt.android.modules.bugreport.base import BugReportModule +from mvt.common.alerts import Alert, AlertLevel +from mvt.common.command import Command +from mvt.common.config import settings +from mvt.common.module import DatabaseCorruptedError, DatabaseNotFoundError, MVTModule +from mvt.common.module_loader import get_plugin_logger +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) +from mvt.common.utils import ( + convert_chrometime_to_datetime, + convert_datetime_to_iso, + convert_mactime_to_datetime, + convert_mactime_to_iso, + convert_unix_to_iso, + convert_unix_to_utc_datetime, +) +from mvt.common.version import MVT_VERSION +from mvt.ios.modules.base import IOSExtraction +from mvt.ios.modules.sysdiagnose.base import SysdiagnoseExtraction + +__all__ = [ + # Classes a plugin subclasses. + "MVTModule", + "IOSExtraction", + "SysdiagnoseExtraction", + "AndroidQFModule", + "AndroidBackupModule", + "BugReportModule", + "Command", + # Results and alerts. + "ModuleAtomicResult", + "ModuleResults", + "ModuleSerializedResult", + "Alert", + "AlertLevel", + # Errors a module raises. + "DatabaseNotFoundError", + "DatabaseCorruptedError", + # Settings. + "settings", + "MVTPluginSettings", + "PluginConfigLoadError", + "plugin_config_path", + "plugin_data_folder", + "plugin_env_prefix", + # Logging. + "get_plugin_logger", + # Timestamps. + "convert_chrometime_to_datetime", + "convert_datetime_to_iso", + "convert_mactime_to_datetime", + "convert_mactime_to_iso", + "convert_unix_to_iso", + "convert_unix_to_utc_datetime", + # MVT's version. + "MVT_VERSION", +] diff --git a/tests/common/test_module_loader.py b/tests/common/test_module_loader.py index dec636e..527d01c 100644 --- a/tests/common/test_module_loader.py +++ b/tests/common/test_module_loader.py @@ -1,9 +1,14 @@ +from pathlib import Path + import pytest +from mvt.common.cli_plugins import _module_name_for_path as _command_module_name from mvt.common.module import MVTModule from mvt.common.module_loader import ( CustomModuleLoadError, + _module_name_for_path, get_module_logger, + get_plugin_logger, load_custom_modules, load_custom_modules_from_path, module_supports_command, @@ -168,9 +173,9 @@ def test_get_module_logger_strips_the_plugin_package_prefix(): class PluginModule(MVTModule): pass - PluginModule.__module__ = "mvt_plugin_amnesty_custom.ios.custom" + PluginModule.__module__ = "mvt_plugin_example_org.ios.custom" - assert get_module_logger(PluginModule).name == "mvt.ext.amnesty_custom.ios.custom" + assert get_module_logger(PluginModule).name == "mvt.ext.example_org.ios.custom" def test_get_module_logger_only_strips_the_prefix_from_the_top_level(): @@ -187,3 +192,25 @@ def test_get_module_logger_names_path_modules_after_their_file(tmp_path): module = load_custom_modules_from_path(str(module_path))[0] assert get_module_logger(module).name == "mvt.ext.my_custom_module" + + +def test_get_plugin_logger_uses_the_same_namespace_as_modules(): + assert ( + get_plugin_logger("mvt_plugin_example_org.commands.summarize").name + == "mvt.ext.example_org.commands.summarize" + ) + assert get_plugin_logger("example_plugin.cli").name == "mvt.ext.example_plugin.cli" + + +def test_get_plugin_logger_keeps_builtin_names(): + assert get_plugin_logger("mvt.ios.cli").name == "mvt.ios.cli" + + +def test_get_plugin_logger_names_loaded_files_after_the_file(): + # A file loaded with --load-command or --load-module is imported under a + # mangled name. The log names the file instead. + command_name = _command_module_name(Path("/tmp/case_summary.py")) + module_name = _module_name_for_path(Path("/tmp/my_custom_module.py")) + + assert get_plugin_logger(command_name).name == "mvt.ext.case_summary" + assert get_plugin_logger(module_name).name == "mvt.ext.my_custom_module" diff --git a/tests/common/test_plugin.py b/tests/common/test_plugin.py new file mode 100644 index 0000000..255f2e8 --- /dev/null +++ b/tests/common/test_plugin.py @@ -0,0 +1,34 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import mvt.plugin +from mvt.android.modules.backup.base import BackupModule +from mvt.common.config import settings + +from ..plugin_fixtures import run_isolated_python + + +def test_the_exported_names_are_the_public_names(): + public = {name for name in vars(mvt.plugin) if not name.startswith("_")} + + assert public == set(mvt.plugin.__all__) + assert mvt.plugin.settings is settings + assert mvt.plugin.AndroidBackupModule is BackupModule + + +def test_the_surface_imports_before_anything_else_of_mvt(tmp_path): + # A plugin can import the surface as its first import of MVT. The + # subprocess gets a temporary home because importing MVT writes its + # configuration file. + result = run_isolated_python( + "from mvt.plugin import IOSExtraction, MVT_VERSION, settings\n" + "assert MVT_VERSION\n" + "assert settings.NETWORK_TIMEOUT > 0\n" + "assert IOSExtraction.__name__ == 'IOSExtraction'\n", + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + assert result.stderr == "" diff --git a/tests/plugin_fixtures.py b/tests/plugin_fixtures.py index ab3359b..ef989dc 100644 --- a/tests/plugin_fixtures.py +++ b/tests/plugin_fixtures.py @@ -73,6 +73,7 @@ def run_isolated_python( } isolated_environment["HOME"] = str(home) isolated_environment["XDG_CONFIG_HOME"] = str(home / "config") + isolated_environment["XDG_DATA_HOME"] = str(home / "data") if site_path is not None: isolated_environment["PYTHONPATH"] = str(site_path) isolated_environment.update(environment) From 47ac8a5a85acae96e1059815b6cef56384915c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:17 +0200 Subject: [PATCH 37/43] Allow custom modules to replace built-in modules (#902) A custom module which extends a built-in one ran alongside it, and both wrote to the same results file when they shared a slug, with the run order deciding the surviving content. Custom modules can now name the module class they supersede in a `replaces` attribute. When both are available to a command, the named module is dropped from the run and the substitution is logged with the origin of the replacement, so it is recorded in command.log. Only the replacements which are applied are reported: a declaration from a disabled module is ignored, so that replacing a module cannot silently disable it, and modules which replace each other in a cycle all keep running and replace nothing, with a warning naming every one of them. Dependencies are remapped along with the modules themselves: a module depending on a replaced class is ordered against, and receives the results of, the module which took its place. Replacing a module which others depend on therefore does not make that dependency unavailable. The remapping applies wherever dependencies are read, so a replacement which cannot run is skipped like any other module with an unavailable dependency, and takes the modules depending on the module it replaced with it. Those warnings name the dependency the author declared as well as the module which replaces it. A replacement does not have to keep the class name of the module it replaces, so `--module` now falls back to the name of a replaced module and runs its replacement. A name which matches no module at all stops the run with a warning instead of silently analyzing nothing, as does a selection left with nothing to run once skipped modules are dropped. Sharing a slug outside a replacement stays possible and is now reported. Two modules writing to the same results file is a forensic-integrity problem rather than an error, so both still run and a warning names them, where each came from, and the file the later one overwrites. Taking over the slug of a replaced module is not reported, because that module is no longer part of the run. --- docs/development/index.md | 53 ++++ src/mvt/common/command.py | 281 ++++++++++++++++++-- src/mvt/common/module.py | 5 + tests/common/test_command.py | 490 +++++++++++++++++++++++++++++++++++ 4 files changed, 813 insertions(+), 16 deletions(-) diff --git a/docs/development/index.md b/docs/development/index.md index d1c167f..cfd5893 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -176,6 +176,59 @@ not, MVT skips the module with a warning. They are the same records it writes to `.json`. Typed results per module are planned. +### Replacing a built-in module + +A custom module which extends a built-in one runs alongside it, and when the +two share a slug they write to the same results file. MVT warns about that +collision, naming both modules, where each came from and the file the later one +overwrites, but it runs them both. Set `replaces` to the class the module +supersedes to take that module's place instead: when both are available to a +command, the named module is dropped from the run. + +```python +from mvt.ios.modules.backup import Manifest as BuiltinManifest + + +class Manifest(BuiltinManifest): + supported_commands = (("ios", "check-backup"),) + replaces = BuiltinManifest +``` + +Keeping the class name of the replaced module, as above, keeps the name +`--module` selects the module by and the slug its results file is named after. +A replacement with a different class name writes to a file named after its own +slug, unless it sets `slug` to the slug of the module it replaces, and +`--module` still selects it by the name of the replaced module, which MVT logs. +Taking over the slug of a replaced module is not a collision and is not warned +about, because that module is no longer part of the run. + +Modules which depend on the replaced module receive the replacement instead, so +`get_dependency_results(BuiltinManifest)` returns the replacement's results. A +module must not depend on the module it replaces: that module does not run, so +the replacement has to produce the data itself, and MVT logs a warning when a +module declares both. + +A replacement takes on the obligations of the module it replaces. If it +declares a dependency the command does not provide, it is skipped like any +other module with an unavailable dependency, and the module it replaced stays +out of the run: neither of them produces results, and the modules depending on +the replaced module are skipped as well. + +Subclassing the replaced module is not required, but a replacement which is not +a subclass is logged with a warning, because its results may not be what the +modules depending on the replaced module expect. Naming a module the command +does not run has no effect, and modules which replace each other in a cycle all +keep running and replace nothing. Every applied substitution is logged, so it +is recorded in `command.log` when the command runs with an `--output` folder. + +Every command resolves replacements on its own. `check-iocs` matches stored +results files against the slugs of the modules available for that command, so a +replacement checks the indicators of its own results only if it also declares +the `("ios", "check-iocs")` pair; otherwise the built-in module it replaced +re-checks the file. It also matches `--module` on the class name only, so pass +a differently named replacement's own name there, not the name of the module +it replaces. + ### Importing from MVT Import from `mvt.plugin` if it has what you need. The names it exports are diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index f2f211d..1242941 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -72,6 +72,10 @@ class Command: # down a password to decrypt a backup or flags which are need by some modules. self.module_options = module_options if module_options else {} + # This dictionary maps the modules which were replaced by a module + # declaring `replaces` to the module which took their place. + self.module_replacements: dict[type[MVTModule], type[MVTModule]] = {} + # This list will contain all executed modules. # We can use this to reference e.g. self.executed[0].results. self.executed: list[MVTModule] = [] @@ -257,7 +261,174 @@ class Command: if module not in deduplicated: deduplicated.append(module) - return deduplicated + available = self._apply_replacements(deduplicated) + self._warn_about_slug_collisions(available) + + return available + + def _warn_about_slug_collisions(self, modules: list[type[MVTModule]]) -> None: + """Report modules writing their results to the same file. + + Results are stored in a file named after the module slug, so two + modules sharing one silently overwrite each other. Replacements are + already resolved here, so a module deliberately taking over the slug + of the module it replaces is not reported: the module it replaced is + no longer part of the run. + """ + modules_by_slug: dict[str, type[MVTModule]] = {} + for module in modules: + slug = module.get_slug() + first = modules_by_slug.setdefault(slug, module) + if first is module: + continue + + self.log.warning( + "Modules %s from %s and %s from %s both use the slug %s. If " + "both run, whichever runs last overwrites the results of the " + "other in %s.json.", + first.__name__, + get_module_origin(first).label, + module.__name__, + get_module_origin(module).label, + slug, + slug, + ) + + def _declared_replacements( + self, modules: list[type[MVTModule]] + ) -> dict[type[MVTModule], type[MVTModule]]: + """Return the replacements declared by the given modules.""" + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for module in modules: + replaced = module.replaces + if replaced is None or replaced is module: + continue + + if not module.enabled: + # Replacing a module must not disable it, as a disabled + # replacement never runs in its place. + self.log.debug( + "Module %s is disabled and does not replace module %s.", + module.__name__, + replaced.__name__, + ) + continue + + if replaced not in modules: + # A module can support several commands while the module it + # replaces is only available in some of them. + self.log.debug( + "Module %s replaces module %s, which is not available " + "for the %s command.", + module.__name__, + replaced.__name__, + self.name, + ) + continue + + if replaced in replacements: + self.log.warning( + "Modules %s and %s both replace module %s. Both of them " + "will run, %s will not, and modules depending on %s will " + "use the results of %s. Replacements which share the slug " + "of %s overwrite each other's results file.", + replacements[replaced].__name__, + module.__name__, + replaced.__name__, + replaced.__name__, + replaced.__name__, + replacements[replaced].__name__, + replaced.__name__, + ) + continue + + replacements[replaced] = module + + return replacements + + def _drop_replacement_cycles( + self, replacements: dict[type[MVTModule], type[MVTModule]] + ) -> None: + """Undo the replacements between modules which replace each other.""" + cyclic: set[type[MVTModule]] = set() + for replaced in replacements: + walked: list[type[MVTModule]] = [] + module = replaced + while module in replacements and module not in cyclic: + if module in walked: + cyclic.update(walked[walked.index(module) :]) + break + walked.append(module) + module = replacements[module] + + if not cyclic: + return + + self.log.warning( + "Modules %s replace each other in a cycle. None of them replaces " + "anything and all of them will run.", + ", ".join(sorted(module.__name__ for module in cyclic)), + ) + for module in cyclic: + replacements.pop(module, None) + + def _log_replacement( + self, + replaced: type[MVTModule], + module: type[MVTModule], + replacement: type[MVTModule], + ) -> None: + """Report an applied replacement, and any problem with it.""" + if not issubclass(module, replaced): + self.log.warning( + "Module %s replaces module %s but is not a subclass of it. " + "Its results might not be compatible with what modules " + "depending on %s expect.", + module.__name__, + replaced.__name__, + replaced.__name__, + ) + + if replaced in module.dependencies: + self.log.warning( + "Module %s depends on module %s, which it also replaces. The " + "dependency cannot be satisfied: a replacement has to produce " + "that data itself.", + module.__name__, + replaced.__name__, + ) + + self.log.info( + "Module %s from %s replaces module %s from %s.", + replacement.__name__, + get_module_origin(replacement).label, + replaced.__name__, + get_module_origin(replaced).label, + ) + + def _apply_replacements( + self, modules: list[type[MVTModule]] + ) -> list[type[MVTModule]]: + """Drop the modules superseded by a module declaring `replaces`.""" + declared = self._declared_replacements(modules) + self._drop_replacement_cycles(declared) + + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for replaced, module in declared.items(): + # Follow chains of replacements, so that a dependency on a + # replaced module always resolves to a module which is part of + # the run. + replacement = module + while replacement in declared: + replacement = declared[replacement] + + # Only the replacements which are applied are reported, so that + # the record matches the modules which actually run. + self._log_replacement(replaced, module, replacement) + replacements[replaced] = replacement + + self.module_replacements = replacements + return [module for module in modules if module not in replacements] def init(self) -> None: raise NotImplementedError @@ -317,6 +488,80 @@ class Command: console.print("") console.print(panel) + def _module_dependencies( + self, module: type[MVTModule] + ) -> list[tuple[type[MVTModule], type[MVTModule]]]: + """Return the (declared, resolved) dependencies of a module. + + A dependency on a module which was replaced is resolved to the module + which took its place. A module which replaces one of its own + dependencies is not made to depend on itself, while a module which + declares itself as a dependency is left alone and still fails the + circular dependency check. + """ + dependencies = [] + for dependency in module.dependencies: + resolved = self.module_replacements.get(dependency, dependency) + if resolved is module and dependency is not module: + continue + dependencies.append((dependency, resolved)) + + return dependencies + + @staticmethod + def _dependency_name( + declared: type[MVTModule], resolved: type[MVTModule] + ) -> str: + """Return how a dependency is named in the messages about it. + + A dependency is named as the module which declared it wrote it, and, + when that module was replaced, as the module which runs in its place. + """ + if declared is resolved: + return declared.__name__ + + return f"{resolved.__name__} (replacing module {declared.__name__})" + + def _selected_modules( + self, modules: list[type[MVTModule]] + ) -> Optional[list[type[MVTModule]]]: + """Return the modules explicitly requested, or all the enabled ones. + + Returns None when a module was requested by name and no module of + that name can be run. + """ + if not self.module_name: + return [module for module in modules if module.enabled] + + selected = [ + module for module in modules if module.__name__ == self.module_name + ] + + # A module replacing another one does not have to keep its name, so + # the name of a replaced module selects its replacement. + if not selected: + for replaced, replacement in self.module_replacements.items(): + if replaced.__name__ != self.module_name or replacement in selected: + continue + self.log.info( + "Module %s was replaced by module %s, which is run " + "in its place.", + replaced.__name__, + replacement.__name__, + ) + selected.append(replacement) + + if not selected: + self.log.warning( + "No module named %s is available for the %s command. " + "No modules will be run.", + self.module_name, + self.name, + ) + return None + + return selected + def _skipped_modules( self, required: list[type[MVTModule]], @@ -349,15 +594,18 @@ class Command: if module in skipped: continue - for dependency in module.dependencies: + for declared, dependency in self._module_dependencies(module): if dependency not in module_indexes: - skipped[module] = (module, dependency) + # A replaced dependency always resolves to a module of + # this command, so an unavailable one is always the + # module class the author declared. + skipped[module] = (module, declared) changed = True self.log.warning( "Module %s will be SKIPPED: it depends on module " "%s, which is not available in this command.%s", module.__name__, - dependency.__name__, + declared.__name__, remainder, ) break @@ -372,7 +620,7 @@ class Command: "module %s, itself skipped for depending on " "unavailable module %s.%s", module.__name__, - dependency.__name__, + self._dependency_name(declared, dependency), missing.__name__, remainder, ) @@ -383,7 +631,7 @@ class Command: "module %s, which depends on unavailable " "module %s.%s", module.__name__, - dependency.__name__, + self._dependency_name(declared, dependency), root.__name__, missing.__name__, remainder, @@ -397,12 +645,9 @@ class Command: modules = self._available_modules() module_indexes = {module: index for index, module in enumerate(modules)} - if self.module_name: - selected = [ - module for module in modules if module.__name__ == self.module_name - ] - else: - selected = [module for module in modules if module.enabled] + selected = self._selected_modules(modules) + if selected is None: + return None required: set[type[MVTModule]] = set() pending = list(selected) @@ -411,7 +656,7 @@ class Command: if module in required: continue required.add(module) - for dependency in module.dependencies: + for _, dependency in self._module_dependencies(module): # Unavailable dependencies are reported by _skipped_modules(). if dependency in module_indexes: pending.append(dependency) @@ -424,13 +669,14 @@ class Command: "Every selected module was skipped for an unavailable " "dependency. No modules will be run." ) + return None dependents: dict[type[MVTModule], list[type[MVTModule]]] = { module: [] for module in runnable } indegree = {module: 0 for module in runnable} for module in runnable: - for dependency in module.dependencies: + for _, dependency in self._module_dependencies(module): if dependency not in indegree: continue dependents[dependency].append(module) @@ -486,9 +732,12 @@ class Command: module_options=self.module_options, log=module_logger, ) + # Dependencies are keyed by the module class they declare, even + # when it was replaced, so that a module asking for the results + # of a replaced module receives those of its replacement. m.dependency_modules = { - dependency: executed_by_type[dependency] - for dependency in module.dependencies + dependency: executed_by_type[resolved] + for dependency, resolved in self._module_dependencies(module) } if self.iocs.total_ioc_count: diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index cd127a6..1257aa6 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -46,6 +46,11 @@ class MVTModule: slug: Optional[str] = None dependencies: Sequence[type["MVTModule"]] = () supported_commands: Sequence[tuple[str, str]] = () + # A custom module can name a module class it supersedes, usually a + # built-in one. When both are available to a command, the named module is + # dropped from the run and this module takes its place, including in the + # dependencies of any other module. + replaces: Optional[type["MVTModule"]] = None def __init__( self, diff --git a/tests/common/test_command.py b/tests/common/test_command.py index 4835fd3..8e09f57 100644 --- a/tests/common/test_command.py +++ b/tests/common/test_command.py @@ -68,6 +68,123 @@ class CustomDependsOnBuiltin(RecordingModule): dependencies = (FirstModule,) +class ReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +class OtherReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + +class UnrelatedReplacementModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = IndependentModule + + +class ReplacementOfReplacementModule(ReplacementModule): + supported_commands = (("ios", "check-backup"),) + replaces = ReplacementModule + + def run(self): + super().run() + self.results = ["replacement of replacement"] + + +class ReplacesOwnDependencyModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (FirstModule,) + + +class DisabledReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + enabled = False + + +class MutualReplacementOne(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class MutualReplacementTwo(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = MutualReplacementOne + + +MutualReplacementOne.replaces = MutualReplacementTwo + + +class CycleOneModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class CycleTwoModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleOneModule + + +class CycleThreeModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleTwoModule + + +CycleOneModule.replaces = CycleThreeModule + + +class UnavailableDependencyModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class ReplacementMissingDependency(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (UnavailableDependencyModule,) + + +class SharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class OtherSharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class ReplacementKeepingTheSlug(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + slug = "first_module" + + +class SameNameReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +# Modules replacing a built-in one usually keep its class name, which is the +# name `--module` matches on. +SameNameReplacementModule.__name__ = "FirstModule" + + +def logged_substitutions(caplog) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.INFO and "replaces module" in record.getMessage() + ] + + class RecordingCommand(Command): def init(self): self.initialized = True @@ -303,3 +420,376 @@ class TestCommand: cmd.run() assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"] + + def test_custom_module_replaces_builtin(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Module ReplacementModule from" in caplog.text + and "replaces module FirstModule" in caplog.text + ) + + def test_modules_without_replaces_all_run(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [CustomIOSBackupModule] + + cmd.run() + + assert RecordingModule.run_order == ["FirstModule", "CustomIOSBackupModule"] + assert cmd.module_replacements == {} + + def test_unavailable_replaced_module_is_ignored(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {} + assert "replaces module FirstModule" not in caplog.text + + def test_dependencies_are_resolved_to_the_replacement(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule", "SecondModule"] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert isinstance(second.dependency_modules[FirstModule], ReplacementModule) + assert second.results == ["replacement", "second"] + + def test_replacement_with_unavailable_dependency_skips_its_dependents( + self, caplog + ): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, SecondModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + # The replacement cannot run, and the module it replaced was dropped + # from the run by the replacement, so neither of them produces + # results and the module depending on the replaced one is skipped. + assert RecordingModule.run_order == ["IndependentModule"] + assert ( + "Module ReplacementMissingDependency will be SKIPPED: it depends " + "on module UnavailableDependencyModule, which is not available in " + "this command." in caplog.text + ) + # The skipped dependent is told which module it actually depends on, + # and the module class its author declared. + assert ( + "Module SecondModule will be SKIPPED: it depends on module " + "ReplacementMissingDependency (replacing module FirstModule), " + "itself skipped for depending on unavailable module " + "UnavailableDependencyModule." in caplog.text + ) + + def test_selected_replacement_with_unavailable_dependency_runs_nothing( + self, caplog, tmp_path + ): + cmd = RecordingCommand(module_name="FirstModule", results_path=str(tmp_path)) + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "Module FirstModule was replaced by module " + "ReplacementMissingDependency, which is run in its place." + in caplog.text + ) + assert "Module ReplacementMissingDependency will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warnings must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + # No module ran, so no results were stored next to the command log. + assert [path.name for path in tmp_path.iterdir()] == ["command.log"] + + def test_chained_replacements_are_resolved(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule, ReplacementOfReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementOfReplacementModule", + "SecondModule", + ] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert second.results == ["replacement of replacement", "second"] + + def test_replacement_which_is_not_a_subclass_warns(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [UnrelatedReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["UnrelatedReplacementModule"] + assert "is not a subclass of it" in caplog.text + + def test_multiple_modules_replacing_the_same_module_warn(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule, OtherReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementModule", + "OtherReplacementModule", + ] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Modules ReplacementModule and OtherReplacementModule both replace " + "module FirstModule. Both of them will run, FirstModule will not, " + "and modules depending on FirstModule will use the results of " + "ReplacementModule." in caplog.text + ) + assert "overwrite each other's results file" in caplog.text + + def test_module_replacing_its_own_dependency_runs(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacesOwnDependencyModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["ReplacesOwnDependencyModule"] + assert ( + "Module ReplacesOwnDependencyModule depends on module FirstModule, " + "which it also replaces" in caplog.text + ) + + def test_literal_self_dependency_is_still_circular(self, caplog): + class SelfDependentModule(RecordingModule): + pass + + SelfDependentModule.dependencies = (SelfDependentModule,) + + cmd = RecordingCommand() + cmd.modules = [SelfDependentModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert "Circular module dependency detected" in caplog.text + + def test_disabled_replacement_keeps_the_replaced_module(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [DisabledReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["FirstModule"] + assert cmd.module_replacements == {} + assert logged_substitutions(caplog) == [] + + def test_modules_replacing_each_other_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [MutualReplacementOne, MutualReplacementTwo] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "MutualReplacementOne", + "MutualReplacementTwo", + ] + assert cmd.module_replacements == {} + assert ( + "Modules MutualReplacementOne, MutualReplacementTwo replace each " + "other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_replacement_cycle_of_three_modules_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [CycleOneModule, CycleTwoModule, CycleThreeModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "CycleOneModule", + "CycleTwoModule", + "CycleThreeModule", + ] + assert cmd.module_replacements == {} + assert ( + "Modules CycleOneModule, CycleThreeModule, CycleTwoModule replace " + "each other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_selected_replaced_module_name_runs_the_replacement(self, caplog): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule"] + assert ( + "Module FirstModule was replaced by module ReplacementModule" + in caplog.text + ) + + def test_unknown_selected_module_warns_and_stops(self, caplog): + cmd = RecordingCommand(module_name="NoSuchModule") + cmd.name = "check-backup" + cmd.modules = [FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "No module named NoSuchModule is available for the check-backup " + "command" in caplog.text + ) + + def test_selected_module_name_matches_the_replacement(self): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [SameNameReplacementModule] + + cmd.run() + + assert len(cmd.executed) == 1 + assert isinstance(cmd.executed[0], SameNameReplacementModule) + assert cmd.executed[0].results == ["replacement"] + + def test_list_modules_reflects_replacements(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + listed = [ + record.getMessage() + for record in caplog.records + if "Modules from" in record.getMessage() + ] + assert any("ReplacementModule" in message for message in listed) + assert not any("FirstModule" in message for message in listed) + + def test_modules_sharing_a_slug_are_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [SharedSlugModule, OtherSharedSlugModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "SharedSlugModule", + "OtherSharedSlugModule", + ] + collisions = [ + record.getMessage() + for record in caplog.records + if "both use the slug" in record.getMessage() + ] + assert len(collisions) == 1 + assert "Modules SharedSlugModule from" in collisions[0] + assert "and OtherSharedSlugModule from" in collisions[0] + assert "both use the slug shared_slug" in collisions[0] + assert "overwrites the results of the other in shared_slug.json" in ( + collisions[0] + ) + + def test_replacement_keeping_the_replaced_slug_is_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementKeepingTheSlug] + + with caplog.at_level(logging.WARNING): + cmd.run() + + # The replaced module is no longer part of the run, so taking over its + # slug is what the replacement is for, not a collision. + assert RecordingModule.run_order == ["ReplacementKeepingTheSlug"] + assert ReplacementKeepingTheSlug.get_slug() == FirstModule.get_slug() + assert "both use the slug" not in caplog.text + + def test_modules_with_distinct_slugs_are_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [CustomIOSBackupModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert "both use the slug" not in caplog.text From 00d892d354d3992d1b275241310cde1af98be1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 14:47:17 +0200 Subject: [PATCH 38/43] Run check-iocs on every module which implements check_indicators() (#903) A module is part of check-iocs for a platform when it declares the check-iocs pair, as before. It is now also part of check-iocs when it overrides check_indicators() and supports at least one command of that platform. The rule lives in module_supports_command(). CmdCheckIOCS already uses that function to pick its modules, so --list-modules and --module follow it too. Built-in modules are unaffected. check-iocs takes them from IOS_CHECK_IOCS_MODULES and ANDROID_CHECK_IOCS_MODULES. A custom module which replaces a built-in module is covered by the same rule. If it subclasses the module it replaces, it inherits its check_indicators() and takes over the re-check of the results file. The "Replacing a built-in module" section of the plugin documentation is updated to say that. --- docs/development/index.md | 18 +++-- src/mvt/common/module_loader.py | 12 ++- tests/common/test_cmd_check_iocs.py | 110 +++++++++++++++++++++++++--- 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/docs/development/index.md b/docs/development/index.md index cfd5893..8b80238 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -78,7 +78,9 @@ MVT logs a warning. The nine pairs are: `check-iocs` re-checks stored results rather than an acquisition. It matches every `.json` file in the results folder to the module with that slug. -It then runs that module's `check_indicators()` again. +It then runs that module's `check_indicators()` again. A module which +implements `check_indicators()` is included in `check-iocs` for its platform. +It does not need to declare the `check-iocs` pair. ### Writing a module @@ -222,12 +224,14 @@ keep running and replace nothing. Every applied substitution is logged, so it is recorded in `command.log` when the command runs with an `--output` folder. Every command resolves replacements on its own. `check-iocs` matches stored -results files against the slugs of the modules available for that command, so a -replacement checks the indicators of its own results only if it also declares -the `("ios", "check-iocs")` pair; otherwise the built-in module it replaced -re-checks the file. It also matches `--module` on the class name only, so pass -a differently named replacement's own name there, not the name of the module -it replaces. +results files against the slugs of the modules available for that command. A +replacement which subclasses the module it replaces inherits its +`check_indicators()`. It is then part of `check-iocs` for its platform and +re-checks the results file named after its slug. A replacement with no +`check_indicators()` is not part of `check-iocs`. The built-in module it +replaced re-checks the file. `check-iocs` matches `--module` on the class name +only. Pass a differently named replacement's own name there, not the name of +the module it replaces. ### Importing from MVT diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index 51f2b80..d0028b8 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -402,4 +402,14 @@ def module_supports_command( ) return False - return (platform, command) in {tuple(entry) for entry in supported_commands} + pairs = {tuple(entry) for entry in supported_commands} + if (platform, command) in pairs: + return True + + # A module which implements check_indicators() is re-checked by check-iocs + # for its platform. It does not need to declare the check-iocs pair. + return ( + command == "check-iocs" + and platform in {entry[0] for entry in pairs if entry} + and module_class.check_indicators is not MVTModule.check_indicators + ) diff --git a/tests/common/test_cmd_check_iocs.py b/tests/common/test_cmd_check_iocs.py index 1bdd232..0c05c1b 100644 --- a/tests/common/test_cmd_check_iocs.py +++ b/tests/common/test_cmd_check_iocs.py @@ -15,6 +15,7 @@ from mvt.common.cmd_check_iocs import CmdCheckIOCS from mvt.common.module import MVTModule from mvt.ios.cli import cli as ios_cli from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup.manifest import Manifest # Keep the banner of the group callback from checking for updates online. OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] @@ -39,34 +40,69 @@ class CustomResultsModule(MVTModule): self.checked.append(list(self.results)) +class BackupCheckerModule(MVTModule): + """An iOS module which implements check_indicators() without declaring check-iocs.""" + + slug = "backup_checker" + supported_commands = (("ios", "check-backup"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BugReportCheckerModule(MVTModule): + """The same for Android.""" + + slug = "bugreport_checker" + supported_commands = (("android", "check-bugreport"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + class BackupOnlyModule(MVTModule): - """A custom module which does not declare check-iocs.""" + """A custom module which does not implement check_indicators().""" slug = "backup_only" supported_commands = (("ios", "check-backup"),) - def check_indicators(self) -> None: - raise AssertionError("must not be re-checked") + def run(self) -> None: + pass @pytest.mark.parametrize( - "platform, builtin_modules", - [("ios", IOS_CHECK_IOCS_MODULES), ("android", ANDROID_CHECK_IOCS_MODULES)], + "platform, builtin_modules, checker_module", + [ + ("ios", IOS_CHECK_IOCS_MODULES, BackupCheckerModule), + ("android", ANDROID_CHECK_IOCS_MODULES, BugReportCheckerModule), + ], ) def test_check_iocs_rechecks_the_stored_results_of_custom_modules( - platform, builtin_modules, tmp_path, caplog + platform, builtin_modules, checker_module, tmp_path, caplog ): # check-iocs matches every .json in the results folder to the module # with that slug, custom modules included, and runs its check_indicators() # again over the stored results. results = [{"domain": "example.org"}] (tmp_path / "custom_results.json").write_text(json.dumps(results)) + (tmp_path / f"{checker_module.slug}.json").write_text(json.dumps(results)) (tmp_path / "backup_only.json").write_text(json.dumps(results)) CustomResultsModule.checked.clear() + checker_module.checked.clear() cmd = CmdCheckIOCS( target_path=str(tmp_path), - custom_modules=[CustomResultsModule, BackupOnlyModule], + custom_modules=[CustomResultsModule, checker_module, BackupOnlyModule], platform=platform, ) cmd.modules = builtin_modules @@ -74,29 +110,79 @@ def test_check_iocs_rechecks_the_stored_results_of_custom_modules( with caplog.at_level(logging.INFO): cmd.run() + # A module which declares the check-iocs pair is re-checked. assert CustomResultsModule.checked == [results] assert ( 'Loading results from "custom_results.json" with module CustomResultsModule' in caplog.text ) - # A module declaring only check-backup is not part of check-iocs. + # So is a module which only implements check_indicators(). + assert checker_module.checked == [results] + # A module which does neither is not part of check-iocs. assert "backup_only.json" not in caplog.text -def test_check_iocs_lists_custom_modules_declaring_the_command(caplog): +@pytest.mark.parametrize( + "platform, builtin_modules, listed, not_listed", + [ + ( + "ios", + IOS_CHECK_IOCS_MODULES, + "BackupCheckerModule", + "BugReportCheckerModule", + ), + ( + "android", + ANDROID_CHECK_IOCS_MODULES, + "BugReportCheckerModule", + "BackupCheckerModule", + ), + ], +) +def test_check_iocs_lists_the_custom_modules_it_runs( + platform, builtin_modules, listed, not_listed, caplog +): cmd = CmdCheckIOCS( - custom_modules=[CustomResultsModule, BackupOnlyModule], - platform="ios", + custom_modules=[ + CustomResultsModule, + BackupCheckerModule, + BugReportCheckerModule, + BackupOnlyModule, + ], + platform=platform, ) - cmd.modules = IOS_CHECK_IOCS_MODULES + cmd.modules = builtin_modules with caplog.at_level(logging.INFO): cmd.list_modules() assert "CustomResultsModule" in caplog.text + # The module which implements check_indicators() for this platform is listed. + assert listed in caplog.text + # The one for the other platform is not, and neither is BackupOnlyModule. + assert not_listed not in caplog.text assert "BackupOnlyModule" not in caplog.text +class ReplacementManifest(Manifest): + """A replacement for a built-in module which does not declare check-iocs.""" + + supported_commands = (("ios", "check-backup"),) + replaces = Manifest + + +def test_check_iocs_uses_a_replacement_of_a_built_in_module(): + # A replacement which subclasses a built-in module inherits its + # check_indicators(). check-iocs then runs it in place of that module. + cmd = CmdCheckIOCS(custom_modules=[ReplacementManifest], platform="ios") + cmd.modules = IOS_CHECK_IOCS_MODULES + + available = cmd._available_modules() + + assert ReplacementManifest in available + assert Manifest not in available + + LOADED_MODULE = ''' from mvt.common.module import MVTModule From ee24121a7326a4cdf6162e0278cc8ebd5bafa467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 15:28:52 +0200 Subject: [PATCH 39/43] Fix redundent docs and CLI help text --- docs/development/custom_commands.md | 6 +----- src/mvt/cli.py | 6 ++---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md index 3161cef..7aac490 100644 --- a/docs/development/custom_commands.md +++ b/docs/development/custom_commands.md @@ -73,11 +73,7 @@ rather than in MVT's own `config.yaml`. ### Commands on `mvt` -The `mvt` command hosts what belongs to neither platform: `version`, -`completion`, `plugins` and `download-iocs`. A plugin command which is not -about the acquisition of one platform, such as one which configures the plugin -or synchronizes the indicators it uses, belongs there too, in the -`mvt.cli_plugins` group: +A MVT plugin command can also add sub-commands to the base `mvt` command. This can be used for commands which are not tied to a particular forensic platform: ```toml [project.entry-points."mvt.cli_plugins"] diff --git a/src/mvt/cli.py b/src/mvt/cli.py index 40a86fe..6e8c667 100644 --- a/src/mvt/cli.py +++ b/src/mvt/cli.py @@ -47,10 +47,8 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): """Mobile Verification Toolkit. - mvt-ios and mvt-android run the forensic analysis of an acquisition: each - provides the check-* commands of its platform. This command hosts what - belongs to neither platform; run it without a command to see the installed - version and the list of what it offers. + The 'mvt-ios' and 'mvt-android' CLI commands are used to perform + forensic analysis on IOS and Android devices. """ ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check From c9a57f5d10f481016b6e073c7842e806d8c8bdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 15:34:53 +0200 Subject: [PATCH 40/43] Do not pin the wording of the mvt help text in the test --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a5cb51..6495243 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -29,7 +29,6 @@ class TestMvtCommand: assert result.exit_code == 0 assert "mvt-ios" in result.output assert "mvt-android" in result.output - assert "check-*" in result.output def test_version_prints_the_installed_version(self): result = CliRunner().invoke(cli, [*OFFLINE, "version"]) From 7d41d0646e0ca20dc4a9962cc5f824731e8ac943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 27 Aug 2026 23:40:47 +0200 Subject: [PATCH 41/43] Keep the start of mvt-ios and mvt-android cheap for shell completion Each console script imports its CLI module before Click can answer a shell completion request, and the completion scripts run the program on every keystroke. Importing mvt.ios.cli or mvt.android.cli took ~230 ms, of which building the command tree needed almost nothing: cli_plugins imported one constant from module_loader, which pulled in MVTModule, the indicators, the pydantic settings, requests and rich; the command implementations pulled in the same, and the iOS CLI imported iOSbackup (pycryptodome) for decrypt-backup. The two platform CLI modules now only build the command tree: each command imports what it runs when it is invoked. cli_plugins owns the custom command prefix instead of importing it from module_loader, and exec_or_profile() loads the settings when it runs. Completion of mvt-ios and mvt-android drops from ~260 ms to ~85 ms per keystroke on a clean install, and every command starts that much sooner. A test fails as soon as a CLI module imports the module machinery again. --- src/mvt/android/cli.py | 35 +++++++++++++++++++++++---------- src/mvt/common/cli_plugins.py | 7 ++++++- src/mvt/common/module_loader.py | 3 +-- src/mvt/common/utils.py | 5 ++++- src/mvt/ios/cli.py | 35 ++++++++++++++++++++++++--------- tests/test_cli_startup.py | 28 ++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 tests/test_cli_startup.py diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index 92b2fe9..7b78a4e 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -14,7 +14,6 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -40,17 +39,12 @@ from mvt.common.help import ( HELP_MSG_VERSION, HELP_MSG_VIRUS_TOTAL, ) -from mvt.common.logo import logo -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import init_logging, set_verbose_logging -from .cmd_check_androidqf import CmdAndroidCheckAndroidQF -from .cmd_check_backup import CmdAndroidCheckBackup -from .cmd_check_bugreport import CmdAndroidCheckBugreport -from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from .command_modules import ANDROID_CHECK_IOCS_MODULES -from .modules.backup.helpers import cli_load_android_backup_password +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-android, including by shell completion on +# every keystroke, so importing it must do no more than build the command tree: +# the forensic modules, the backup parsers and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -74,6 +68,8 @@ def _get_verbose(ctx): def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -101,6 +97,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.obj["disable_indicator_check"] = disable_indicator_update_check ctx.obj["verbose"] = verbose set_verbose_logging(verbose) + + from mvt.common.logo import logo + logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -164,6 +163,8 @@ def check_bugreport( verbose, bugreport_path, ): + from .cmd_check_bugreport import CmdAndroidCheckBugreport + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. @@ -233,6 +234,9 @@ def check_backup( verbose, backup_path, ): + from .cmd_check_backup import CmdAndroidCheckBackup + from .modules.backup.helpers import cli_load_android_backup_password + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -311,6 +315,9 @@ def check_androidqf( verbose, androidqf_path, ): + from .cmd_check_androidqf import CmdAndroidCheckAndroidQF + from .modules.backup.helpers import cli_load_android_backup_password + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -393,6 +400,8 @@ def check_intrusion_logs( verbose, logs_path, ): + from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -446,6 +455,10 @@ def check_intrusion_logs( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import ANDROID_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -472,6 +485,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_indicators(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index 57f7181..17766ac 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -15,7 +15,9 @@ from typing import Iterable import click -from .module_loader import CUSTOM_COMMAND_MODULE_PREFIX +# This module is imported by every CLI at start-up, and by shell completion on +# every keystroke, so it must stay cheap: nothing here may import the module +# machinery (mvt.common.module_loader and what it pulls in). IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" @@ -24,6 +26,9 @@ NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins" MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS" MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS" MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS" +# Prefix of the import name given to a command file loaded from a path. Shared +# with module_loader, which recognises such files when naming their loggers. +CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" log = logging.getLogger(__name__) diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index d0028b8..51b3836 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -18,6 +18,7 @@ from pathlib import Path from types import ModuleType from typing import Iterable, Optional +from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX from .module import MVTModule from .version import MVT_VERSION @@ -27,8 +28,6 @@ EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" _ORIGIN_ATTRIBUTE = "_mvt_module_origin" _PATH_MODULE_PREFIX = "_mvt_custom_module_" -# Shared with cli_plugins, which names a loaded command file this way. -CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" _LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$") log = logging.getLogger(__name__) diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index b71c4ab..2a3e39a 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -14,7 +14,6 @@ from dataclasses import asdict, is_dataclass from typing import Any, Iterator, Union from .log import MVTLogHandler -from mvt.common.config import settings class CustomJSONEncoder(json.JSONEncoder): @@ -272,6 +271,10 @@ def set_verbose_logging(verbose: bool = False): def exec_or_profile(module, globals, locals): """Hook for profiling MVT modules""" + # Imported here so that the CLI modules, which import this one at start-up, + # do not load the settings (and pydantic) before a command runs. + from .config import settings + if settings.PROFILE: cProfile.runctx(module, globals, locals) else: diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 8251fd9..edf7724 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -14,10 +14,7 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import ( generate_hashes_from_path, init_logging, @@ -47,13 +44,12 @@ from mvt.common.help import ( HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.password import prompt_password -from .cmd_check_backup import CmdIOSCheckBackup -from .cmd_check_fs import CmdIOSCheckFS -from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose -from .decrypt import DecryptBackup -from .command_modules import IOS_CHECK_IOCS_MODULES + +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-ios, including by shell completion on every +# keystroke, so importing it must do no more than build the command tree: the +# forensic modules, the backup decryption and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -79,6 +75,8 @@ def _get_verbose(ctx): def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -106,6 +104,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.obj["disable_indicator_check"] = disable_indicator_update_check ctx.obj["verbose"] = verbose set_verbose_logging(verbose) + + from mvt.common.logo import logo + logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -146,6 +147,8 @@ def version(): @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path, destination) if key_file: @@ -209,6 +212,8 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): ) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def extract_key(password, key_file, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path) if password: @@ -276,6 +281,8 @@ def check_backup( verbose, backup_path, ): + from .cmd_check_backup import CmdIOSCheckBackup + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -345,6 +352,8 @@ def check_fs( verbose, dump_path, ): + from .cmd_check_fs import CmdIOSCheckFS + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -413,6 +422,8 @@ def check_sysdiagnose( verbose, sysdiagnose_path, ): + from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckSysdiagnose( @@ -467,6 +478,10 @@ def check_sysdiagnose( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import IOS_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -493,6 +508,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_iocs(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() diff --git a/tests/test_cli_startup.py b/tests/test_cli_startup.py new file mode 100644 index 0000000..e009256 --- /dev/null +++ b/tests/test_cli_startup.py @@ -0,0 +1,28 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import pytest + +from .plugin_fixtures import run_isolated_python + +# Importing a platform CLI must only build its command tree: the console +# scripts import it before Click can answer a shell completion request, which +# the completion scripts make on every keystroke. Every command imports what +# it runs when it is invoked. Each of these costs tens of milliseconds to +# import and is the sign that a command implementation is imported too early. +HEAVY_MODULES = ("pydantic", "requests", "Crypto", "mvt.common.module") + + +@pytest.mark.parametrize("cli_module", ("mvt.ios.cli", "mvt.android.cli")) +def test_importing_a_cli_does_not_import_the_module_machinery(cli_module, tmp_path): + result = run_isolated_python( + "import sys\n" + f"import {cli_module}\n" + f"print(','.join(name for name in {HEAVY_MODULES!r} if name in sys.modules))\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "", f"{cli_module} imported {result.stdout.strip()}" From 531d63ab071dc039f0837b94626edc6816f2eafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Tue, 1 Sep 2026 15:49:25 +0200 Subject: [PATCH 42/43] Release monthly from a scheduled workflow The package version now comes from the latest v* git tag, so a release is a tag and nothing else. A scheduled workflow tags main on the first of the month when something was merged since the last release, creates the GitHub release with generated notes, and publishes to PyPI and the container registry. Pushing a v* tag by hand goes through the same path. --- .github/workflows/publish-release-docker.yml | 2 + .github/workflows/release.yml | 44 ++++++++++++++++++++ Dockerfile | 1 + Makefile | 5 --- pyproject.toml | 7 ++-- src/mvt/common/version.py | 4 +- 6 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/publish-release-docker.yml b/.github/workflows/publish-release-docker.yml index b3fde3a..a90202d 100644 --- a/.github/workflows/publish-release-docker.yml +++ b/.github/workflows/publish-release-docker.yml @@ -37,6 +37,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + fetch-depth: 0 # the package version comes from the git tags # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..adec25c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + schedule: + - cron: "0 9 1 * *" # first of the month + workflow_dispatch: + push: + tags: ["v*"] + +jobs: + tag: + if: github.ref_type != 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - run: | + last=$(git describe --tags --abbrev=0 --match 'v*') + if [ "$(git rev-list "$last..HEAD" --count)" = 0 ]; then + echo "Nothing merged since $last"; exit 0 + fi + tag=v$(date -u +%Y.%-m.%-d) + gh release create "$tag" --target "$GITHUB_SHA" --generate-notes + # Events made with GITHUB_TOKEN don't start other workflows; run them by hand. + gh workflow run release.yml --ref "$tag" + gh workflow run publish-release-docker.yml --ref "$tag" + + publish: + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + id-token: write # PyPI trusted publishing + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v10.0.0 + - run: uv build && uv publish diff --git a/Dockerfile b/Dockerfile index 58bc216..138edc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,6 +122,7 @@ RUN apt-get update \ binutils \ default-jre-headless \ file \ + git \ jq \ less \ libcurl4 \ diff --git a/Makefile b/Makefile index 528c697..6214009 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,3 @@ clean: dist: $(UV) build -upload: - $(UV) tool run twine upload dist/* - -test-upload: - $(UV) tool run twine upload --repository testpypi dist/* diff --git a/pyproject.toml b/pyproject.toml index fb90f8d..3124c46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ docs = [ ] [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=61.0", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" [tool.coverage.run] @@ -121,5 +121,6 @@ where = ["src"] [tool.setuptools.package-data] mvt = ["ios/data/*.json"] -[tool.setuptools.dynamic] -version = { attr = "mvt.common.version.MVT_VERSION" } +[tool.setuptools_scm] +# The version is the latest v* tag; ignore the archive/* tags. +git_describe_command = "git describe --dirty --tags --long --match 'v*'" diff --git a/src/mvt/common/version.py b/src/mvt/common/version.py index c8e7716..3149604 100644 --- a/src/mvt/common/version.py +++ b/src/mvt/common/version.py @@ -3,4 +3,6 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -MVT_VERSION = "2026.7.29" +from importlib.metadata import version + +MVT_VERSION = version("mvt") From 27cfe7f1e7d98919a6db3b221651e06d2d90b4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Tue, 1 Sep 2026 19:31:23 +0200 Subject: [PATCH 43/43] Release weekly instead of monthly --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adec25c..3c20dc7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release on: schedule: - - cron: "0 9 1 * *" # first of the month + - cron: "0 9 * * 1" # Mondays, 09:00 UTC workflow_dispatch: push: tags: ["v*"]