Fix merge conflicts

This commit is contained in:
Hamza Z
2021-08-08 20:01:02 +02:00
committed by Hamza Z
82 changed files with 1261 additions and 579 deletions
+3 -3
View File
@@ -1,6 +1,6 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from .cli import cli
+33 -14
View File
@@ -1,17 +1,19 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import sys
import click
import argparse
import logging
import os
import sys
import click
from rich.logging import RichHandler
from mvt.common.module import run_module, save_timeline
from mvt.common.indicators import Indicators
from mvt.common.module import run_module, save_timeline
from .download_apks import DownloadAPKs
from .lookups.koodous import koodous_lookup
from .lookups.virustotal import virustotal_lookup
@@ -45,8 +47,8 @@ def cli():
@click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
@click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
@click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
@click.option("--output", "-o", type=click.Path(exists=True),
help="Specify a path to a folder where you want to store JSON results")
@click.option("--output", "-o", type=click.Path(exists=False),
help="Specify a path to a folder where you want to store the APKs")
@click.option("--from-file", "-f", type=click.Path(exists=True),
help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)")
@click.option("--serial", "-s", type=str, help="Use the Android device with a given serial number")
@@ -55,9 +57,12 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file,
if from_file:
download = DownloadAPKs.from_json(from_file)
else:
if not output:
log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device")
sys.exit(-1)
if output and not os.path.exists(output):
try:
os.makedirs(output)
except Exception as e:
log.critical("Unable to create output folder %s: %s", output, e)
sys.exit(-1)
download = DownloadAPKs(output_folder=output, all_apks=all_apks, serial=serial)
download.run()
@@ -82,7 +87,7 @@ def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file,
#==============================================================================
@cli.command("check-adb", help="Check an Android device over adb")
@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
@click.option("--output", "-o", type=click.Path(exists=True),
@click.option("--output", "-o", type=click.Path(exists=False),
help="Specify a path to a folder where you want to store JSON results")
@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
@@ -97,6 +102,13 @@ def check_adb(iocs, output, list_modules, module, serial):
log.info("Checking Android through adb bridge")
if output and not os.path.exists(output):
try:
os.makedirs(output)
except Exception as e:
log.critical("Unable to create output folder %s: %s", output, e)
sys.exit(-1)
if iocs:
# Pre-load indicators for performance reasons.
log.info("Loading indicators from provided file at %s", iocs)
@@ -129,12 +141,19 @@ def check_adb(iocs, output, list_modules, module, serial):
#==============================================================================
@cli.command("check-backup", help="Check an Android Backup")
@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE)
@click.option("--serial", "-s", type=str, help="Use the Android device with a given serial")
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
def check_backup(iocs, output, backup_path, serial):
log.info("Checking ADB backup located at: %s", backup_path)
if output and not os.path.exists(output):
try:
os.makedirs(output)
except Exception as e:
log.critical("Unable to create output folder %s: %s", output, e)
sys.exit(-1)
if iocs:
# Pre-load indicators for performance reasons.
log.info("Loading indicators from provided file at %s", iocs)
+8 -6
View File
@@ -1,22 +1,24 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import json
import logging
import os
import pkg_resources
from tqdm import tqdm
from mvt.common.utils import get_sha256_from_file_path
from .modules.adb.base import AndroidExtraction
log = logging.getLogger(__name__)
# TODO: Would be better to replace tqdm with rich.progress to reduce
# the number of dependencies. Need to investigate whether
# it's possible to have a simialr callback system.
# it's possible to have a similar callback system.
class PullProgress(tqdm):
"""PullProgress is a tqdm update system for APK downloads."""
@@ -42,7 +44,7 @@ class DownloadAPKs(AndroidExtraction):
"""Initialize module.
:param output_folder: Path to the folder where data should be stored
:param all_apks: Boolean indicating whether to download all packages
or filter known-goods
or filter known-goods
:param packages: Provided list of packages, typically for JSON checks
:param serial: The USB device serial ID
"""
+3 -3
View File
@@ -1,4 +1,4 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
+7 -7
View File
@@ -1,15 +1,15 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import requests
import logging
from rich.text import Text
from rich.table import Table
from rich.progress import track
import requests
from rich.console import Console
from rich.progress import track
from rich.table import Table
from rich.text import Text
log = logging.getLogger(__name__)
+8 -7
View File
@@ -1,14 +1,15 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
import requests
import logging
from rich.text import Text
from rich.table import Table
from rich.progress import track
from rich.console import Console
from rich.progress import track
from rich.table import Table
from rich.text import Text
log = logging.getLogger(__name__)
+3 -3
View File
@@ -1,4 +1,4 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
+5 -5
View File
@@ -1,17 +1,17 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from .chrome_history import ChromeHistory
from .dumpsys_batterystats import DumpsysBatterystats
from .dumpsys_packages import DumpsysPackages
from .dumpsys_procstats import DumpsysProcstats
from .packages import Packages
from .processes import Processes
from .rootbinaries import RootBinaries
from .sms import SMS
from .whatsapp import Whatsapp
from .packages import Packages
from .rootbinaries import RootBinaries
ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
DumpsysBatterystats, DumpsysProcstats,
+60 -12
View File
@@ -1,20 +1,22 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import sys
import time
import logging
import os
import random
import string
import sys
import tempfile
import time
from adb_shell.adb_device import AdbDeviceUsb, AdbDeviceTcp
from adb_shell.auth.keygen import keygen, write_public_keyfile
from adb_shell.auth.sign_pythonrsa import PythonRSASigner
from adb_shell.exceptions import DeviceAuthError, AdbCommandFailureException
from usb1 import USBErrorBusy, USBErrorAccess
from adb_shell.exceptions import AdbCommandFailureException, DeviceAuthError
from usb1 import USBErrorAccess, USBErrorBusy
from mvt.common.module import MVTModule
from mvt.common.module import MVTModule, InsufficientPrivileges
log = logging.getLogger(__name__)
@@ -106,13 +108,13 @@ class AndroidExtraction(MVTModule):
"""Check if we have a `su` binary on the Android device.
:returns: Boolean indicating whether a `su` binary is present or not
"""
return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
return bool(self._adb_command("command -v su"))
def _adb_root_or_die(self):
"""Check if we have a `su` binary, otherwise raise an Exception.
"""
if not self._adb_check_if_root():
raise Exception("The Android device does not seem to have a `su` binary. Cannot run this module.")
raise InsufficientPrivileges("The Android device does not seem to have a `su` binary. Cannot run this module.")
def _adb_command_as_root(self, command):
"""Execute an adb shell command.
@@ -120,8 +122,23 @@ class AndroidExtraction(MVTModule):
:returns: Output of command
"""
return self._adb_command(f"su -c {command}")
def _adb_check_file_exists(self, file):
"""Verify that a file exists.
:param file: Path of the file
:returns: Boolean indicating whether the file exists or not
"""
def _adb_download(self, remote_path, local_path, progress_callback=None):
# TODO: Need to support checking files without root privileges as well.
# Connect to the device over adb.
self._adb_connect()
# Check if we have root, if not raise an Exception.
self._adb_root_or_die()
return bool(self._adb_command_as_root(f"[ ! -f {file} ] || echo 1"))
def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True):
"""Download a file form the device.
:param remote_path: Path to download from the device
:param local_path: Path to where to locally store the copy of the file
@@ -129,6 +146,37 @@ class AndroidExtraction(MVTModule):
"""
try:
self.device.pull(remote_path, local_path, progress_callback)
except AdbCommandFailureException as e:
if retry_root:
self._adb_download_root(remote_path, local_path, progress_callback)
else:
raise Exception(f"Unable to download file {remote_path}: {e}")
def _adb_download_root(self, remote_path, local_path, progress_callback=None):
try:
# Check if we have root, if not raise an Exception.
self._adb_root_or_die()
# We generate a random temporary filename.
tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10))
# We create a temporary local file.
new_remote_path = f"/sdcard/{tmp_filename}"
# We copy the file from the data folder to /sdcard/.
cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
if cp.startswith("cp: ") and "No such file or directory" in cp:
raise Exception(f"Unable to process file {remote_path}: File not found")
elif cp.startswith("cp: ") and "Permission denied" in cp:
raise Exception(f"Unable to process file {remote_path}: Permission denied")
# We download from /sdcard/ to the local temporary file.
# If it doesn't work now, don't try again (retry_root=False)
self._adb_download(new_remote_path, local_path, retry_root=False)
# Delete the copy on /sdcard/.
self._adb_command(f"rm -rf {new_remote_path}")
except AdbCommandFailureException as e:
raise Exception(f"Unable to download file {remote_path}: {e}")
+6 -5
View File
@@ -1,13 +1,14 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
import os
import sqlite3
import logging
from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso
from mvt.common.utils import (convert_chrometime_to_unix,
convert_timestamp_to_iso)
from .base import AndroidExtraction
@@ -1,10 +1,10 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import logging
import os
from .base import AndroidExtraction
+4 -4
View File
@@ -1,10 +1,10 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import logging
import os
from .base import AndroidExtraction
+4 -4
View File
@@ -1,10 +1,10 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import logging
import os
from .base import AndroidExtraction
+5 -4
View File
@@ -1,10 +1,11 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import logging
import os
import pkg_resources
from .base import AndroidExtraction
+3 -3
View File
@@ -1,7 +1,7 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
+5 -4
View File
@@ -1,10 +1,11 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import logging
import os
import pkg_resources
from .base import AndroidExtraction
+46 -21
View File
@@ -1,18 +1,43 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
import os
import sqlite3
import logging
from mvt.common.utils import check_for_links, convert_timestamp_to_iso
from .base import AndroidExtraction
from mvt.common.utils import convert_timestamp_to_iso, check_for_links
log = logging.getLogger(__name__)
SMS_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
SMS_BUGLE_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
SMS_BUGLE_QUERY = """
SELECT
ppl.normalized_destination AS number,
p.timestamp AS timestamp,
CASE WHEN m.sender_id IN
(SELECT _id FROM participants WHERE contact_id=-1)
THEN 2 ELSE 1 END incoming, p.text AS text
FROM messages m, conversations c, parts p,
participants ppl, conversation_participants cp
WHERE (m.conversation_id = c._id)
AND (m._id = p.message_id)
AND (cp.conversation_id = c._id)
AND (cp.participant_id = ppl._id);
"""
SMS_MMSSMS_PATH = "data/data/com.android.providers.telephony/databases/mmssms.db"
SMS_MMSMS_QUERY = """
SELECT
address AS number,
date_sent AS timestamp,
type as incoming,
body AS text
FROM sms;
"""
class SMS(AndroidExtraction):
"""This module extracts all SMS messages containing links."""
@@ -50,20 +75,12 @@ class SMS(AndroidExtraction):
"""
conn = sqlite3.connect(db_path)
cur = conn.cursor()
cur.execute("""
SELECT
ppl.normalized_destination AS number,
p.timestamp AS timestamp,
CASE WHEN m.sender_id IN
(SELECT _id FROM participants WHERE contact_id=-1)
THEN 2 ELSE 1 END incoming, p.text AS text
FROM messages m, conversations c, parts p,
participants ppl, conversation_participants cp
WHERE (m.conversation_id = c._id)
AND (m._id = p.message_id)
AND (cp.conversation_id = c._id)
AND (cp.participant_id = ppl._id);
""")
if (self.SMS_DB_TYPE == 1):
cur.execute(SMS_BUGLE_QUERY)
elif (self.SMS_DB_TYPE == 2):
cur.execute(SMS_MMSMS_QUERY)
names = [description[0] for description in cur.description]
for item in cur:
@@ -85,7 +102,15 @@ class SMS(AndroidExtraction):
log.info("Extracted a total of %d SMS messages containing links", len(self.results))
def run(self):
# Checking the SMS database path
try:
self._adb_process_file(os.path.join("/", SMS_PATH), self._parse_db)
if (self._adb_check_file_exists(os.path.join("/", SMS_BUGLE_PATH))):
self.SMS_DB_TYPE = 1
self._adb_process_file(os.path.join("/", SMS_BUGLE_PATH), self._parse_db)
elif (self._adb_check_file_exists(os.path.join("/", SMS_MMSSMS_PATH))):
self.SMS_DB_TYPE = 2
self._adb_process_file(os.path.join("/", SMS_MMSSMS_PATH), self._parse_db)
else:
self.log.error("No SMS database found")
except Exception as e:
self.log.error(e)
+9 -5
View File
@@ -1,14 +1,16 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import base64
import logging
import os
import sqlite3
import logging
from mvt.common.utils import check_for_links, convert_timestamp_to_iso
from .base import AndroidExtraction
from mvt.common.utils import convert_timestamp_to_iso, check_for_links
log = logging.getLogger(__name__)
@@ -69,6 +71,8 @@ class Whatsapp(AndroidExtraction):
# If we find links in the messages or if they are empty we add them to the list.
if check_for_links(message["data"]) or message["data"].strip() == "":
if (message.get('thumb_image') is not None):
message['thumb_image'] = base64.b64encode(message['thumb_image'])
messages.append(message)
cur.close()
+3 -3
View File
@@ -1,7 +1,7 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from .sms import SMS
+6 -6
View File
@@ -1,15 +1,15 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021 MVT Project Developers.
# See the file 'LICENSE' for usage and copying permissions, or find a copy at
# https://github.com/mvt-project/mvt/blob/main/LICENSE
# Copyright (c) 2021 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import os
import json
import os
import zlib
from mvt.common.module import MVTModule
from mvt.common.utils import check_for_links
from mvt.common.utils import convert_timestamp_to_iso
from mvt.common.utils import check_for_links, convert_timestamp_to_iso
class SMS(MVTModule):