From 2781f33fb5c0ec4e79d17958ee7c0e8a81a542b7 Mon Sep 17 00:00:00 2001 From: Nex Date: Sat, 13 Aug 2022 14:04:10 +0200 Subject: [PATCH] Added more date conversion wrappers --- mvt/android/parsers/backup.py | 1 - mvt/common/utils.py | 73 +++++++++++-------- mvt/ios/modules/fs/filesystem.py | 1 - mvt/ios/modules/fs/webkit_base.py | 6 +- mvt/ios/modules/mixed/firefox_favicon.py | 1 - .../mixed/webkit_resource_load_statistics.py | 15 ++-- setup.cfg | 2 +- tests/common/test_date_conversions.py | 14 +++- tests/common/test_indicators.py | 1 + 9 files changed, 66 insertions(+), 48 deletions(-) diff --git a/mvt/android/parsers/backup.py b/mvt/android/parsers/backup.py index a5a4098..55088c6 100644 --- a/mvt/android/parsers/backup.py +++ b/mvt/android/parsers/backup.py @@ -3,7 +3,6 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import datetime import io import json import tarfile diff --git a/mvt/common/utils.py b/mvt/common/utils.py index 025408c..90c301b 100644 --- a/mvt/common/utils.py +++ b/mvt/common/utils.py @@ -9,36 +9,6 @@ import re from typing import Union -def convert_mactime_to_datetime(timestamp: Union[int, float], - from_2001: bool = True): - """Converts Mac Standard Time to a datetime. - - :param timestamp: MacTime timestamp (either int or float). - :type timestamp: int - :param from_2001: bool: Whether to (Default value = True) - :param from_2001: Default value = True) - :returns: datetime. - - """ - if not timestamp: - return None - - # This is to fix formats in case of, for example, SMS messages database - # timestamp format. - if isinstance(timestamp, int) and len(str(timestamp)) == 18: - timestamp = int(str(timestamp)[:9]) - - # MacTime counts from 2001-01-01. - if from_2001: - timestamp = timestamp + 978307200 - - # TODO: This is rather ugly. Happens sometimes with invalid timestamps. - try: - return datetime.datetime.utcfromtimestamp(timestamp) - except Exception: - return None - - def convert_chrometime_to_datetime(timestamp: int) -> int: """Converts Chrome timestamp to a datetime. @@ -67,6 +37,17 @@ def convert_datetime_to_iso(datetime: datetime.datetime) -> str: return "" +def convert_unix_to_utc_datetime(timestamp: int) -> datetime.datetime: + """Converts a unix epoch timestamp to UTC datetime. + + :param timestamp: Epoc timestamp to convert. + :type timestamp: int + :returns: datetime. + + """ + return datetime.datetime.utcfromtimestamp(int(timestamp)) + + def convert_unix_to_iso(timestamp: int) -> str: """Converts a unix epoch to ISO string. @@ -77,11 +58,41 @@ def convert_unix_to_iso(timestamp: int) -> str: """ try: - return convert_datetime_to_iso(datetime.datetime.utcfromtimestamp(int(timestamp))) + return convert_datetime_to_iso(convert_unix_to_utc_datetime(timestamp)) except Exception: return "" +def convert_mactime_to_datetime(timestamp: Union[int, float], + from_2001: bool = True): + """Converts Mac Standard Time to a datetime. + + :param timestamp: MacTime timestamp (either int or float). + :type timestamp: int + :param from_2001: bool: Whether to (Default value = True) + :param from_2001: Default value = True) + :returns: datetime. + + """ + if not timestamp: + return None + + # This is to fix formats in case of, for example, SMS messages database + # timestamp format. + if isinstance(timestamp, int) and len(str(timestamp)) == 18: + timestamp = int(str(timestamp)[:9]) + + # MacTime counts from 2001-01-01. + if from_2001: + timestamp = timestamp + 978307200 + + # TODO: This is rather ugly. Happens sometimes with invalid timestamps. + try: + return convert_unix_to_utc_datetime(timestamp) + except Exception: + return None + + def convert_mactime_to_iso(timestamp: int, from_2001: bool = True): """Wraps two conversions from mactime to iso date. diff --git a/mvt/ios/modules/fs/filesystem.py b/mvt/ios/modules/fs/filesystem.py index 4cca600..cb4017e 100644 --- a/mvt/ios/modules/fs/filesystem.py +++ b/mvt/ios/modules/fs/filesystem.py @@ -3,7 +3,6 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import datetime import logging import os from typing import Union diff --git a/mvt/ios/modules/fs/webkit_base.py b/mvt/ios/modules/fs/webkit_base.py index b944664..e922291 100644 --- a/mvt/ios/modules/fs/webkit_base.py +++ b/mvt/ios/modules/fs/webkit_base.py @@ -3,10 +3,9 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import datetime import os -from mvt.common.utils import convert_datetime_to_iso +from mvt.common.utils import convert_unix_to_iso from ..base import IOSExtraction @@ -35,10 +34,9 @@ class WebkitBase(IOSExtraction): name = name.replace("http_", "http://") name = name.replace("https_", "https://") url = name.split("_")[0] - utc_timestamp = datetime.datetime.utcfromtimestamp(os.stat(found_path).st_mtime) self.results.append({ "folder": key, "url": url, - "isodate": convert_datetime_to_iso(utc_timestamp), + "isodate": convert_unix_to_iso(os.stat(found_path).st_mtime), }) diff --git a/mvt/ios/modules/mixed/firefox_favicon.py b/mvt/ios/modules/mixed/firefox_favicon.py index 8e4b87b..3dda16a 100644 --- a/mvt/ios/modules/mixed/firefox_favicon.py +++ b/mvt/ios/modules/mixed/firefox_favicon.py @@ -5,7 +5,6 @@ import logging import sqlite3 -from datetime import datetime from typing import Union from mvt.common.utils import convert_unix_to_iso diff --git a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py index 83764d9..210b3f7 100644 --- a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py +++ b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py @@ -3,12 +3,11 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import datetime import logging import os import sqlite3 -from mvt.common.utils import convert_datetime_to_iso +from mvt.common.utils import convert_unix_to_iso from ..base import IOSExtraction @@ -72,11 +71,12 @@ class WebkitResourceLoadStatistics(IOSExtraction): "registrable_domain": row[1], "last_seen": row[2], "had_user_interaction": bool(row[3]), - "last_seen_isodate": convert_datetime_to_iso(datetime.datetime.utcfromtimestamp(int(row[2]))), + "last_seen_isodate": convert_unix_to_iso(row[2]), }) if len(self.results[key]) > 0: - self.log.info("Extracted a total of %d records from %s", len(self.results[key]), db_path) + self.log.info("Extracted a total of %d records from %s", + len(self.results[key]), db_path) def run(self) -> None: if self.is_backup: @@ -87,9 +87,8 @@ class WebkitResourceLoadStatistics(IOSExtraction): if db_path: self._process_observations_db(db_path=db_path, key=key) except Exception as exc: - self.log.info("Unable to search for WebKit observations.db: %s", - exc) + self.log.info("Unable to find WebKit observations.db: %s", exc) elif self.is_fs_dump: for db_path in self._get_fs_files_from_patterns(WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS): - self._process_observations_db(db_path=db_path, - key=os.path.relpath(db_path, self.target_path)) + db_rel_path = os.path.relpath(db_path, self.target_path) + self._process_observations_db(db_path=db_path, key=db_rel_path) diff --git a/setup.cfg b/setup.cfg index 3603f31..0101de6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ output-format = colorized max-locals = 25 max-args = 10 -good-names = i,e,m +good-names = i,m min-similarity-lines = 10 ignore-comments = yes diff --git a/tests/common/test_date_conversions.py b/tests/common/test_date_conversions.py index a292e52..8511d3f 100644 --- a/tests/common/test_date_conversions.py +++ b/tests/common/test_date_conversions.py @@ -3,7 +3,9 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from mvt.common.utils import convert_mactime_to_iso, convert_unix_to_iso +from mvt.common.utils import (convert_datetime_to_iso, convert_mactime_to_iso, + convert_unix_to_iso, + convert_unix_to_utc_datetime) TEST_DATE_EPOCH = 1626566400 TEST_DATE_ISO = "2021-07-18 00:00:00.000000" @@ -17,3 +19,13 @@ class TestDateConversions: def test_convert_mactime_to_iso(self): assert convert_mactime_to_iso(TEST_DATE_MAC) == TEST_DATE_ISO + + def test_convert_unix_to_utc_datetime(self): + converted = convert_unix_to_utc_datetime(TEST_DATE_EPOCH) + assert converted.year == 2021 + assert converted.month == 7 + assert converted.day == 18 + + def test_convert_datetime_to_iso(self): + converted = convert_unix_to_utc_datetime(TEST_DATE_EPOCH) + assert convert_datetime_to_iso(converted) == TEST_DATE_ISO diff --git a/tests/common/test_indicators.py b/tests/common/test_indicators.py index bc18c07..d2daa07 100644 --- a/tests/common/test_indicators.py +++ b/tests/common/test_indicators.py @@ -10,6 +10,7 @@ from mvt.common.indicators import Indicators class TestIndicators: + def test_parse_stix2(self, indicator_file): ind = Indicators(log=logging) ind.load_indicators_files([indicator_file], load_default=False)