From 1042354be5f91b2f943c0c1c77fe9a2c87377160 Mon Sep 17 00:00:00 2001 From: tek Date: Fri, 13 Jan 2023 12:58:26 +0100 Subject: [PATCH] Adds serializing to iOS module webkit_resource_load_statistics --- .../mixed/webkit_resource_load_statistics.py | 48 ++++++++++-------- .../6e9d0cb750a70e7fa10943c134a5f021dab327e7 | Bin 0 -> 110592 bytes .../test_webkit_resource_load_statistics.py | 23 +++++++++ tests/ios_fs/test_filesystem.py | 8 +-- 4 files changed, 53 insertions(+), 26 deletions(-) create mode 100644 tests/artifacts/ios_backup/6e/6e9d0cb750a70e7fa10943c134a5f021dab327e7 create mode 100644 tests/ios_backup/test_webkit_resource_load_statistics.py diff --git a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py index 78a5640..4dd9405 100644 --- a/mvt/ios/modules/mixed/webkit_resource_load_statistics.py +++ b/mvt/ios/modules/mixed/webkit_resource_load_statistics.py @@ -6,7 +6,7 @@ import logging import os import sqlite3 -from typing import Optional +from typing import Optional, Union from mvt.common.utils import convert_unix_to_iso @@ -22,7 +22,6 @@ WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS = [ class WebkitResourceLoadStatistics(IOSExtraction): """This module extracts records from WebKit ResourceLoadStatistics observations.db.""" - # TODO: Add serialize(). def __init__( self, @@ -37,24 +36,31 @@ class WebkitResourceLoadStatistics(IOSExtraction): results_path=results_path, fast_mode=fast_mode, log=log, results=results) - self.results = {} if not results else results + self.results = [] if not results else results + + def serialize(self, record: dict) -> Union[dict, list]: + msg = f"Webkit resource loaded from {record['registrable_domain']}" + if record["domain"] != "": + msg += f" by app in domain {record['domain']}" + return { + "timestamp": record["last_seen_isodate"], + "module": self.__class__.__name__, + "event": "visit", + "data": msg + } def check_indicators(self) -> None: if not self.indicators: return self.detected = {} - for key, items in self.results.items(): - for item in items: - ioc = self.indicators.check_domain(item["registrable_domain"]) - if ioc: - item["matched_indicator"] = ioc - if key not in self.detected: - self.detected[key] = [item, ] - else: - self.detected[key].append(item) + for result in self.results: + ioc = self.indicators.check_domain(result["registrable_domain"]) + if ioc: + result["matched_indicator"] = ioc + self.detected.append(result) - def _process_observations_db(self, db_path, key): + def _process_observations_db(self, db_path: str, domain: str, path: str) -> None: self.log.info("Found WebKit ResourceLoadStatistics observations.db file at path %s", db_path) @@ -68,21 +74,20 @@ class WebkitResourceLoadStatistics(IOSExtraction): except sqlite3.OperationalError: return - if key not in self.results: - self.results[key] = [] - for row in cur: - self.results[key].append({ + self.results.append({ "domain_id": row[0], "registrable_domain": row[1], "last_seen": row[2], "had_user_interaction": bool(row[3]), "last_seen_isodate": convert_unix_to_iso(row[2]), + "domain": domain, + "path": path }) - if len(self.results[key]) > 0: + if len(self.results) > 0: self.log.info("Extracted a total of %d records from %s", - len(self.results[key]), db_path) + len(self.results), db_path) def run(self) -> None: if self.is_backup: @@ -91,13 +96,12 @@ class WebkitResourceLoadStatistics(IOSExtraction): relative_path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH): db_path = self._get_backup_file_from_id(backup_file["file_id"]) - key = f"{backup_file['domain']}/{WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH}" if db_path: - self._process_observations_db(db_path=db_path, key=key) + self._process_observations_db(db_path=db_path, domain=backup_file['domain'], path=WEBKIT_RESOURCELOADSTATICS_BACKUP_RELPATH) except Exception as exc: self.log.info("Unable to find WebKit observations.db: %s", exc) elif self.is_fs_dump: for db_path in self._get_fs_files_from_patterns( WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS): db_rel_path = os.path.relpath(db_path, self.target_path) - self._process_observations_db(db_path=db_path, key=db_rel_path) + self._process_observations_db(db_path=db_path, domain="", path=db_rel_path) diff --git a/tests/artifacts/ios_backup/6e/6e9d0cb750a70e7fa10943c134a5f021dab327e7 b/tests/artifacts/ios_backup/6e/6e9d0cb750a70e7fa10943c134a5f021dab327e7 new file mode 100644 index 0000000000000000000000000000000000000000..12dab73b0fc677366a8ab9939149bcbb31f01e34 GIT binary patch literal 110592 zcmeI&&2QW09l&wXZ}LkNrAZSdEw-}D#X^h@MGrd-nJQ785YLV!d2QNl0wdA3W=V8N zI%tsfV8uiC2kgFHy8Dg;2JE!`4FiTyG+P;z~kM50tg_000IagfB*sr zAbehBu-P+yU+)!J(?d%>kbjN6@qT)@@AxG*|=-I{XvQ~O; zOTDK(2sD-z^%b?Uww2c^>h?q1upS$YL4fOxUZAByR^8rG3)-e8BFg6~`CLKUP`$PK zJ27sp+3wLw?}9shzsElm-!%1x(Wo9Yt;T)ba-LM`R`bZQV-BxMgAh|#UH1nOG7e&> zEWaR#CrQH9al#74(Z3Z6=%D~ax#<8E2ld$d9!m^FzTkIJ594=7cH}WCKx=l z5>6c~=rZ@GS14V?MBfuVh&WT8j>CfEs>s~lrpK)@+bt*i&UYl|-A?n-u~9Y}P0Oe| zl5@zR+`;mtJe+N4$V>G3M@*r zvGu8EIYG#ZA?|gX#$%&3a?Nqmi2U|eakG@Ks>NJMWH#b$rCQF3lt|l z`3GR^;7&EA7u>IMz2JO#q3`XS6rAs1$b^xCvW)$v?O1NE-5-6ms(m2R-a+cC?rxQK zcD0a+<7NsK!|3?);RC&~E3(*9$1yCu?&dBbGqr8nPT8o7rQcV!+H4#C8uu-|)7aCU z1H%&OcQ83`+V?HvvECB3g&zH}VLgc|A<}wXbjGwAcG+l~BL5Cn1y=((q*5!UkX;xv@7)kqjB6a8b#B}wOa2UJ9*PQY#N1L??t!u zR^YVk_ve-}J9m^sQ*56;wjZ^`&9|l>J7({F%{NtQZ~H!&7dy9QclVZB`)%TK;?uyzvTmZ@jvkIk|pM>Av3!KwQVY zjoXObNljdi(#Eb9lD5axkr`ZDgUg__RnR_AW6n8OyQ;V@V(L-XReuBR zk{LU%bg#)S!SrABZMewSy!q!@SAwV9h$IM>AJv2uaqzTWw8Oq_sNT+r0Dpc}q?sbX z+fsnOe6c8YC^b3l4h=HowChnphLn$L#v5qJscxWBI`0p*yZefm=HGp)JX?#3HZHNP z$=Rr9ScZe`>s(xgufBj$TwS9(7oitdsgRXp-Vhhia7u|u^ph*%D!J>wN&*U_`!*8p z+6&Q)ZXh!FxbQwni)k~zt#se%O&jT^5|*3R8iUNvU9hD5C-&5M+1?Y)b37)Kme^og z<=q2>+;zsyZ7<*YT<^^Sr_lPt!aL)0F6+_h8(i%%1&u zCKZGaeO^DDm0`=#rP2Jsx?ai%$k}1w`juzugx20rrQ&jtuu9xpi(E?FN8H8+EBfGvyMSj; zKF+weQ0?p%8nIf0Z6-W9R@6fCT394eC&aYXyL(1-)lD3pA08$|a&pwYaYNc$N!1oiSkGt@IRrJ^Rjvz|6)`97_S33}zeu_&(zWbnR?-K8WN*~p zGjERY7Bg`^b(5I_I{1Q0*~0R#|0009K1DZu`J zn)1NB7gt_2q1s}0tg_000IagFi8RS|C7|FEd&ri009ILKmY**5I_I{1g0av{(m~| zw2A-%2q1s}0tg_000IagfWRaL*#A#bo3;=@009ILKmY**5I_I{1Q3{x0Q>*xxYH^E z2q1s}0tg_000IagfB*uM6kz{9Np0Ff009ILKmY**5I_I{1Q0-AIs)wfr{hkm2q1s} z0tg_000IagfB*srOj3aT|0K0(3jqWWKmY**5I_I{1Q0*~f$0da|DTRKts;N`0tg_0 R00IagfB*srATUXRZvcn31dRXy literal 0 HcmV?d00001 diff --git a/tests/ios_backup/test_webkit_resource_load_statistics.py b/tests/ios_backup/test_webkit_resource_load_statistics.py new file mode 100644 index 0000000..07f657f --- /dev/null +++ b/tests/ios_backup/test_webkit_resource_load_statistics.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2022 Claudio Guarnieri. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging + +from mvt.common.indicators import Indicators +from mvt.common.module import run_module +from mvt.ios.modules.mixed.webkit_resource_load_statistics import WebkitResourceLoadStatistics + +from ..utils import get_ios_backup_folder + + +class TestWebkitResourceLoadStatisticsModule: + + def test_webkit(self): + m = WebkitResourceLoadStatistics(target_path=get_ios_backup_folder()) + m.is_backup = True + run_module(m) + assert len(m.results) == 2 + assert len(m.timeline) == 2 + assert len(m.detected) == 0 diff --git a/tests/ios_fs/test_filesystem.py b/tests/ios_fs/test_filesystem.py index 935aba1..dd82cf4 100644 --- a/tests/ios_fs/test_filesystem.py +++ b/tests/ios_fs/test_filesystem.py @@ -17,8 +17,8 @@ class TestFilesystem: def test_filesystem(self): m = Filesystem(target_path=get_ios_backup_folder()) run_module(m) - assert len(m.results) == 10 - assert len(m.timeline) == 10 + assert len(m.results) == 12 + assert len(m.timeline) == 12 assert len(m.detected) == 0 def test_detection(self, indicator_file): @@ -29,6 +29,6 @@ class TestFilesystem: ind.ioc_collections[0]["processes"].append("64d0019cb3d46bfc8cce545a8ba54b93e7ea9347") m.indicators = ind run_module(m) - assert len(m.results) == 10 - assert len(m.timeline) == 10 + assert len(m.results) == 12 + assert len(m.timeline) == 12 assert len(m.detected) == 1