Parse Android browser history databases

This commit is contained in:
Janik Besendorf
2026-08-17 14:12:50 +02:00
parent 0fdc2c34b0
commit c5d7f5a1d0
9 changed files with 747 additions and 0 deletions
+21
View File
@@ -62,6 +62,27 @@ AndroidQF will prompt the user to download, decrypt and collect device intrusion
For cases where intrusion logs were collected outside of an AndroidQF acquisition, the standalone `mvt-android check-intrusion-logs` command can analyse them directly. See [Check Android Intrusion Logs](intrusion_logs.md) for details, and the [feature announcment from Amnesty International's Security Lab](https://securitylab.amnesty.org/latest/2026/05/android-intrusion-logging-as-a-new-source-of-data-for-consensual-forensic-analysis/) for background on the data source.
## Browser history
AndroidQF can optionally collect browser `History` databases when a device
already has working root access. When its acquisition contains the
`browser_history/manifest.json` file, `mvt-android check-androidqf` parses every
listed database, including any collected SQLite WAL and SHM sidecars. Browser
visits are added to the module results, timeline, URL output, and indicator
checks.
MVT can also inspect the supported database locations in a full Android
filesystem dump, or inspect one explicitly supplied Chromium `History` file:
```bash
mvt-android check-fs --output /path/to/results /path/to/filesystem-dump
mvt-android check-fs --output /path/to/results /path/to/History
```
The built-in locations cover Chrome, Brave, Microsoft Edge, and Samsung
Internet. Other Chromium-compatible databases can be supplied explicitly as a
file without assigning an unverified browser identity.
## Android Debug Bridge analysis removed
The ability to analyze Android devices directly over ADB has been removed from MVT. Direct extraction of data from ADB was error-prone and frequently resulted in inconsistent data collection between ADB and AndroidQF acquisitions. Use AndroidQF for device acquisition and `mvt-android check-androidqf` for analysis.
@@ -0,0 +1,162 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import sqlite3
import shutil
import tempfile
from pathlib import Path
from typing import Any
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
from mvt.common.utils import convert_chrometime_to_datetime, convert_datetime_to_iso
from .artifact import AndroidArtifact
class BrowserHistoryArtifact(AndroidArtifact):
"""Shared Chromium History database parsing and result handling."""
def _parse_browser_history(
self,
connection: sqlite3.Connection,
*,
browser: str,
package: str,
profile: str,
source_path: str,
) -> None:
cursor = connection.cursor()
try:
cursor.execute(
"""
SELECT
urls.id,
urls.url,
urls.title,
urls.visit_count,
urls.typed_count,
visits.id,
visits.visit_time,
visits.from_visit,
visits.transition
FROM urls
JOIN visits ON visits.url = urls.id
ORDER BY visits.visit_time;
"""
)
for row in cursor:
timestamp = int(row[6])
self.results.append(
{
"id": row[0],
"url": row[1],
"title": row[2],
"visit_count": row[3],
"typed_count": row[4],
"visit_id": row[5],
"timestamp": timestamp,
"isodate": convert_datetime_to_iso(
convert_chrometime_to_datetime(timestamp)
),
"redirect_source": row[7],
"transition": row[8],
"browser": browser,
"package": package,
"profile": profile,
"source_path": source_path,
}
)
finally:
cursor.close()
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
return {
"timestamp": record["isodate"],
"module": self.__class__.__name__,
"event": "browser_history",
"data": (
f"{record['browser']} visit to {record['url']} "
f"(visit ID: {record['visit_id']}, profile: {record['profile']})"
),
}
def check_indicators(self) -> None:
if not self.indicators:
return
for result, match in zip(
self.results,
self.indicators.check_url_batches(
[[result["url"]] for result in self.results]
),
):
if match:
self.alertstore.critical(
match.message, "", result, matched_indicator=match.ioc
)
def collect_url_results(self) -> None:
for result in self.results:
self.add_url_result(result["url"], result.get("isodate"), "browser_history")
class TemporarySQLiteConnection(sqlite3.Connection):
temporary_directory: tempfile.TemporaryDirectory | None = None
def close(self) -> None:
try:
super().close()
finally:
if self.temporary_directory:
self.temporary_directory.cleanup()
self.temporary_directory = None
def open_browser_history_database(database_path: Path) -> sqlite3.Connection:
"""Open a staged History database without modifying forensic evidence."""
database_uri = database_path.resolve().as_uri()
if not Path(f"{database_path}-wal").is_file():
return sqlite3.connect(f"{database_uri}?mode=ro&immutable=1", uri=True)
temporary_directory = tempfile.TemporaryDirectory(prefix="mvt_sqlite_")
temporary_path = Path(temporary_directory.name) / database_path.name
shutil.copy2(database_path, temporary_path)
for suffix in ("-wal", "-shm"):
sidecar = Path(f"{database_path}{suffix}")
if sidecar.is_file():
shutil.copy2(sidecar, Path(f"{temporary_path}{suffix}"))
try:
connection = sqlite3.connect(
f"{temporary_path.resolve().as_uri()}?mode=ro",
uri=True,
factory=TemporarySQLiteConnection,
)
except Exception:
temporary_directory.cleanup()
raise
connection.temporary_directory = temporary_directory
return connection
def validate_manifest_database(database: Any) -> dict[str, Any]:
if not isinstance(database, dict):
raise ValueError("database entry is not an object")
required = ("browser", "package", "profile", "device_path", "archive_path")
for field in required:
if not isinstance(database.get(field), str) or not database[field]:
raise ValueError(f"database entry has invalid {field}")
archive_path = database["archive_path"]
path = Path(archive_path)
if (
"\\" in archive_path
or path.is_absolute()
or ".." in path.parts
or path.parts[:1] != ("browser_history",)
):
raise ValueError(f"unsafe browser history archive path: {archive_path}")
return database
+53
View File
@@ -28,6 +28,7 @@ from mvt.common.help import (
HELP_MSG_CHECK_ANDROID_BACKUP,
HELP_MSG_CHECK_ANDROIDQF,
HELP_MSG_CHECK_BUGREPORT,
HELP_MSG_CHECK_FS,
HELP_MSG_CHECK_IOCS,
HELP_MSG_CHECK_INTRUSION_LOGS,
HELP_MSG_DELAY_CHECKS,
@@ -54,6 +55,7 @@ from mvt.common.utils import init_logging, set_verbose_logging
from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
from .cmd_check_backup import CmdAndroidCheckBackup
from .cmd_check_bugreport import CmdAndroidCheckBugreport
from .cmd_check_fs import CmdAndroidCheckFS
from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
from .modules.intrusion_logs import INTRUSION_LOGS_MODULES
from .modules.androidqf import ANDROIDQF_MODULES
@@ -379,6 +381,57 @@ def check_androidqf(
cmd.show_support_message()
# ==============================================================================
# Command: check-fs
# ==============================================================================
@cli.command("check-fs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_CHECK_FS)
@click.option(
"--iocs",
"-i",
type=click.Path(exists=True),
multiple=True,
default=[],
help=HELP_MSG_IOC,
)
@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
@click.option("--module", "-m", help=HELP_MSG_MODULE)
@click.option(
"--load-module",
type=click.Path(exists=True),
multiple=True,
default=[],
help=HELP_MSG_LOAD_MODULE,
)
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
@click.argument("DUMP_PATH", type=click.Path(exists=True))
@click.pass_context
def check_fs(
ctx, iocs, output, list_modules, module, load_module, hashes, verbose, dump_path
):
set_verbose_logging(verbose)
custom_modules = _load_custom_modules(load_module)
cmd = CmdAndroidCheckFS(
target_path=dump_path,
results_path=output,
ioc_files=iocs,
module_name=module,
hashes=hashes,
disable_version_check=_get_disable_flags(ctx)[0],
disable_indicator_check=_get_disable_flags(ctx)[1],
custom_modules=custom_modules,
)
if list_modules:
cmd.list_modules()
return
log.info("Checking Android filesystem located at: %s", dump_path)
cmd.run()
cmd.show_alerts_brief()
cmd.show_support_message()
# ==============================================================================
# Command: check-intrusion-logs
# ==============================================================================
+47
View File
@@ -0,0 +1,47 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
from typing import Optional
from mvt.common.command import Command
from mvt.common.indicators import Indicators
from mvt.common.module import MVTModule
from .modules.fs import FS_MODULES
log = logging.getLogger(__name__)
class CmdAndroidCheckFS(Command):
def __init__(
self,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
ioc_files: Optional[list] = None,
iocs: Optional[Indicators] = None,
module_name: Optional[str] = None,
module_options: Optional[dict] = None,
hashes: bool = False,
disable_version_check: bool = False,
disable_indicator_check: bool = False,
custom_modules: Optional[list[type[MVTModule]]] = None,
) -> None:
super().__init__(
target_path=target_path,
results_path=results_path,
ioc_files=ioc_files,
iocs=iocs,
module_name=module_name,
module_options=module_options,
hashes=hashes,
log=log,
disable_version_check=disable_version_check,
disable_indicator_check=disable_indicator_check,
custom_modules=custom_modules,
)
self.platform = "android"
self.name = "check-fs"
self.modules = FS_MODULES
@@ -9,6 +9,7 @@ from .aqf_log_timestamps import AQFLogTimestamps
from .aqf_packages import AQFPackages
from .aqf_processes import AQFProcesses
from .aqf_settings import AQFSettings
from .browser_history import BrowserHistory
from .mounts import Mounts
from .root_binaries import RootBinaries
@@ -21,4 +22,5 @@ ANDROIDQF_MODULES = [
AQFLogTimestamps,
RootBinaries,
Mounts,
BrowserHistory,
]
@@ -0,0 +1,114 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import json
import sqlite3
import tempfile
from pathlib import Path, PurePosixPath
from mvt.android.artifacts.browser_history import (
BrowserHistoryArtifact,
open_browser_history_database,
validate_manifest_database,
)
from .base import AndroidQFModule
class BrowserHistory(BrowserHistoryArtifact, AndroidQFModule):
"""Extract browser visits collected by AndroidQF."""
supported_commands = (("android", "check-androidqf"),)
def _find_manifest(self) -> str | None:
manifests = [
file_path
for file_path in self.files
if file_path.replace("\\", "/").endswith("browser_history/manifest.json")
]
if not manifests:
return None
if len(manifests) > 1:
self.log.warning(
"Found multiple browser history manifests; using %s", manifests[0]
)
return manifests[0]
def _stage_database(
self, archive_path: str, prefix: str, temporary_path: Path
) -> Path:
available_files = {
file_path.replace("\\", "/"): file_path for file_path in self.files
}
normalized_path = str(PurePosixPath(prefix, archive_path))
source_path = available_files.get(normalized_path)
if not source_path:
raise FileNotFoundError(archive_path)
staged_path = temporary_path / "History"
staged_path.write_bytes(self._get_file_content(source_path))
for suffix in ("-wal", "-shm"):
sidecar = available_files.get(normalized_path + suffix)
if sidecar:
Path(f"{staged_path}{suffix}").write_bytes(
self._get_file_content(sidecar)
)
return staged_path
def run(self) -> None:
manifest_path = self._find_manifest()
if not manifest_path:
self.log.info("No AndroidQF browser history manifest found")
return
try:
manifest = json.loads(self._get_file_content(manifest_path))
except (json.JSONDecodeError, OSError, TypeError, UnicodeDecodeError) as exc:
self.log.error("Unable to read browser history manifest: %s", exc)
return
if not isinstance(manifest, dict) or manifest.get("schema_version") != 1:
self.log.error("Unsupported AndroidQF browser history manifest")
return
databases = manifest.get("databases", [])
if not isinstance(databases, list):
self.log.error("Invalid AndroidQF browser history database list")
return
normalized_manifest = manifest_path.replace("\\", "/")
marker = "browser_history/manifest.json"
prefix = normalized_manifest[: -len(marker)].rstrip("/")
for raw_database in databases:
try:
database = validate_manifest_database(raw_database)
with tempfile.TemporaryDirectory(prefix="mvt_browser_history_") as temp:
staged_path = self._stage_database(
database["archive_path"], prefix, Path(temp)
)
connection = open_browser_history_database(staged_path)
try:
self._parse_browser_history(
connection,
browser=database["browser"],
package=database["package"],
profile=database["profile"],
source_path=database["device_path"],
)
finally:
connection.close()
except (
FileNotFoundError,
OSError,
OverflowError,
sqlite3.Error,
TypeError,
ValueError,
) as exc:
self.log.error("Unable to parse browser history database: %s", exc)
self.log.info(
"Extracted a total of %d browser history items", len(self.results)
)
+10
View File
@@ -0,0 +1,10 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from mvt.common.module import MVTModule
from .browser_history import BrowserHistory
FS_MODULES: list[type[MVTModule]] = [BrowserHistory]
@@ -0,0 +1,115 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import logging
import sqlite3
from pathlib import Path
from typing import Optional
from mvt.android.artifacts.browser_history import (
BrowserHistoryArtifact,
open_browser_history_database,
)
from mvt.common.module import MVTModule
from mvt.common.module_types import ModuleResults
# These locations are deliberately limited to paths backed by public parser
# fixtures or the historical MVT implementation.
BROWSER_HISTORY_PATHS = {
"data/data/com.android.chrome/app_chrome/Default/History": (
"Chrome",
"com.android.chrome",
"Default",
),
"data/data/com.brave.browser/app_chrome/Default/History": (
"Brave",
"com.brave.browser",
"Default",
),
"data/data/com.microsoft.emmx/app_chrome/Default/History": (
"Microsoft Edge",
"com.microsoft.emmx",
"Default",
),
"data/data/com.sec.android.app.sbrowser/app_sbrowser/Default/History": (
"Samsung Internet",
"com.sec.android.app.sbrowser",
"Default",
),
}
class BrowserHistory(BrowserHistoryArtifact, MVTModule):
"""Extract supported Chromium History databases from a filesystem dump."""
supported_commands = (("android", "check-fs"),)
def __init__(
self,
file_path: Optional[str] = None,
target_path: Optional[str] = None,
results_path: Optional[str] = None,
module_options: Optional[dict] = None,
log: logging.Logger = logging.getLogger(__name__),
results: Optional[ModuleResults] = None,
) -> None:
super().__init__(
file_path=file_path,
target_path=target_path,
results_path=results_path,
module_options=module_options,
log=log,
results=results,
)
def _database_paths(self) -> list[tuple[Path, str, str, str]]:
if not self.target_path:
return []
target = Path(self.target_path)
if target.is_file():
normalized_target = target.resolve().as_posix()
for relative_path, identity in BROWSER_HISTORY_PATHS.items():
if normalized_target.endswith(f"/{relative_path}"):
return [(target, *identity)]
return [(target, "Chromium", "unknown", "unknown")]
databases = []
for relative_path, identity in BROWSER_HISTORY_PATHS.items():
database_path = target / relative_path
if database_path.is_file():
databases.append((database_path, *identity))
return databases
def run(self) -> None:
for database_path, browser, package, profile in self._database_paths():
try:
connection = open_browser_history_database(database_path)
try:
self._parse_browser_history(
connection,
browser=browser,
package=package,
profile=profile,
source_path=str(database_path),
)
finally:
connection.close()
except (
OSError,
OverflowError,
sqlite3.Error,
TypeError,
ValueError,
) as exc:
self.log.error(
"Unable to parse browser history database %s: %s",
database_path,
exc,
)
self.log.info(
"Extracted a total of %d browser history items", len(self.results)
)
+223
View File
@@ -0,0 +1,223 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2026 The MVT Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import json
import sqlite3
import zipfile
from pathlib import Path
import pytest
from mvt.android.modules.androidqf.browser_history import (
BrowserHistory as AndroidQFBrowserHistory,
)
from mvt.android.modules.fs.browser_history import BrowserHistory as FSBrowserHistory
from mvt.android.cmd_check_fs import CmdAndroidCheckFS
CHROME_TIME = 13_348_540_800_000_000
URL = "https://example.org/path"
def create_history_database(path: Path, *, url: str = URL) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with sqlite3.connect(path) as connection:
connection.executescript(
"""
CREATE TABLE urls (
id INTEGER PRIMARY KEY,
url TEXT,
title TEXT,
visit_count INTEGER,
typed_count INTEGER
);
CREATE TABLE visits (
id INTEGER PRIMARY KEY,
url INTEGER,
visit_time INTEGER,
from_visit INTEGER,
transition INTEGER
);
"""
)
connection.execute("INSERT INTO urls VALUES (1, ?, 'Example', 1, 0)", (url,))
connection.execute(
"INSERT INTO visits VALUES (7, 1, ?, 0, 805306368)",
(CHROME_TIME,),
)
def manifest(database_paths: list[tuple[str, str]]) -> dict:
return {
"schema_version": 1,
"status": "collected",
"databases": [
{
"browser": "Chrome",
"package": package,
"profile": "Default",
"device_path": f"/data/data/{package}/app_chrome/Default/History",
"archive_path": archive_path,
"sidecars": [],
}
for package, archive_path in database_paths
],
}
@pytest.mark.parametrize("use_zip", [False, True])
def test_androidqf_browser_history_directory_and_zip(tmp_path, use_zip):
source_database = tmp_path / "source" / "History"
create_history_database(source_database)
archive_path = "browser_history/com.android.chrome/Default/History"
manifest_data = manifest([("com.android.chrome", archive_path)])
module = AndroidQFBrowserHistory()
if use_zip:
acquisition_path = tmp_path / "acquisition.zip"
with zipfile.ZipFile(acquisition_path, "w") as archive:
archive.write(source_database, archive_path)
archive.writestr("browser_history/manifest.json", json.dumps(manifest_data))
with zipfile.ZipFile(acquisition_path) as archive:
module.from_zip(archive, archive.namelist())
module.run()
else:
acquisition = tmp_path / "acquisition"
database_path = acquisition / archive_path
database_path.parent.mkdir(parents=True)
database_path.write_bytes(source_database.read_bytes())
manifest_path = acquisition / "browser_history" / "manifest.json"
manifest_path.write_text(json.dumps(manifest_data))
files = [
path.relative_to(tmp_path).as_posix()
for path in acquisition.rglob("*")
if path.is_file()
]
module.from_dir(str(tmp_path), files)
module.run()
assert len(module.results) == 1
assert module.results[0]["url"] == URL
assert module.results[0]["browser"] == "Chrome"
assert module.results[0]["source_path"].endswith("/History")
module.collect_url_results()
module.to_timeline()
assert module.url_results[0]["url"] == URL
assert module.timeline[0]["event"] == "browser_history"
def test_androidqf_browser_history_isolates_malformed_database(tmp_path, caplog):
acquisition = tmp_path / "acquisition"
good_archive_path = "browser_history/com.android.chrome/Default/History"
bad_archive_path = "browser_history/com.brave.browser/Default/History"
create_history_database(acquisition / good_archive_path)
bad_path = acquisition / bad_archive_path
bad_path.parent.mkdir(parents=True)
bad_path.write_bytes(b"not sqlite")
manifest_path = acquisition / "browser_history" / "manifest.json"
manifest_path.write_text(
json.dumps(
manifest(
[
("com.brave.browser", bad_archive_path),
("com.android.chrome", good_archive_path),
]
)
)
)
module = AndroidQFBrowserHistory()
files = [
path.relative_to(tmp_path).as_posix()
for path in acquisition.rglob("*")
if path.is_file()
]
module.from_dir(str(tmp_path), files)
module.run()
assert [result["url"] for result in module.results] == [URL]
assert "Unable to parse browser history database" in caplog.text
def test_androidqf_browser_history_rejects_unsafe_manifest_path(tmp_path, caplog):
acquisition = tmp_path / "acquisition"
manifest_path = acquisition / "browser_history" / "manifest.json"
manifest_path.parent.mkdir(parents=True)
manifest_path.write_text(
json.dumps(manifest([("com.android.chrome", "browser_history/../secret")]))
)
module = AndroidQFBrowserHistory()
module.from_dir(str(tmp_path), [manifest_path.relative_to(tmp_path).as_posix()])
module.run()
assert module.results == []
assert "unsafe browser history archive path" in caplog.text
def test_filesystem_browser_history_reads_wal_only_visit(tmp_path):
database_path = tmp_path / "History"
connection = sqlite3.connect(database_path)
connection.execute("PRAGMA journal_mode=WAL")
connection.executescript(
"""
CREATE TABLE urls (
id INTEGER PRIMARY KEY,
url TEXT,
title TEXT,
visit_count INTEGER,
typed_count INTEGER
);
CREATE TABLE visits (
id INTEGER PRIMARY KEY,
url INTEGER,
visit_time INTEGER,
from_visit INTEGER,
transition INTEGER
);
"""
)
connection.commit()
connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
connection.execute("INSERT INTO urls VALUES (1, ?, 'WAL', 1, 0)", (URL,))
connection.execute(
"INSERT INTO visits VALUES (8, 1, ?, 0, 805306368)", (CHROME_TIME,)
)
connection.commit()
try:
assert Path(f"{database_path}-wal").stat().st_size > 0
module = FSBrowserHistory(target_path=str(database_path))
module.run()
finally:
connection.close()
assert [result["url"] for result in module.results] == [URL]
assert module.results[0]["browser"] == "Chromium"
def test_android_check_fs_finds_all_supported_browser_paths(tmp_path):
for index, relative_path in enumerate(
(
"data/data/com.android.chrome/app_chrome/Default/History",
"data/data/com.brave.browser/app_chrome/Default/History",
"data/data/com.microsoft.emmx/app_chrome/Default/History",
"data/data/com.sec.android.app.sbrowser/app_sbrowser/Default/History",
)
):
create_history_database(
tmp_path / relative_path, url=f"https://example.org/{index}"
)
command = CmdAndroidCheckFS(target_path=str(tmp_path), module_name="BrowserHistory")
command.run()
assert len(command.executed) == 1
assert {result["package"] for result in command.executed[0].results} == {
"com.android.chrome",
"com.brave.browser",
"com.microsoft.emmx",
"com.sec.android.app.sbrowser",
}
assert len(command.url_results) == 4
assert len(command.timeline) == 4