Compare commits

..
Author SHA1 Message Date
Janik Besendorf 3f83d9ba6f Update GitHub Actions 2026-08-12 15:37:18 +02:00
github-actions[bot]andDonnchaC 47fd771b2a Add new iOS versions and build numbers (#865)
Co-authored-by: DonnchaC <3081375+DonnchaC@users.noreply.github.com>
2026-08-11 10:07:52 +02:00
besendorf 8123db26d9 Fix iOS update workflow and refresh dependencies (#864) 2026-08-10 22:06:15 +02:00
besendorf d92a60c9be Preserve tombstone crash causes (#863) 2026-08-10 20:59:56 +02:00
besendorf 0b48d9fe1d Speed up compressed sysdiagnose analysis (#861)
* Speed up compressed sysdiagnose analysis

* ci: retrigger Ruff check
2026-08-07 09:07:33 +02:00
besendorf 067f053627 Add extensible CLI commands (#853)
* Add extensible CLI commands

* Handle plugin SystemExit failures
2026-08-05 23:30:28 +02:00
14 changed files with 1086 additions and 787 deletions

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
name: Add issue to project name: Add issue to project
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/add-to-project@v0.5.0 - uses: actions/add-to-project@v2
with: with:
# You can target a project in a different organization # You can target a project in a different organization
# to the issue # to the issue
+3 -3
View File
@@ -11,13 +11,13 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v6 uses: actions/checkout@v7
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v6 uses: actions/setup-python@v7
with: with:
python-version: "3.10" python-version: "3.10"
- name: Install uv - name: Install uv
uses: astral-sh/setup-uv@v7 uses: astral-sh/setup-uv@v10.0.0
with: with:
enable-cache: true enable-cache: true
- name: Install Dependencies - name: Install Dependencies
+5 -6
View File
@@ -36,10 +36,10 @@ jobs:
tag-suffix: "-android" tag-suffix: "-android"
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v7
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry - name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with: with:
registry: ${{ env.REGISTRY }} registry: ${{ env.REGISTRY }}
username: ${{ github.actor }} username: ${{ github.actor }}
@@ -47,7 +47,7 @@ jobs:
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker - name: Extract metadata (tags, labels) for Docker
id: meta id: meta
uses: docker/metadata-action@v5 uses: docker/metadata-action@v6
with: with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
flavor: | flavor: |
@@ -61,13 +61,13 @@ jobs:
type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }} type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }}
# This step sets up some additional capabilities to generate the provenance and sbom attestations # This step sets up some additional capabilities to generate the provenance and sbom attestations
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v4
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Build and push Docker image - name: Build and push Docker image
id: push id: push
uses: docker/build-push-action@v6 uses: docker/build-push-action@v7
with: with:
file: ${{ matrix.platform.dockerfile }} file: ${{ matrix.platform.dockerfile }}
context: . context: .
@@ -76,4 +76,3 @@ jobs:
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
provenance: mode=max provenance: mode=max
sbom: true sbom: true
+3 -3
View File
@@ -12,13 +12,13 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v6 uses: actions/checkout@v7
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v6 uses: actions/setup-python@v7
with: with:
python-version: "3.10" python-version: "3.10"
- name: Install uv - name: Install uv
uses: astral-sh/setup-uv@v7 uses: astral-sh/setup-uv@v10.0.0
with: with:
enable-cache: true enable-cache: true
- name: Install Dependencies - name: Install Dependencies
+3 -3
View File
@@ -15,13 +15,13 @@ jobs:
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
steps: steps:
- uses: actions/checkout@v6 - uses: actions/checkout@v7
- name: Install uv - name: Install uv
uses: astral-sh/setup-uv@v7 uses: astral-sh/setup-uv@v10.0.0
with: with:
enable-cache: true enable-cache: true
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6 uses: actions/setup-python@v7
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
- name: Install Python dependencies - name: Install Python dependencies
+5 -3
View File
@@ -11,10 +11,12 @@ jobs:
update-ios-version: update-ios-version:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v7
- uses: actions/setup-python@v4 - uses: actions/setup-python@v7
- name: Install script dependencies
run: python -m pip install "packaging==26.3"
- name: Run script to fetch latest iOS releases from Apple RSS feed. - name: Run script to fetch latest iOS releases from Apple RSS feed.
run: python3 .github/workflows/scripts/update-ios-releases.py run: python .github/workflows/scripts/update-ios-releases.py
- name: Create Pull Request - name: Create Pull Request
uses: peter-evans/create-pull-request@v8 uses: peter-evans/create-pull-request@v8
with: with:
+13 -13
View File
@@ -17,25 +17,25 @@ classifiers = [
"Programming Language :: Python", "Programming Language :: Python",
] ]
dependencies = [ dependencies = [
"click==8.3.3", "click==8.4.2",
"rich==14.3.3", "rich==15.0.0",
"tld==0.13.2", "tld==0.13.2",
"requests==2.33.1", "requests==2.34.2",
"simplejson==3.20.2", "simplejson==4.1.1",
"packaging==26.2", "packaging==26.3",
"appdirs==1.4.4", "appdirs==1.4.4",
"iOSbackup==0.9.925", "iOSbackup==0.9.925",
"adb-shell[usb]==0.4.4", "adb-shell[usb]==0.4.4",
"libusb1==3.3.1", "libusb1==3.4.0",
"cryptography==47.0.0", "cryptography==50.0.0",
"PyYAML>=6.0.2", "PyYAML>=6.0.2",
"pyahocorasick==2.3.1", "pyahocorasick==2.3.1",
"betterproto2==0.9.1", "betterproto2==0.10.0",
"pydantic==2.13.3", "pydantic==2.13.4",
"pydantic-settings==2.14.0", "pydantic-settings==2.15.0",
"NSKeyedUnArchiver==1.5.2", "NSKeyedUnArchiver==1.5.2",
"python-dateutil==2.9.0.post0", "python-dateutil==2.9.0.post0",
"tzdata==2026.2", "tzdata==2026.3",
] ]
requires-python = ">= 3.10" requires-python = ">= 3.10"
@@ -65,9 +65,9 @@ dev = [
docs = [ docs = [
"mkdocs==1.6.1", "mkdocs==1.6.1",
"mkdocs-autorefs==1.4.4", "mkdocs-autorefs==1.4.4",
"mkdocs-material==9.7.6", "mkdocs-material==9.7.7",
"mkdocs-material-extensions==1.3.1", "mkdocs-material-extensions==1.3.1",
"mkdocstrings==1.0.4", "mkdocstrings==1.0.6",
] ]
[build-system] [build-system]
@@ -34,6 +34,7 @@ TOMBSTONE_TEXT_KEY_MAPPINGS = {
"signal": "signal_info", "signal": "signal_info",
"code": "code", "code": "code",
"Cause": "cause", "Cause": "cause",
"Abort message": "abort_message",
} }
@@ -67,6 +68,8 @@ class TombstoneCrashResult(pydantic.BaseModel):
uid: int uid: int
signal_info: SignalInfo signal_info: SignalInfo
cause: Optional[str] = None cause: Optional[str] = None
causes: Optional[List[dict]] = None
abort_message: Optional[str] = None
extra: Optional[str] = None extra: Optional[str] = None
+66 -12
View File
@@ -6,8 +6,10 @@
import json import json
import logging import logging
import os import os
import shutil
import tarfile import tarfile
from pathlib import Path from pathlib import Path, PurePosixPath
from tempfile import TemporaryDirectory
from typing import Any, Optional from typing import Any, Optional
from mvt.common.command import Command from mvt.common.command import Command
@@ -54,6 +56,8 @@ class CmdIOSCheckSysdiagnose(Command):
self.sysdiagnose_archive: Optional[tarfile.TarFile] = None self.sysdiagnose_archive: Optional[tarfile.TarFile] = None
self.sysdiagnose_files: list[str] = [] self.sysdiagnose_files: list[str] = []
self.ips_files: list[dict[str, Any]] = [] self.ips_files: list[dict[str, Any]] = []
self.temp_sysdiagnose_dir: Optional[TemporaryDirectory[str]] = None
self.extracted_sysdiagnose_path: Optional[str] = None
@staticmethod @staticmethod
def _parse_bugtype_header(data: bytes) -> Optional[int]: def _parse_bugtype_header(data: bytes) -> Optional[int]:
@@ -91,21 +95,68 @@ class CmdIOSCheckSysdiagnose(Command):
self.log.info("Parsing sysdiagnose archive. This might take a while...") self.log.info("Parsing sysdiagnose archive. This might take a while...")
self.sysdiagnose_format = "tar" self.sysdiagnose_format = "tar"
self.sysdiagnose_archive = tarfile.open(self.target_path, "r:gz") self.sysdiagnose_archive = tarfile.open(self.target_path, "r:gz")
for member in self.sysdiagnose_archive: self._extract_sysdiagnose_archive()
self.sysdiagnose_files.append(member.name)
if member.isfile() and member.name.endswith(".ips"): def _extract_sysdiagnose_archive(self) -> None:
archive_handle = self.sysdiagnose_archive.extractfile(member) archive = self.sysdiagnose_archive
if archive_handle is not None: if archive is None:
with archive_handle: raise RuntimeError("Sysdiagnose archive has not been initialized")
self._add_ips_file(member.name, archive_handle.read())
self.temp_sysdiagnose_dir = TemporaryDirectory()
extraction_root = Path(self.temp_sysdiagnose_dir.name).resolve()
archive_roots = set()
for member in archive:
member_path = PurePosixPath(member.name.replace("\\", "/"))
if member_path.is_absolute() or ".." in member_path.parts:
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
continue
destination = extraction_root.joinpath(*member_path.parts).resolve()
if not destination.is_relative_to(extraction_root):
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
continue
if not member_path.parts:
continue
archive_roots.add(member_path.parts[0])
if member.isdir():
destination.mkdir(parents=True, exist_ok=True)
continue
# Modules only need directories and regular files. Do not materialize
# links or device nodes from an untrusted sysdiagnose archive.
if not member.isfile():
self.log.warning("Skipping unsafe sysdiagnose member %r", member.name)
continue
normalized_name = member_path.as_posix()
self.sysdiagnose_files.append(normalized_name)
source = archive.extractfile(member)
if source is None:
continue
destination.parent.mkdir(parents=True, exist_ok=True)
with source, destination.open("wb") as output:
shutil.copyfileobj(source, output)
if normalized_name.endswith(".ips"):
self._add_ips_file(str(destination), destination.read_bytes())
if len(archive_roots) != 1:
raise ValueError("Sysdiagnose archive must contain one top-level directory")
self.extracted_sysdiagnose_path = str(extraction_root / archive_roots.pop())
def module_init(self, module) -> None: def module_init(self, module) -> None:
module.ips_files = self.ips_files module.ips_files = self.ips_files
if self.sysdiagnose_format == "tar": if self.sysdiagnose_format == "tar":
if self.sysdiagnose_archive is None: if self.extracted_sysdiagnose_path is None:
raise RuntimeError("Sysdiagnose archive has not been initialized") raise RuntimeError("Sysdiagnose archive has not been extracted")
module.from_sysdiagnose_tar( module.from_sysdiagnose_folder(
self.sysdiagnose_archive, self.sysdiagnose_files self.extracted_sysdiagnose_path, self.sysdiagnose_files
) )
return return
if self.sysdiagnose_format == "dir" and self.target_path: if self.sysdiagnose_format == "dir" and self.target_path:
@@ -117,3 +168,6 @@ class CmdIOSCheckSysdiagnose(Command):
if self.sysdiagnose_archive is not None: if self.sysdiagnose_archive is not None:
self.sysdiagnose_archive.close() self.sysdiagnose_archive.close()
self.sysdiagnose_archive = None self.sysdiagnose_archive = None
if self.temp_sysdiagnose_dir is not None:
self.temp_sysdiagnose_dir.cleanup()
self.temp_sysdiagnose_dir = None
+8
View File
@@ -1216,6 +1216,10 @@
"version": "18.7.9", "version": "18.7.9",
"build": "22H355" "build": "22H355"
}, },
{
"version": "18.7.10",
"build": "22H373"
},
{ {
"version": "26", "version": "26",
"build": "23A341" "build": "23A341"
@@ -1255,5 +1259,9 @@
{ {
"version": "26.5", "version": "26.5",
"build": "23F77" "build": "23F77"
},
{
"version": "26.6.1",
"build": "23G82"
} }
] ]
+3 -1
View File
@@ -73,7 +73,9 @@ class Calls(IOSExtraction):
"duration": row[1], "duration": row[1],
"location": row[2], "location": row[2],
"number": ( "number": (
row[3].decode("utf-8") if row[3] and row[3] is bytes else row[3] row[3].decode("utf-8")
if isinstance(row[3], bytes)
else row[3]
), ),
"provider": row[4], "provider": row[4],
} }
+42
View File
@@ -8,6 +8,7 @@ import datetime
import pytest import pytest
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
from mvt.android.parsers.proto.tombstone import Tombstone
from ..utils import get_artifact from ..utils import get_artifact
@@ -42,6 +43,47 @@ class TestTombstoneCrashArtifact:
assert len(tombstone_artifact.results) == 1 assert len(tombstone_artifact.results) == 1
self.validate_tombstone_result(tombstone_artifact.results[0]) self.validate_tombstone_result(tombstone_artifact.results[0])
def test_text_tombstone_preserves_abort_message(self):
tombstone_artifact = TombstoneCrashArtifact()
artifact_path = "android_data/bugreport/FS/data/tombstones/tombstone_00"
file = get_artifact(artifact_path)
with open(file, "rb") as f:
data = f.read()
tombstone_artifact.parse(
os.path.basename(artifact_path),
datetime.datetime(2021, 9, 29, 17, 43, 49),
data,
)
assert tombstone_artifact.results[0]["abort_message"] == (
"Check failed: payload.size() <= bytes_left "
"(payload.size()=99, bytes_left=51) "
)
def test_protobuf_tombstone_preserves_abort_message_and_causes(self):
tombstone_artifact = TombstoneCrashArtifact()
artifact_path = "android_data/tombstone_process.pb"
file = get_artifact(artifact_path)
with open(file, "rb") as f:
tombstone = Tombstone().parse(f.read())
tombstone.abort_message = "synthetic abort reason"
tombstone_artifact.parse_protobuf(
os.path.basename(artifact_path),
datetime.datetime(2023, 4, 12, 12, 32, 40, 518290),
bytes(tombstone),
)
result = tombstone_artifact.results[0]
assert result["abort_message"] == "synthetic abort reason"
assert result["causes"] == [
{
"human_readable": "null pointer dereference",
"memory_error": None,
}
]
def test_text_tombstone_keeps_crashing_thread(self): def test_text_tombstone_keeps_crashing_thread(self):
tombstone_artifact = TombstoneCrashArtifact() tombstone_artifact = TombstoneCrashArtifact()
artifact_path = "android_data/tombstone_process.txt" artifact_path = "android_data/tombstone_process.txt"
+43 -1
View File
@@ -1,5 +1,7 @@
import io
import tarfile import tarfile
from datetime import timedelta from datetime import timedelta
from pathlib import Path
from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
@@ -70,6 +72,46 @@ def test_check_sysdiagnose_from_archive_closes_archive(tmp_path):
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds} {"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
] ]
assert command.executed[0].ips_files == [ assert command.executed[0].ips_files == [
{"file_path": "sysdiagnose/report.ips", "bug_type": 210} {
"file_path": str(
Path(command.extracted_sysdiagnose_path) / "report.ips"
),
"bug_type": 210,
}
] ]
assert command.sysdiagnose_archive is None assert command.sysdiagnose_archive is None
def test_archive_is_extracted_once_and_unsafe_members_are_skipped(tmp_path):
archive_path = tmp_path / "sysdiagnose.tar.gz"
escaped_path = tmp_path / "escaped.txt"
content = b"test content"
member = tarfile.TarInfo("sysdiagnose/artifact.txt")
member.size = len(content)
with tarfile.open(archive_path, "w:gz") as archive:
archive.addfile(member, io.BytesIO(content))
escaped = tarfile.TarInfo(f"sysdiagnose/../../{escaped_path.name}")
escaped.size = len(content)
archive.addfile(escaped, io.BytesIO(content))
link = tarfile.TarInfo("sysdiagnose/link")
link.type = tarfile.SYMTYPE
link.linkname = "/etc/hostname"
archive.addfile(link)
command = CmdIOSCheckSysdiagnose(target_path=str(archive_path))
try:
command.init()
extracted_path = Path(command.extracted_sysdiagnose_path)
assert (extracted_path / "artifact.txt").read_bytes() == content
assert not escaped_path.exists()
assert not (extracted_path / "link").exists()
module = SysdiagnoseExtraction()
command.module_init(module)
assert module.tar is None
assert module.parent_path == str(extracted_path.parent)
finally:
command.finish()
assert not extracted_path.exists()
Generated
+888 -741
View File
File diff suppressed because it is too large. Load diff