Merge pull request #544 from mvt-project/feature/use-pyproject-toml

Configure project to use pyproject.toml and consistent CI and test tooling

.github/workflows/black.yml

@@ -1,16 +0,0 @@
-name: Black
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  black:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: psf/black@stable
-        with:
-          options: "--check"

.github/workflows/mypy.yml

@@ -0,0 +1,23 @@
+name: Mypy
+on: workflow_dispatch
+
+jobs:
+  mypy_py3:
+    name: Mypy check
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: 'pip'
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Install Dependencies
+        run: |
+          pip install mypy
+      - name: mypy
+        run: |
+          make mypy

.github/workflows/python-package.yml

@@ -1,52 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: CI
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ['3.8', '3.9', '3.10']  # , '3.11']
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade setuptools
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest safety stix2 pytest-mock pytest-cov
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python -m pip install .
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Safety checks
-      run: safety check
-    - name: Test with pytest and coverage
-      run: |
-        set -o pipefail
-        pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=mvt tests/ | tee pytest-coverage.txt
-    - name: Pytest coverage comment
-      continue-on-error: true # Workflows running on a fork can't post comments
-      uses: MishaKav/pytest-coverage-comment@main
-      if: github.event_name == 'pull_request'
-      with:
-        pytest-coverage-path: ./pytest-coverage.txt
-        junitxml-path: ./pytest.xml

.github/workflows/ruff.yml

@@ -4,16 +4,24 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+
 jobs:
   ruff_py3:
     name: Ruff syntax check
     runs-on: ubuntu-latest
+
     steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: 'pip'
       - name: Checkout
         uses: actions/checkout@master
       - name: Install Dependencies
         run: |
-          pip install --user ruff
+          pip install ruff
       - name: ruff
         run: |
-          ruff check --output-format github .
+          make ruff

.github/workflows/tests.yml

@@ -0,0 +1,38 @@
+name: Tests
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    name: Run Python Tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']  # , '3.11']
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python dependencies
+        run: |
+          make install
+          make test-requirements
+      - name: Test with pytest
+        run: |
+          set -o pipefail
+          make test-ci | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        continue-on-error: true # Workflows running on a fork can't post comments
+        uses: MishaKav/pytest-coverage-comment@main
+        if: github.event_name == 'pull_request'
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml

Makefile

@@ -1,23 +1,39 @@
 PWD = $(shell pwd)
 
-check:
-	flake8
-	ruff check -q .
-	black --check .
-	pytest -q
+autofix:
+	ruff format .
+	ruff check --fix .
 
+check: ruff mypy
+
+ruff:
+	ruff format --check .
+	ruff check -q .
+
+mypy:
+	mypy
+
+test:
+	python3 -m pytest
+
+test-ci:
+	python3 -m pytest -v
+
+install:
+	python3 -m pip install --upgrade -e .
+
+test-requirements:
+	python3 -m pip install --upgrade -r test-requirements.txt
 
 clean:
-	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/mvt.egg-info
+	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/src/mvt.egg-info
 
 dist:
-	python3 setup.py sdist bdist_wheel
+	python3 -m pip install --upgrade build
+	python3 -m build
 
 upload:
 	python3 -m twine upload dist/*
 
 test-upload:
 	python3 -m twine upload --repository testpypi dist/*
-
-pylint:
-	pylint --rcfile=setup.cfg mvt

dev/mvt-android

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from mvt import android
-
-android.cli()

dev/mvt-ios

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from mvt import ios
-
-ios.cli()

pyproject.toml

@@ -0,0 +1,102 @@
+[project]
+name = "mvt"
+dynamic = ["version"]
+authors = [
+    {name = "Claudio Guarnieri", email = "nex@nex.sx"}
+]
+maintainers = [
+    {name = "Etienne Maynier", email = "tek@randhome.io"},
+    {name = "Donncha Ó Cearbhaill", email = "donncha.ocearbhaill@amnesty.org"},
+    {name = "Rory Flynn", email = "rory.flynn@amnesty.org"}
+]
+description = "Mobile Verification Toolkit"
+readme = "README.md"
+keywords = ["security", "mobile", "forensics", "malware"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Information Technology",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python"
+]
+dependencies = [
+    "click >=8.1.3",
+    "rich >=12.6.0",
+    "tld >=0.12.6",
+    "requests >=2.28.1",
+    "simplejson >=3.17.6",
+    "packaging >=21.3",
+    "appdirs >=1.4.4",
+    "iOSbackup >=0.9.923",
+    "adb-shell[usb] >=0.4.3",
+    "libusb1 >=3.0.0",
+    "cryptography >=42.0.5",
+    "pyyaml >=6.0",
+    "pyahocorasick >= 2.0.0",
+]
+requires-python = ">= 3.8"
+
+[project.urls]
+homepage = "https://docs.mvt.re/en/latest/"
+repository = "https://github.com/mvt-project/mvt"
+
+[project.scripts]
+    mvt-ios = "mvt.ios:cli"
+    mvt-android = "mvt.android:cli"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.coverage.run]
+omit = [
+    "tests/*",
+]
+
+[tool.coverage.html]
+directory= "htmlcov"
+
+[tool.mypy]
+install_types = true
+non_interactive = true
+ignore_missing_imports = true
+packages = "src"
+
+[tool.pytest.ini_options]
+addopts = "-ra -q --cov=mvt --cov-report html --junitxml=pytest.xml --cov-report=term-missing:skip-covered"
+testpaths = [
+    "tests"
+]
+
+[tool.ruff.lint]
+select = ["C90", "E", "F", "W"]  # flake8 default set
+ignore = [
+    "E501",  # don't enforce line length violations
+    "C901",  # complex-structure
+
+    # These were previously ignored but don't seem to be required:
+    # "E265",  # no-space-after-block-comment
+    # "F401",  # unused-import
+    # "E127",  # not included in ruff
+    # "W503",  # not included in ruff
+    # "E226",  # missing-whitespace-around-arithmetic-operator
+    # "E203",  # whitespace-before-punctuation
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]  # unused-import
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.setuptools]
+include-package-data = true
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+mvt = ["ios/data/*.json"]
+
+[tool.setuptools.dynamic]
+version = {attr = "mvt.common.version.MVT_VERSION"}

ruff.toml

@@ -1,6 +0,0 @@
-# Never enforce `E501` (line length violations).
-ignore = ["E501"]
-
-# Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`.
-[per-file-ignores]
-"__init__.py" = ["F401"]

scripts/lint.sh

@@ -1,14 +0,0 @@
-#!/bin/sh -e
-
-export SOURCE="mvt tests"
-
-export PREFIX=""
-if [ -d 'venv' ] ; then
-    export PREFIX="venv/bin/"
-fi
-
-set -x
-
-${PREFIX}autoflake --in-place --recursive --exclude venv ${SOURCE}
-${PREFIX}isort ${SOURCE}
-${PREFIX}black --exclude venv ${SOURCE}

setup.cfg

@@ -1,96 +0,0 @@
-[metadata]
-name = mvt
-version = attr: mvt.common.version.MVT_VERSION
-author = Claudio Guarnieri
-author_email = nex@nex.sx
-description = Mobile Verification Toolkit
-long_description = file: README.md
-long_description_content_type = text/markdown
-url = https://github.com/mvt-project/mvt
-keywords = security, mobile, forensics, malware
-license = MVT v1.1
-classifiers =
-    Development Status :: 5 - Production/Stable
-    Intended Audience :: Information Technology
-    Operating System :: OS Independent
-    Programming Language :: Python
-
-[options]
-packages = find:
-package_dir = = ./
-include_package_data = True
-python_requires = >= 3.8
-install_requires =
-    click >=8.1.3
-    rich >=12.6.0
-    tld >=0.12.6
-    requests >=2.28.1
-    simplejson >=3.17.6
-    packaging >=21.3
-    appdirs >=1.4.4
-    iOSbackup >=0.9.923
-    adb-shell[usb] >=0.4.3
-    libusb1 >=3.0.0
-    cryptography >=42.0.5
-    pyyaml >=6.0
-    pyahocorasick >= 2.0.0
-
-[options.packages.find]
-where = ./
-
-[options.entry_points]
-console_scripts =
-    mvt-ios = mvt.ios:cli
-    mvt-android = mvt.android:cli
-
-[options.package_data]
-mvt = ios/data/*.json
-
-[flake8]
-max-complexity = 10
-max-line-length = 1000
-ignore =
-    C901,
-    E265,
-    F401,
-    E127,
-    W503,
-    E226,
-    E203
-
-[pylint]
-score = no
-reports = no
-output-format = colorized
-
-max-locals = 25
-max-args = 10
-
-good-names = i,m
-
-min-similarity-lines = 10
-ignore-comments = yes
-ignore-docstrings = yes
-ignore-imports = yes
-
-ignored-argument-names=args|kwargs
-
-# https://pylint.pycqa.org/en/stable/technical_reference/features.html
-disable =
-    too-many-instance-attributes,
-    broad-except,
-    abstract-method,
-    dangerous-default-value,
-    too-few-public-methods,
-    missing-docstring,
-    missing-module-docstring,
-    missing-class-docstring,
-    missing-function-docstring,
-    #duplicate-code,
-    #line-too-long,
-
-[mypy]
-ignore_missing_imports = True
-
-[isort]
-profile=black

setup.py

@@ -1,8 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2023 The MVT Authors.
-# Use of this software is governed by the MVT License 1.1 that can be found at
-#   https://license.mvt.re/1.1/
-
-from setuptools import setup
-
-setup()