From c1a12752910167543bd7a6135e05a8bace99901e Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:26:46 -0400 Subject: [PATCH] ci: add conditional environment test gates --- .github/actionlint.yaml | 19 + .github/workflows/ci.yml | 17 +- .github/workflows/conditional-tests.yml | 368 ++++++++++++++++++ CONTRIBUTING.md | 10 + README.md | 14 +- ci/conditional-test-policy.json | 121 ++++++ docs/conditional-testing.md | 102 +++++ obliteratus/remote.py | 11 +- pyproject.toml | 5 + scripts/check_conditional_policy.py | 88 +++++ scripts/conditional_gate_summary.py | 64 +++ scripts/run_conditional_gate.py | 131 +++++++ tests/conditional/test_cuda_runtime.py | 36 ++ .../test_external_evaluation_runtime.py | 32 ++ tests/conditional/test_mlx_runtime.py | 20 + .../test_model_download_runtime.py | 49 +++ tests/conditional/test_mps_runtime.py | 24 ++ tests/conditional/test_network_services.py | 53 +++ tests/conditional/test_operator_ui.py | 33 ++ tests/conditional/test_remote_runtime.py | 36 ++ tests/test_conditional_gate_scripts.py | 94 +++++ tests/test_remote_boundaries.py | 36 ++ uv.lock | 97 ++++- 23 files changed, 1449 insertions(+), 11 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/conditional-tests.yml create mode 100644 ci/conditional-test-policy.json create mode 100644 docs/conditional-testing.md create mode 100644 scripts/check_conditional_policy.py create mode 100644 scripts/conditional_gate_summary.py create mode 100644 scripts/run_conditional_gate.py create mode 100644 tests/conditional/test_cuda_runtime.py create mode 100644 tests/conditional/test_external_evaluation_runtime.py create mode 100644 tests/conditional/test_mlx_runtime.py create mode 100644 tests/conditional/test_model_download_runtime.py create mode 100644 tests/conditional/test_mps_runtime.py create mode 100644 tests/conditional/test_network_services.py create mode 100644 tests/conditional/test_operator_ui.py create mode 100644 tests/conditional/test_remote_runtime.py create mode 100644 tests/test_conditional_gate_scripts.py create mode 100644 tests/test_remote_boundaries.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..a85f6a0 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,19 @@ +self-hosted-runner: + # Project-owned capability labels used by conditional test runners. + labels: [cuda, mps, mlx] + +# Configuration variables in array of strings defined in your repository or +# organization. `null` means disabling configuration variables check. +# Empty array means no configuration variable is allowed. +config-variables: null + +# Configuration for file paths. The keys are glob patterns to match to file +# paths relative to the repository root. The values are the configurations for +# the file paths. Note that the path separator is always '/'. +# The following configurations are available. +# +# "ignore" is an array of regular expression patterns. Matched error messages +# are ignored. This is similar to the "-ignore" command line option. +paths: +# .github/workflows/**/*.yml: +# ignore: [] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64f6c41..5c0cd0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,6 +202,9 @@ jobs: scripts/check_coverage_thresholds.py scripts/check_mutation_score.py scripts/check_quality_policy.py + scripts/check_conditional_policy.py + scripts/conditional_gate_summary.py + scripts/run_conditional_gate.py scripts/run_repeat_gate.py scripts/check_supply_chain_policy.py scripts/gemma4_12b_recursive_loop.py @@ -213,6 +216,9 @@ jobs: scripts/check_coverage_thresholds.py scripts/check_mutation_score.py scripts/check_quality_policy.py + scripts/check_conditional_policy.py + scripts/conditional_gate_summary.py + scripts/run_conditional_gate.py scripts/run_repeat_gate.py scripts/check_supply_chain_policy.py scripts/gemma4_12b_recursive_loop.py || true @@ -275,7 +281,7 @@ jobs: run: | mkdir -p test-results "$TEST_ENV/bin/python" -m pytest \ - -m "not slow and not gpu and not mps and not mlx and not network and not download and not remote" \ + -m "not slow and not gpu and not mps and not mlx and not network and not download and not remote and not operator_ui" \ --cov-branch \ --cov-fail-under=0 \ --junitxml="test-results/junit-py${{ matrix.python-version }}.xml" \ @@ -304,10 +310,11 @@ jobs: --base-ref "$COVERAGE_BASE" - name: Enforce mature CPU-scope coverage and immutable quality policy - run: >- - "$TEST_ENV/bin/python" scripts/check_quality_policy.py - --policy ci/test-quality-policy.json - --coverage "test-results/coverage-py${{ matrix.python-version }}.json" + run: | + "$TEST_ENV/bin/python" scripts/check_quality_policy.py \ + --policy ci/test-quality-policy.json \ + --coverage "test-results/coverage-py${{ matrix.python-version }}.json" + "$TEST_ENV/bin/python" scripts/check_conditional_policy.py - name: Upload test and coverage evidence if: always() diff --git a/.github/workflows/conditional-tests.yml b/.github/workflows/conditional-tests.yml new file mode 100644 index 0000000..8c1de43 --- /dev/null +++ b/.github/workflows/conditional-tests.yml @@ -0,0 +1,368 @@ +name: Conditional tests + +on: + workflow_dispatch: + inputs: + run_model: + description: Run pinned tiny-model download and evaluation gates + type: boolean + default: true + run_network: + description: Run disposable network-service boundary gate + type: boolean + default: true + run_ui: + description: Construct the optional operator UI without a listener + type: boolean + default: true + run_cuda: + description: Run CUDA and bitsandbytes on the labeled self-hosted runner + type: boolean + default: false + run_mps: + description: Run MPS on the labeled Apple Silicon runner + type: boolean + default: false + run_mlx: + description: Run MLX on the labeled Apple Silicon runner + type: boolean + default: false + run_remote: + description: Run the least-privileged SSH provider gate + type: boolean + default: false + schedule: + - cron: "17 6 * * 0" + release: + types: [published] + +permissions: + contents: read + +concurrency: + group: conditional-tests-${{ github.ref }} + cancel-in-progress: false + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_INPUT: "1" + UV_VERSION: "0.12.4" + +jobs: + policy: + name: Conditional policy + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate gate mappings and evidence policy + run: python3 scripts/check_conditional_policy.py + + model_runtime: + name: Pinned model runtime and evaluation + needs: policy + if: github.event_name != 'workflow_dispatch' || inputs.run_model + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-model + HF_HOME: /tmp/obliteratus-hf-cache + HF_HUB_DISABLE_TELEMETRY: "1" + TOKENIZERS_PARALLELISM: "false" + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Restore pinned model cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.0.3 + with: + path: /tmp/obliteratus-hf-cache + key: hf-tiny-random-gpt2-71034c5-py3.12-${{ runner.os }} + - name: Install locked runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --no-editable + - name: Run model download and cache replay + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py + model-download-runtime + - name: Run external evaluation adapter + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py + external-evaluation + - name: Upload model-runtime evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-model-runtime-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + network_services: + name: Network service boundary + needs: policy + if: github.event_name != 'workflow_dispatch' || inputs.run_network + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-network + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --no-editable + - name: Run disposable service probe + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py + network-services + - name: Upload network evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-network-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + operator_ui: + name: Operator UI construction + needs: policy + if: github.event_name != 'workflow_dispatch' || inputs.run_ui + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-ui + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked UI runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --all-extras --no-default-groups --no-editable + - name: Construct UI without opening a listener + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py + operator-ui + - name: Upload UI evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-operator-ui-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + cuda: + name: CUDA and bitsandbytes runtime + needs: policy + if: >- + (github.event_name == 'workflow_dispatch' && inputs.run_cuda) || + (github.event_name != 'workflow_dispatch' && vars.ENABLE_CUDA_GATE == 'true') + runs-on: [self-hosted, linux, x64, cuda] + timeout-minutes: 20 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-cuda + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked CUDA runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --no-editable + - name: Run CUDA placement and operation probe + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py cuda-runtime + - name: Run bitsandbytes quantization probe + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py bitsandbytes-runtime + - name: Upload CUDA evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-cuda-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + mps: + name: Apple MPS runtime + needs: policy + if: >- + (github.event_name == 'workflow_dispatch' && inputs.run_mps) || + (github.event_name != 'workflow_dispatch' && vars.ENABLE_MPS_GATE == 'true') + runs-on: [self-hosted, macOS, ARM64, mps] + timeout-minutes: 15 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-mps + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked MPS runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --no-editable + - name: Run MPS selection and operation probe + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py mps-runtime + - name: Upload MPS evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-mps-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + mlx: + name: Apple MLX runtime + needs: policy + if: >- + (github.event_name == 'workflow_dispatch' && inputs.run_mlx) || + (github.event_name != 'workflow_dispatch' && vars.ENABLE_MLX_GATE == 'true') + runs-on: [self-hosted, macOS, ARM64, mlx] + timeout-minutes: 20 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-mlx + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked MLX runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --group mlx --no-editable + - name: Run MLX placement and operation probe + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py mlx-runtime + - name: Upload MLX evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-mlx-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: error + retention-days: 30 + + remote: + name: Least-privileged remote provider + needs: policy + if: >- + (github.event_name == 'workflow_dispatch' && inputs.run_remote) || + (github.event_name != 'workflow_dispatch' && vars.ENABLE_REMOTE_GATE == 'true') + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CONDITIONAL_ENV: /tmp/obliteratus-conditional-remote + OBLITERATUS_REMOTE_HOST: ${{ vars.OBLITERATUS_REMOTE_HOST }} + OBLITERATUS_REMOTE_USER: ${{ vars.OBLITERATUS_REMOTE_USER }} + OBLITERATUS_REMOTE_PORT: ${{ vars.OBLITERATUS_REMOTE_PORT }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install locked runtime + run: | + python -m pip install "uv==${UV_VERSION}" + UV_PROJECT_ENVIRONMENT="$CONDITIONAL_ENV" \ + uv sync --locked --no-default-groups --extra dev --no-editable + - name: Materialize protected SSH prerequisites + env: + REMOTE_KEY_CONTENT: ${{ secrets.OBLITERATUS_REMOTE_KEY }} + REMOTE_KNOWN_HOSTS_CONTENT: ${{ secrets.OBLITERATUS_REMOTE_KNOWN_HOSTS }} + run: | + if [ -z "$OBLITERATUS_REMOTE_HOST" ] || [ -z "$OBLITERATUS_REMOTE_USER" ]; then + echo "remote gate selected: configure non-root OBLITERATUS_REMOTE_HOST and OBLITERATUS_REMOTE_USER variables" + exit 1 + fi + if [ -z "$REMOTE_KEY_CONTENT" ] || [ -z "$REMOTE_KNOWN_HOSTS_CONTENT" ]; then + echo "remote gate selected: configure OBLITERATUS_REMOTE_KEY and pinned OBLITERATUS_REMOTE_KNOWN_HOSTS secrets" + exit 1 + fi + install -m 700 -d "$RUNNER_TEMP/obliteratus-ssh" + install -m 600 /dev/null "$RUNNER_TEMP/obliteratus-ssh/key" + install -m 600 /dev/null "$RUNNER_TEMP/obliteratus-ssh/known_hosts" + printf '%s\n' "$REMOTE_KEY_CONTENT" > "$RUNNER_TEMP/obliteratus-ssh/key" + printf '%s\n' "$REMOTE_KNOWN_HOSTS_CONTENT" > "$RUNNER_TEMP/obliteratus-ssh/known_hosts" + - name: Run remote provider probe + env: + OBLITERATUS_REMOTE_KEY: ${{ runner.temp }}/obliteratus-ssh/key + OBLITERATUS_REMOTE_KNOWN_HOSTS: ${{ runner.temp }}/obliteratus-ssh/known_hosts + run: >- + "$CONDITIONAL_ENV/bin/python" scripts/run_conditional_gate.py remote-execution + - name: Upload remote evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-remote-${{ github.run_attempt }} + path: conditional-evidence/ + if-no-files-found: warn + retention-days: 30 + + summary: + name: Conditional result and freshness summary + if: always() + needs: [policy, model_runtime, network_services, operator_ui, cuda, mps, mlx, remote] + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + CONDITIONAL_RESULTS: >- + {"policy":"${{ needs.policy.result }}","model_runtime":"${{ needs.model_runtime.result }}", + "network_services":"${{ needs.network_services.result }}","operator_ui":"${{ needs.operator_ui.result }}", + "cuda":"${{ needs.cuda.result }}","mps":"${{ needs.mps.result }}","mlx":"${{ needs.mlx.result }}", + "remote":"${{ needs.remote.result }}"} + CONDITIONAL_SELECTED: >- + {"model_runtime":${{ github.event_name != 'workflow_dispatch' || inputs.run_model }}, + "network_services":${{ github.event_name != 'workflow_dispatch' || inputs.run_network }}, + "operator_ui":${{ github.event_name != 'workflow_dispatch' || inputs.run_ui }}, + "cuda":${{ (github.event_name == 'workflow_dispatch' && inputs.run_cuda) || (github.event_name != 'workflow_dispatch' && vars.ENABLE_CUDA_GATE == 'true') }}, + "mps":${{ (github.event_name == 'workflow_dispatch' && inputs.run_mps) || (github.event_name != 'workflow_dispatch' && vars.ENABLE_MPS_GATE == 'true') }}, + "mlx":${{ (github.event_name == 'workflow_dispatch' && inputs.run_mlx) || (github.event_name != 'workflow_dispatch' && vars.ENABLE_MLX_GATE == 'true') }}, + "remote":${{ (github.event_name == 'workflow_dispatch' && inputs.run_remote) || (github.event_name != 'workflow_dispatch' && vars.ENABLE_REMOTE_GATE == 'true') }}} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Enforce selected job results and publish freshness summary + run: python3 scripts/conditional_gate_summary.py + - name: Upload conditional summary + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: conditional-summary-${{ github.run_attempt }} + path: conditional-evidence/summary.json + if-no-files-found: error + retention-days: 30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b298e7..34c205a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,6 +60,16 @@ CI retains repeat timing/output, mutation timing, test selection, score, and survivor evidence for 14 days. Thresholds may move downward only through a time-bounded exception linked to a reviewed repository issue. +Hardware, model-download, network-service, operator-UI, and remote-provider tests +run separately so the pull-request baseline stays offline and credential-free. See +[`docs/conditional-testing.md`](docs/conditional-testing.md) for manual commands, +runner labels, pinned resources, credential handling, costs, cadence, and evidence +freshness. Validate the mapping between those gates and CPU coverage exclusions with: + +```bash +python scripts/check_conditional_policy.py +``` + ## Code Style We use [ruff](https://docs.astral.sh/ruff/) for linting and formatting: diff --git a/README.md b/README.md index 37329dd..9d8c360 100644 --- a/README.md +++ b/README.md @@ -567,7 +567,7 @@ obliteratus obliterate meta-llama/Llama-3.1-70B-Instruct \ # With SSH key and custom options obliteratus obliterate meta-llama/Llama-3.1-70B-Instruct \ - --remote root@10.0.0.5 \ + --remote obliteratus@10.0.0.5 \ --ssh-key ~/.ssh/id_rsa \ --ssh-port 2222 \ --remote-dir /data/obliteratus \ @@ -594,6 +594,11 @@ remote: sync_results: true # copy results back when done ``` +Remote SSH connections require strict host-key verification. Verify the provider +fingerprint through an independent channel and add it to your SSH `known_hosts` +before running OBLITERATUS. Use a non-root, least-privileged account restricted to +the intended compute directory and commands. + The remote runner: 1. Tests SSH connectivity 2. Detects GPUs on the remote (`nvidia-smi`) @@ -745,7 +750,7 @@ If you use OBLITERATUS in your research, please cite: author = {{OBLITERATUS Contributors}}, year = {2026}, url = {https://github.com/elder-plinius/OBLITERATUS}, - note = {15 analysis modules, 1,001 tests} + note = {15 analysis modules, 1,130 mandatory CPU tests} } ``` @@ -756,7 +761,7 @@ pip install -e ".[dev]" pytest ``` -The mandatory CPU suite currently contains 1,123 tests across 52 test files, +The mandatory CPU suite currently contains 1,130 tests across 54 test files, including a repository-owned synthetic model that exercises the offline pipeline, installed-wheel CLI, study runner, transactional checkpoint recovery, and resumable auto-obliteration state. The suite also covers model/device/quantization/MLX @@ -766,6 +771,9 @@ community contributions, edge cases, and evaluation metrics. CI enforces at leas changed-line coverage, and 80% statement / 75% branch coverage for the documented mature CPU-only scope. Deterministic property, order-repeat, and selective mutation gates provide additional depth for numerical and policy-critical behavior. +Eight environment-bound test files run through the separately documented conditional +workflow for model downloads, network services, operator UI, CUDA, bitsandbytes, MPS, +MLX, and least-privileged remote execution. ## License diff --git a/ci/conditional-test-policy.json b/ci/conditional-test-policy.json new file mode 100644 index 0000000..8616da5 --- /dev/null +++ b/ci/conditional-test-policy.json @@ -0,0 +1,121 @@ +{ + "schema_version": 1, + "owner": "OBLITERATUS maintainers", + "workflow": ".github/workflows/conditional-tests.yml", + "cadence": "weekly and on published releases; manual dispatch is always available", + "evidence_retention_days": 30, + "maximum_evidence_age_days": 8, + "resources": { + "tiny_model": { + "repository": "hf-internal-testing/tiny-random-gpt2", + "revision": "71034c5d8bde858ff824298bdedc65515b97d2b9", + "trust_remote_code": false, + "cache": "GitHub Actions cache keyed by repository, revision, runner OS, and Python version", + "timeout_minutes": 20 + } + }, + "gates": [ + { + "id": "model-download-runtime", + "job": "model_runtime", + "marker": "network and download", + "runner": "ubuntu-latest", + "prerequisites": "public HTTPS access to the pinned tiny model", + "expected_cost": "under 20 runner-minutes and 100 MB download", + "coverage_paths": [ + "obliteratus/abliterate.py", + "obliteratus/auto_obliterate.py", + "obliteratus/bayesian_optimizer.py", + "obliteratus/evaluation/baselines.py", + "obliteratus/evaluation/evaluator.py", + "obliteratus/informed_pipeline.py", + "obliteratus/lora_ablation.py", + "obliteratus/sweep.py" + ] + }, + { + "id": "external-evaluation", + "job": "model_runtime", + "marker": "network and download", + "runner": "ubuntu-latest", + "prerequisites": "public HTTPS access to the pinned tiny model", + "expected_cost": "included in model-runtime", + "coverage_paths": [ + "obliteratus/evaluation/heretic_eval.py", + "obliteratus/evaluation/lm_eval_integration.py", + "obliteratus/tourney.py" + ] + }, + { + "id": "network-services", + "job": "network_services", + "marker": "network", + "runner": "ubuntu-latest", + "prerequisites": "loopback networking; no credentials", + "expected_cost": "under 5 runner-minutes", + "coverage_paths": [ + "obliteratus/bestiary_sync.py", + "obliteratus/models_client.py", + "obliteratus/watchtower.py" + ] + }, + { + "id": "operator-ui", + "job": "operator_ui", + "marker": "operator_ui", + "runner": "ubuntu-latest", + "prerequisites": "locked spaces extra; no public listener", + "expected_cost": "under 10 runner-minutes", + "coverage_paths": [ + "obliteratus/interactive.py", + "obliteratus/local_ui.py", + "obliteratus/ui_watchtower.py" + ] + }, + { + "id": "cuda-runtime", + "job": "cuda", + "marker": "gpu", + "runner": "self-hosted, linux, x64, cuda", + "prerequisites": "ENABLE_CUDA_GATE=true and a dedicated CUDA runner", + "expected_cost": "under 15 self-hosted runner-minutes", + "coverage_paths": ["obliteratus/device.py", "obliteratus/models/loader.py"] + }, + { + "id": "bitsandbytes-runtime", + "job": "cuda", + "marker": "gpu", + "runner": "self-hosted, linux, x64, cuda", + "prerequisites": "ENABLE_CUDA_GATE=true, NVIDIA CUDA, and locked bitsandbytes", + "expected_cost": "included in cuda", + "coverage_paths": ["obliteratus/models/loader.py"] + }, + { + "id": "mps-runtime", + "job": "mps", + "marker": "mps", + "runner": "self-hosted, macOS, ARM64, mps", + "prerequisites": "ENABLE_MPS_GATE=true and an Apple Silicon MPS runner", + "expected_cost": "under 10 self-hosted runner-minutes", + "coverage_paths": ["obliteratus/device.py"] + }, + { + "id": "mlx-runtime", + "job": "mlx", + "marker": "mlx", + "runner": "self-hosted, macOS, ARM64, mlx", + "prerequisites": "ENABLE_MLX_GATE=true and an Apple Silicon runner with MLX", + "expected_cost": "under 15 self-hosted runner-minutes", + "coverage_paths": ["obliteratus/mlx_backend.py"] + }, + { + "id": "remote-execution", + "job": "remote", + "marker": "remote", + "runner": "ubuntu-latest", + "prerequisites": "ENABLE_REMOTE_GATE=true plus least-privileged SSH host, user, key, and pinned known_hosts secrets", + "expected_cost": "under 5 runner-minutes plus provider charges", + "coverage_paths": ["obliteratus/remote.py"] + } + ] +} diff --git a/docs/conditional-testing.md b/docs/conditional-testing.md new file mode 100644 index 0000000..d9ba3af --- /dev/null +++ b/docs/conditional-testing.md @@ -0,0 +1,102 @@ +# Conditional test operations + +The mandatory pull-request workflow remains CPU-only, offline, and credential-free. +Environment-bound contracts run through `Conditional tests` on manual dispatch, +weekly at 06:17 UTC Sunday, and for published releases. OBLITERATUS maintainers own +the workflow, policy, runner labels, credentials, and evidence review. + +The canonical machine-readable policy is `ci/conditional-test-policy.json`. It maps +every environment-bound mature-coverage exclusion to a runnable gate, defines the +pinned tiny model, records prerequisites and expected cost, retains evidence for 30 +days, and treats evidence older than eight days as stale. Every workflow run publishes +a summary showing selected, successful, failed, and not-selected/no-fresh-evidence +gates. + +## Hosted gates + +The model gate downloads only +`hf-internal-testing/tiny-random-gpt2@71034c5d8bde858ff824298bdedc65515b97d2b9` +with `trust_remote_code=False`. Its cache key includes the model revision, Python +version, and runner OS. The gate performs a forward pass, reopens the cache with Hub +and Transformers offline modes enabled, asserts an uncached model fails offline, and +runs the evaluator on two tiny samples. Its timeout is 25 minutes and its expected +download is below 100 MB. + +The network-service gate uses a disposable loopback HTTP server and no credentials. +The operator-UI gate installs the locked `spaces` extra and constructs the Gradio +application without launching a listener. These gates cost less than ten hosted +runner-minutes each under normal conditions. + +Run the same probes locally with: + +```bash +uv sync --locked --extra dev +uv run --extra dev python scripts/run_conditional_gate.py model-download-runtime +uv run --extra dev python scripts/run_conditional_gate.py external-evaluation +uv run --extra dev python scripts/run_conditional_gate.py network-services + +uv sync --locked --all-extras +uv run --all-extras python scripts/run_conditional_gate.py operator-ui +``` + +## CUDA and bitsandbytes + +Attach a dedicated runner with the labels `self-hosted`, `linux`, `x64`, and `cuda`. +Set the repository variable `ENABLE_CUDA_GATE=true` for scheduled/release evidence, +or select CUDA during manual dispatch. The gate verifies CUDA discovery, automatic +selection, dtype selection, tensor placement, matrix multiplication, bitsandbytes +availability, NF4 quantization, dequantization, shape, placement, and finite output. +The expected cost is below 20 self-hosted runner-minutes. + +For an operator run on the labeled machine: + +```bash +uv sync --locked --extra dev +uv run --extra dev python scripts/run_conditional_gate.py cuda-runtime +uv run --extra dev python scripts/run_conditional_gate.py bitsandbytes-runtime +``` + +## Apple MPS and MLX + +MPS uses a runner labeled `self-hosted`, `macOS`, `ARM64`, and `mps`; enable its +schedule with `ENABLE_MPS_GATE=true`. MLX uses the same first three labels plus `mlx` +and `ENABLE_MLX_GATE=true`. The MPS probe checks discovery, selection, dtype and +float64 fallback, placement, and a real matrix operation. The MLX probe uses the +locked `mlx==0.32.0` and `mlx-lm==0.31.3` packages and verifies imports, array +placement, evaluation, and matrix multiplication. Each should cost less than 20 +self-hosted runner-minutes. + +If the repository has no attached Apple runner, collect equivalent operator evidence +on Apple Silicon and attach the JSON and JUnit files to the tracking issue: + +```bash +uv sync --locked --extra dev +uv run --extra dev python scripts/run_conditional_gate.py mps-runtime + +uv sync --locked --extra dev --group mlx +uv run --extra dev --group mlx python scripts/run_conditional_gate.py mlx-runtime +``` + +## Remote provider + +Remote evidence is opt-in. Configure a non-root, command-limited test account and: + +- variables `OBLITERATUS_REMOTE_HOST`, `OBLITERATUS_REMOTE_USER`, and optionally + `OBLITERATUS_REMOTE_PORT`; +- secrets `OBLITERATUS_REMOTE_KEY` and `OBLITERATUS_REMOTE_KNOWN_HOSTS`. + +The known-hosts entry must be pinned after verifying the provider fingerprint through +an independent channel. The workflow writes credentials to mode-0600 temporary files, +never prints their contents, uses batch mode and strict host-key checking, and only +runs `echo ok` plus `python3 -c 'print(6 * 7)'`. Missing prerequisites produce an +actionable failure when the gate was selected; a direct local invocation may use +`--allow-missing` to record explicit `not_run` evidence. Expected cost is below five +hosted runner-minutes plus any provider charge. + +## Result semantics + +`scripts/run_conditional_gate.py` requires at least one executed test and rejects any +failure, error, or skip. A selected workflow job therefore cannot become green through +an availability skip or unconditional success conversion. The final summary also +fails if any selected job is not successful. Unselected jobs are explicitly reported +as `not_selected_no_fresh_evidence`; they are not evidence of backend support. diff --git a/obliteratus/remote.py b/obliteratus/remote.py index 05fa314..5dd8c5f 100644 --- a/obliteratus/remote.py +++ b/obliteratus/remote.py @@ -39,6 +39,7 @@ class RemoteConfig: user: str = "root" port: int = 22 ssh_key: str | None = None + known_hosts_file: str | None = None remote_dir: str = "/tmp/obliteratus_run" install_timeout: int = 600 # seconds python: str = "python3" # remote python binary @@ -79,11 +80,14 @@ class RemoteRunner: """Build base SSH command with common options.""" cmd = [ "ssh", - "-o", "StrictHostKeyChecking=no", + "-o", "StrictHostKeyChecking=yes", "-o", "BatchMode=yes", "-o", "ConnectTimeout=30", "-p", str(self.config.port), ] + if self.config.known_hosts_file: + known_hosts = os.path.expanduser(self.config.known_hosts_file) + cmd.extend(["-o", f"UserKnownHostsFile={known_hosts}"]) if self.config.ssh_key: key_path = os.path.expanduser(self.config.ssh_key) cmd.extend(["-i", key_path]) @@ -94,11 +98,14 @@ class RemoteRunner: """Build base SCP command.""" cmd = [ "scp", - "-o", "StrictHostKeyChecking=no", + "-o", "StrictHostKeyChecking=yes", "-o", "BatchMode=yes", "-P", str(self.config.port), "-r", ] + if self.config.known_hosts_file: + known_hosts = os.path.expanduser(self.config.known_hosts_file) + cmd.extend(["-o", f"UserKnownHostsFile={known_hosts}"]) if self.config.ssh_key: key_path = os.path.expanduser(self.config.ssh_key) cmd.extend(["-i", key_path]) diff --git a/pyproject.toml b/pyproject.toml index 2032fab..595128c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,10 @@ ci = [ quality = [ "mutmut==3.7.0", ] +mlx = [ + "mlx==0.32.0; sys_platform == 'darwin' and platform_machine == 'arm64'", + "mlx-lm==0.31.3; sys_platform == 'darwin' and platform_machine == 'arm64'", +] [project.scripts] obliteratus = "obliteratus.cli:main" @@ -106,6 +110,7 @@ markers = [ "network: tests that access external network resources", "download: tests that download model or dataset artifacts", "remote: tests that require a remote execution provider or credentials", + "operator_ui: tests that require the optional UI runtime", ] [tool.mutmut] diff --git a/scripts/check_conditional_policy.py b/scripts/check_conditional_policy.py new file mode 100644 index 0000000..e38e79d --- /dev/null +++ b/scripts/check_conditional_policy.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Validate conditional-gate policy and its CPU-coverage mappings.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +REQUIRED_GATE_FIELDS = { + "id", "job", "marker", "runner", "prerequisites", "expected_cost", "coverage_paths" +} + + +def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list[str]: + errors: list[str] = [] + policy = json.loads(policy_path.read_text()) + quality = json.loads(quality_path.read_text()) + workflow = workflow_path.read_text() + + if policy.get("schema_version") != 1: + errors.append("conditional policy schema_version must be 1") + for key in ("owner", "cadence", "evidence_retention_days", "maximum_evidence_age_days"): + if not policy.get(key): + errors.append(f"conditional policy is missing {key}") + + gates = policy.get("gates") + if not isinstance(gates, list) or not gates: + return errors + ["conditional policy gates must be a non-empty list"] + + by_id: dict[str, dict] = {} + for index, gate in enumerate(gates): + missing = REQUIRED_GATE_FIELDS - set(gate) + if missing: + errors.append(f"gate {index} is missing fields: {sorted(missing)}") + continue + gate_id = gate["id"] + if gate_id in by_id: + errors.append(f"duplicate conditional gate id: {gate_id}") + by_id[gate_id] = gate + if not gate["coverage_paths"]: + errors.append(f"gate {gate_id} has no coverage paths") + if f"{gate['job']}:" not in workflow: + errors.append(f"workflow job {gate['job']!r} for {gate_id} was not found") + for source_path in gate["coverage_paths"]: + if not Path(source_path).is_file(): + errors.append(f"gate {gate_id} maps missing source path: {source_path}") + + exclusions = quality.get("mature_cpu_scope", {}).get("exclusions", []) + for exclusion in exclusions: + gate_id = exclusion.get("conditional_gate") + source_path = exclusion.get("path") + if gate_id not in by_id: + errors.append(f"CPU exclusion {source_path} references unknown gate {gate_id}") + continue + if source_path not in by_id[gate_id]["coverage_paths"]: + errors.append(f"CPU exclusion {source_path} is not mapped by gate {gate_id}") + + required_workflow_tokens = ( + "workflow_dispatch:", "schedule:", "release:", "permissions:", "contents: read", + "scripts/run_conditional_gate.py", "scripts/conditional_gate_summary.py", + ) + for token in required_workflow_tokens: + if token not in workflow: + errors.append(f"conditional workflow is missing {token!r}") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--policy", type=Path, default=Path("ci/conditional-test-policy.json")) + parser.add_argument("--quality", type=Path, default=Path("ci/test-quality-policy.json")) + parser.add_argument( + "--workflow", type=Path, default=Path(".github/workflows/conditional-tests.yml") + ) + args = parser.parse_args() + errors = validate(args.policy, args.quality, args.workflow) + if errors: + for error in errors: + print(f"ERROR: {error}") + return 1 + print("conditional test policy: valid") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/conditional_gate_summary.py b/scripts/conditional_gate_summary.py new file mode 100644 index 0000000..6ac1913 --- /dev/null +++ b/scripts/conditional_gate_summary.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Build a final conditional-workflow result and freshness summary.""" + +from __future__ import annotations + +import argparse +import json +import os +from datetime import datetime, timezone +from pathlib import Path + + +def summarize(policy: dict, results: dict, selected: dict) -> tuple[list[dict], list[str]]: + """Return per-gate rows and selected-job failures.""" + failures: list[str] = [] + rows: list[dict] = [] + failed_jobs: set[str] = set() + for gate in policy["gates"]: + job = gate["job"] + is_selected = bool(selected.get(job, False)) + result = results.get(job, "unknown") + status = result if is_selected else "not_selected_no_fresh_evidence" + if is_selected and result != "success" and job not in failed_jobs: + failures.append(f"{job}: selected but result was {result}") + failed_jobs.add(job) + rows.append({"gate": gate["id"], "job": job, "selected": is_selected, "status": status}) + return rows, failures + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--policy", type=Path, default=Path("ci/conditional-test-policy.json")) + parser.add_argument("--output", type=Path, default=Path("conditional-evidence/summary.json")) + parser.add_argument("--results-json", default=os.environ.get("CONDITIONAL_RESULTS", "{}")) + parser.add_argument("--selected-json", default=os.environ.get("CONDITIONAL_SELECTED", "{}")) + args = parser.parse_args() + policy = json.loads(args.policy.read_text()) + results = json.loads(args.results_json) + selected = json.loads(args.selected_json) + rows, failures = summarize(policy, results, selected) + payload = { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "git_sha": os.environ.get("GITHUB_SHA", "local"), + "maximum_evidence_age_days": policy["maximum_evidence_age_days"], + "gates": rows, + "failures": failures, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + lines = ["## Conditional test evidence", "", "| Gate | Job | Status |", "|---|---|---|"] + lines.extend(f"| {row['gate']} | {row['job']} | {row['status']} |" for row in rows) + lines.extend(["", f"Evidence becomes stale after {policy['maximum_evidence_age_days']} days."]) + with Path(summary_path).open("a") as handle: + handle.write("\n".join(lines) + "\n") + for failure in failures: + print(f"ERROR: {failure}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_conditional_gate.py b/scripts/run_conditional_gate.py new file mode 100644 index 0000000..1e78ac9 --- /dev/null +++ b/scripts/run_conditional_gate.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Run one conditional pytest gate and reject empty or silently skipped evidence.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from xml.etree import ElementTree + + +GATES = { + "model-download-runtime": "tests/conditional/test_model_download_runtime.py", + "external-evaluation": "tests/conditional/test_external_evaluation_runtime.py", + "network-services": "tests/conditional/test_network_services.py", + "operator-ui": "tests/conditional/test_operator_ui.py", + "cuda-runtime": "tests/conditional/test_cuda_runtime.py", + "bitsandbytes-runtime": "tests/conditional/test_cuda_runtime.py", + "mps-runtime": "tests/conditional/test_mps_runtime.py", + "mlx-runtime": "tests/conditional/test_mlx_runtime.py", + "remote-execution": "tests/conditional/test_remote_runtime.py", +} + + +def missing_prerequisites(gate: str) -> list[str]: + missing: list[str] = [] + if gate in {"cuda-runtime", "bitsandbytes-runtime", "mps-runtime"}: + import torch + + if gate.startswith("cuda") or gate.startswith("bitsandbytes"): + if not torch.cuda.is_available(): + missing.append("a CUDA-capable PyTorch runtime") + elif not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()): + missing.append("an available Apple MPS backend") + if gate == "bitsandbytes-runtime" and importlib.util.find_spec("bitsandbytes") is None: + missing.append("bitsandbytes") + if gate == "mlx-runtime": + for module in ("mlx", "mlx_lm"): + if importlib.util.find_spec(module) is None: + missing.append(module) + if gate == "remote-execution": + for variable in ( + "OBLITERATUS_REMOTE_HOST", + "OBLITERATUS_REMOTE_USER", + "OBLITERATUS_REMOTE_KEY", + "OBLITERATUS_REMOTE_KNOWN_HOSTS", + ): + if not os.environ.get(variable): + missing.append(variable) + return missing + + +def counts(junit_path: Path) -> dict[str, int]: + root = ElementTree.parse(junit_path).getroot() + suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) + return { + key: sum(int(suite.attrib.get(key, "0")) for suite in suites) + for key in ("tests", "failures", "errors", "skipped") + } + + +def write_report(path: Path, gate: str, status: str, **extra: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema_version": 1, + "gate": gate, + "status": status, + "generated_at": datetime.now(timezone.utc).isoformat(), + "git_sha": os.environ.get("GITHUB_SHA", "local"), + **extra, + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("gate", choices=sorted(GATES)) + parser.add_argument("--evidence-dir", type=Path, default=Path("conditional-evidence")) + parser.add_argument("--allow-missing", action="store_true") + args = parser.parse_args() + report = args.evidence_dir / f"{args.gate}.json" + junit = args.evidence_dir / f"{args.gate}.xml" + + missing = missing_prerequisites(args.gate) + if missing: + message = "Missing prerequisites: " + ", ".join(missing) + write_report(report, args.gate, "not_run", reason=message) + print(message, file=sys.stderr) + return 0 if args.allow_missing else 2 + + command = [ + sys.executable, + "-m", + "pytest", + GATES[args.gate], + "--no-cov", + "-q", + f"--junitxml={junit}", + ] + result = subprocess.run(command, check=False) + if not junit.is_file(): + write_report(report, args.gate, "failed", exit_code=result.returncode, reason="no JUnit") + return result.returncode or 1 + + result_counts = counts(junit) + passed = ( + result.returncode == 0 + and result_counts["tests"] > 0 + and result_counts["failures"] == 0 + and result_counts["errors"] == 0 + and result_counts["skipped"] == 0 + ) + write_report( + report, + args.gate, + "passed" if passed else "failed", + exit_code=result.returncode, + counts=result_counts, + ) + if not passed: + print(f"conditional gate did not produce unskipped green evidence: {result_counts}") + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conditional/test_cuda_runtime.py b/tests/conditional/test_cuda_runtime.py new file mode 100644 index 0000000..2ac4bee --- /dev/null +++ b/tests/conditional/test_cuda_runtime.py @@ -0,0 +1,36 @@ +"""Real CUDA and bitsandbytes placement/operation probes.""" + +from __future__ import annotations + +import pytest +import torch + +from obliteratus import device + + +pytestmark = pytest.mark.gpu + + +def test_cuda_discovery_dtype_placement_and_operation(): + if not torch.cuda.is_available(): + pytest.skip("requires a CUDA runner; set ENABLE_CUDA_GATE and attach the cuda label") + assert device.is_cuda() + assert device.get_device("auto") == "cuda" + assert device.default_dtype("cuda") is torch.float16 + tensor = torch.arange(16, device="cuda", dtype=torch.float32).reshape(4, 4) + result = tensor @ tensor.T + assert result.device.type == "cuda" + assert torch.isfinite(result).all() + + +def test_bitsandbytes_quantization_operation(): + if not torch.cuda.is_available(): + pytest.skip("requires a CUDA runner; set ENABLE_CUDA_GATE and attach the cuda label") + bnb = pytest.importorskip("bitsandbytes", reason="install locked bitsandbytes>=0.46.1") + assert device.supports_bitsandbytes("cuda") + source = torch.randn(16, 16, device="cuda", dtype=torch.float16) + quantized, state = bnb.functional.quantize_4bit(source, quant_type="nf4") + restored = bnb.functional.dequantize_4bit(quantized, state) + assert restored.device.type == "cuda" + assert restored.shape == source.shape + assert torch.isfinite(restored).all() diff --git a/tests/conditional/test_external_evaluation_runtime.py b/tests/conditional/test_external_evaluation_runtime.py new file mode 100644 index 0000000..72b6fc3 --- /dev/null +++ b/tests/conditional/test_external_evaluation_runtime.py @@ -0,0 +1,32 @@ +"""Run the evaluator against the pinned tiny causal language model.""" + +from __future__ import annotations + +import math + +import pytest +from datasets import Dataset + +from obliteratus.evaluation.evaluator import Evaluator +from obliteratus.models.loader import load_model + + +pytestmark = [pytest.mark.network, pytest.mark.download] +MODEL = "hf-internal-testing/tiny-random-gpt2" +REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9" + + +def test_pinned_tiny_model_evaluator_produces_finite_perplexity(): + handle = load_model( + MODEL, + revision=REVISION, + device="cpu", + trust_remote_code=False, + skip_snapshot=True, + ) + handle.tokenizer.pad_token = handle.tokenizer.eos_token + dataset = Dataset.from_dict({"text": ["A short test sentence.", "A second sentence."]}) + result = Evaluator(handle, dataset, batch_size=2, max_length=24).evaluate() + assert math.isfinite(result["perplexity"]) + assert result["perplexity"] > 0 + handle.cleanup() diff --git a/tests/conditional/test_mlx_runtime.py b/tests/conditional/test_mlx_runtime.py new file mode 100644 index 0000000..db4c3e7 --- /dev/null +++ b/tests/conditional/test_mlx_runtime.py @@ -0,0 +1,20 @@ +"""Real Apple MLX selection and minimal operation probe.""" + +from __future__ import annotations + +import pytest + + +pytestmark = pytest.mark.mlx + + +def test_mlx_import_array_placement_and_operation(): + mx = pytest.importorskip("mlx.core", reason="requires Apple Silicon with mlx installed") + pytest.importorskip("mlx_lm", reason="requires mlx-lm installed") + from obliteratus import mlx_backend + + assert mlx_backend.MLX_AVAILABLE + left = mx.arange(16, dtype=mx.float32).reshape((4, 4)) + result = mx.matmul(left, mx.transpose(left)) + mx.eval(result) + assert result.shape == (4, 4) diff --git a/tests/conditional/test_model_download_runtime.py b/tests/conditional/test_model_download_runtime.py new file mode 100644 index 0000000..223ad26 --- /dev/null +++ b/tests/conditional/test_model_download_runtime.py @@ -0,0 +1,49 @@ +"""Pinned, networked tiny-model load plus cache-only replay.""" + +from __future__ import annotations + +import uuid + +import pytest +import torch + +from obliteratus.models.loader import load_model + + +pytestmark = [pytest.mark.network, pytest.mark.download] +MODEL = "hf-internal-testing/tiny-random-gpt2" +REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9" + + +def test_pinned_tiny_model_download_inference_and_offline_cache(monkeypatch): + handle = load_model( + MODEL, + revision=REVISION, + device="cpu", + dtype="float32", + trust_remote_code=False, + skip_snapshot=True, + ) + encoded = handle.tokenizer("conditional gate", return_tensors="pt") + with torch.no_grad(): + output = handle.model(**encoded) + assert output.logits.shape[:2] == encoded["input_ids"].shape + assert next(handle.model.parameters()).device.type == "cpu" + handle.cleanup() + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + cached = load_model( + MODEL, + revision=REVISION, + device="cpu", + local_files_only=True, + trust_remote_code=False, + skip_snapshot=True, + ) + assert cached.model_name == MODEL + cached.cleanup() + + missing = f"obliteratus/offline-missing-{uuid.uuid4().hex}" + with pytest.raises(OSError): + load_model(missing, revision=REVISION, device="cpu", local_files_only=True) diff --git a/tests/conditional/test_mps_runtime.py b/tests/conditional/test_mps_runtime.py new file mode 100644 index 0000000..e673d67 --- /dev/null +++ b/tests/conditional/test_mps_runtime.py @@ -0,0 +1,24 @@ +"""Real Apple MPS discovery, selection, placement, and operation probe.""" + +from __future__ import annotations + +import pytest +import torch + +from obliteratus import device + + +pytestmark = pytest.mark.mps + + +def test_mps_discovery_selection_placement_and_operation(): + if not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()): + pytest.skip("requires an Apple Silicon MPS runner with ENABLE_MPS_GATE=true") + assert device.is_mps() + assert device.get_device("auto") == "mps" + assert device.default_dtype("mps") is torch.float16 + assert not device.supports_float64("mps") + tensor = torch.arange(16, device="mps", dtype=torch.float32).reshape(4, 4) + result = tensor @ tensor.T + assert result.device.type == "mps" + assert torch.isfinite(result).all().item() diff --git a/tests/conditional/test_network_services.py b/tests/conditional/test_network_services.py new file mode 100644 index 0000000..c95b278 --- /dev/null +++ b/tests/conditional/test_network_services.py @@ -0,0 +1,53 @@ +"""Exercise the service boundary over a disposable loopback HTTP endpoint.""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from obliteratus import models_client + + +pytestmark = pytest.mark.network + + +def test_bestiary_catalog_http_boundary(monkeypatch): + catalog = { + "models": [ + { + "id": "example/tiny", + "vendor": "example", + "open_weight": True, + "channels": ["huggingface"], + "capabilities": ["text"], + } + ] + } + body = json.dumps(catalog).encode() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + monkeypatch.setenv( + "BESTIARY_CATALOG", f"http://127.0.0.1:{server.server_port}/catalog.json" + ) + assert models_client.model_ids(open_weight=True) == ["example/tiny"] + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() diff --git a/tests/conditional/test_operator_ui.py b/tests/conditional/test_operator_ui.py new file mode 100644 index 0000000..3ac4c17 --- /dev/null +++ b/tests/conditional/test_operator_ui.py @@ -0,0 +1,33 @@ +"""Construct the optional Gradio UI without opening a listener.""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + + +pytestmark = pytest.mark.operator_ui + + +def test_gradio_application_constructs_without_public_listener(): + pytest.importorskip("gradio", reason="install the locked spaces extra") + probe = subprocess.run( + [ + sys.executable, + "-c", + "import app, gradio; " + "assert type(app.demo).__name__ == 'Blocks'; " + "assert app.demo.local_url is None; " + "assert int(gradio.__version__.split('.')[0]) >= 6; " + "print('operator UI constructed without a listener')", + ], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + assert probe.returncode == 0, probe.stderr + assert probe.stdout.strip() == "operator UI constructed without a listener" + assert probe.stderr == "" diff --git a/tests/conditional/test_remote_runtime.py b/tests/conditional/test_remote_runtime.py new file mode 100644 index 0000000..b0abd65 --- /dev/null +++ b/tests/conditional/test_remote_runtime.py @@ -0,0 +1,36 @@ +"""Least-privileged, strict-host-key remote execution probe.""" + +from __future__ import annotations + +import os + +import pytest + +from obliteratus.remote import RemoteConfig, RemoteRunner + + +pytestmark = pytest.mark.remote + + +def _required(name: str) -> str: + value = os.environ.get(name) + if not value: + pytest.skip(f"set {name} and enable the remote conditional gate") + return value + + +def test_remote_provider_connection_and_minimal_command(): + config = RemoteConfig( + host=_required("OBLITERATUS_REMOTE_HOST"), + user=_required("OBLITERATUS_REMOTE_USER"), + port=int(os.environ.get("OBLITERATUS_REMOTE_PORT") or "22"), + ssh_key=_required("OBLITERATUS_REMOTE_KEY"), + known_hosts_file=_required("OBLITERATUS_REMOTE_KNOWN_HOSTS"), + ) + assert config.user != "root", "the conditional provider must be least-privileged" + runner = RemoteRunner(config, on_log=lambda _message: None) + assert "StrictHostKeyChecking=yes" in runner._ssh_base_cmd() + assert runner.check_connection() + result = runner.run_ssh("python3 -c 'print(6 * 7)'", timeout=30) + assert result.returncode == 0 + assert result.stdout.strip() == "42" diff --git a/tests/test_conditional_gate_scripts.py b/tests/test_conditional_gate_scripts.py new file mode 100644 index 0000000..92d1ad0 --- /dev/null +++ b/tests/test_conditional_gate_scripts.py @@ -0,0 +1,94 @@ +"""Tests for conditional policy, runner, and evidence helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from scripts import check_conditional_policy +from scripts import conditional_gate_summary +from scripts import run_conditional_gate + + +ROOT = Path(__file__).parents[1] + + +def test_committed_conditional_policy_is_complete(): + assert check_conditional_policy.validate( + ROOT / "ci" / "conditional-test-policy.json", + ROOT / "ci" / "test-quality-policy.json", + ROOT / ".github" / "workflows" / "conditional-tests.yml", + ) == [] + + +def test_policy_rejects_unknown_cpu_exclusion_gate(tmp_path): + policy = json.loads((ROOT / "ci" / "conditional-test-policy.json").read_text()) + quality = { + "mature_cpu_scope": { + "exclusions": [{"path": "obliteratus/device.py", "conditional_gate": "missing"}] + } + } + policy_path = tmp_path / "policy.json" + quality_path = tmp_path / "quality.json" + policy_path.write_text(json.dumps(policy)) + quality_path.write_text(json.dumps(quality)) + errors = check_conditional_policy.validate( + policy_path, + quality_path, + ROOT / ".github" / "workflows" / "conditional-tests.yml", + ) + assert errors == ["CPU exclusion obliteratus/device.py references unknown gate missing"] + + +def test_junit_counts_and_report_are_machine_readable(tmp_path): + junit = tmp_path / "result.xml" + junit.write_text( + '' + '' + ) + assert run_conditional_gate.counts(junit) == { + "tests": 5, + "failures": 1, + "errors": 1, + "skipped": 1, + } + report = tmp_path / "report.json" + run_conditional_gate.write_report(report, "network-services", "passed", counts={"tests": 1}) + evidence = json.loads(report.read_text()) + assert evidence["gate"] == "network-services" + assert evidence["status"] == "passed" + assert evidence["counts"] == {"tests": 1} + + +def test_remote_prerequisites_are_actionable(monkeypatch): + names = { + "OBLITERATUS_REMOTE_HOST", + "OBLITERATUS_REMOTE_USER", + "OBLITERATUS_REMOTE_KEY", + "OBLITERATUS_REMOTE_KNOWN_HOSTS", + } + for name in names: + monkeypatch.delenv(name, raising=False) + assert set(run_conditional_gate.missing_prerequisites("remote-execution")) == names + for name in names: + monkeypatch.setenv(name, "fixture") + assert run_conditional_gate.missing_prerequisites("remote-execution") == [] + + +def test_summary_rejects_selected_failure_and_marks_unselected_stale(): + policy = { + "gates": [ + {"id": "first", "job": "shared"}, + {"id": "second", "job": "shared"}, + {"id": "optional", "job": "optional"}, + ] + } + rows, failures = conditional_gate_summary.summarize( + policy, + {"shared": "failure", "optional": "skipped"}, + {"shared": True, "optional": False}, + ) + assert failures == ["shared: selected but result was failure"] + assert rows[0]["status"] == "failure" + assert rows[1]["status"] == "failure" + assert rows[2]["status"] == "not_selected_no_fresh_evidence" diff --git a/tests/test_remote_boundaries.py b/tests/test_remote_boundaries.py new file mode 100644 index 0000000..8ad20d1 --- /dev/null +++ b/tests/test_remote_boundaries.py @@ -0,0 +1,36 @@ +"""Offline contracts for strict SSH command construction.""" + +from __future__ import annotations + +from obliteratus.remote import RemoteConfig, RemoteRunner + + +def test_remote_commands_require_strict_host_key_verification(tmp_path): + key = tmp_path / "key" + known_hosts = tmp_path / "known_hosts" + config = RemoteConfig( + host="compute.example", + user="runner", + port=2222, + ssh_key=str(key), + known_hosts_file=str(known_hosts), + ) + runner = RemoteRunner(config) + for command in (runner._ssh_base_cmd(), runner._scp_base_cmd()): + assert "StrictHostKeyChecking=yes" in command + assert f"UserKnownHostsFile={known_hosts}" in command + assert "StrictHostKeyChecking=no" not in command + assert str(key) in command + + +def test_remote_config_accepts_versioned_known_hosts_setting(): + config = RemoteConfig.from_dict( + { + "host": "compute.example", + "user": "runner", + "known_hosts_file": "/secure/known_hosts", + "unknown": "ignored", + } + ) + assert config.known_hosts_file == "/secure/known_hosts" + assert config.ssh_target == "runner@compute.example" diff --git a/uv.lock b/uv.lock index f2b6245..53f8a6d 100644 --- a/uv.lock +++ b/uv.lock @@ -15,7 +15,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-11T17:42:28.049938237Z" +exclude-newer = "2026-08-11T18:17:23.461940614Z" exclude-newer-span = "P3D" [manifest] @@ -1996,6 +1996,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mlx" +version = "0.32.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/38/a034159df4d21ef7b5721225a2c46092e20a2c627724f57be3e89fa7dbce/mlx-0.32.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:c6feb17e32160b70c7634aab925cf3f8c5c7bebbf99f227c48450478e1008af2", size = 562899, upload-time = "2026-07-07T17:55:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8c/054d26695279155b590a95330440b592bd9c2dfe9cf450812b18ac598962/mlx-0.32.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:73303259f2bda7fb4a0c782a7299e0e28a2890b9b6fbfc4b635fb8032208ec7e", size = 562898, upload-time = "2026-07-07T17:55:26.995Z" }, + { url = "https://files.pythonhosted.org/packages/16/06/9f74a23e8d3931a87e0fa24044fe178656d69852ca20493b50e7da2326c3/mlx-0.32.0-cp310-cp310-macosx_26_0_arm64.whl", hash = "sha256:8637003c6eb089443d149fdb483f5d7a7846d6cc43d112148d7aa07cf1356c04", size = 562871, upload-time = "2026-07-07T17:55:28.668Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c7/cb62301b01dbccd66b256cab0c98fc29e7533dd76aa599fe44c1bb1f4168/mlx-0.32.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:72c605368d145c756877057d7e3c54f169c9899fe1f83232bfb3a6342561e234", size = 562792, upload-time = "2026-07-07T17:55:35.24Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b1/c2eba5bb18c94a9fe1b5d6599f18ba2ef5de2d8b42439716d9241ab196c4/mlx-0.32.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:0a0e38a409b9cae29647ec9e75ce9747b224ce7e5d91adbfad7eac37b6118ffd", size = 562792, upload-time = "2026-07-07T17:55:36.876Z" }, + { url = "https://files.pythonhosted.org/packages/1f/43/5b481bfb8f1234153fd8fef9dcacfe34860b0934eb507c0eb811ba599afe/mlx-0.32.0-cp311-cp311-macosx_26_0_arm64.whl", hash = "sha256:2a180cd39ac68b397b85cc4658d0b8e0ab58166c2549fc9ab2ca5f99d15ef0c3", size = 562776, upload-time = "2026-07-07T17:55:38.314Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a6/489176d8a2a06137a299910057cc44dc3fccfb73151f7562fea2b75894d9/mlx-0.32.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:ea5a594355c89c0095eaba413fd39d4caa8642fa13432dfb0c9354d141046467", size = 558889, upload-time = "2026-07-07T17:55:45.268Z" }, + { url = "https://files.pythonhosted.org/packages/85/2a/5d1f1cb1b073c39c822e0c0be1e68f4ced6fd32ab15cf8bb1f448028842c/mlx-0.32.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5f778001562ccce26cf6e5be1050d2afc78e2902bad206201ab9f5a6d0f886a", size = 558890, upload-time = "2026-07-07T17:55:46.701Z" }, + { url = "https://files.pythonhosted.org/packages/0f/01/02110ceacf4efd00ec172f60b4cd42c8b1509ac50ffa65a6a77d723133aa/mlx-0.32.0-cp312-cp312-macosx_26_0_arm64.whl", hash = "sha256:8dfb577faa4dc413cfd0d6eb78f230d3b3b6169df4473e84408abdeb21346e9d", size = 558856, upload-time = "2026-07-07T17:55:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/b6/be/ddc888d4a20c7602da06ad1a244f495010c6c7d2457f6253e8fa3c99ceea/mlx-0.32.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:abb786ee1e9638759be82583222fc7d09c5650ef90ad2b7c5da7d1931a8676dc", size = 558795, upload-time = "2026-07-07T17:55:54.907Z" }, + { url = "https://files.pythonhosted.org/packages/38/b9/4f0ae7785fdb3fa7e44434908d832cbc7a9a1e096d046ac6fe953812c52c/mlx-0.32.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:deb284f3a5cd0c3e87bed80c2bee9dcbf946bdad44d75592f6fb784da878c1c0", size = 558799, upload-time = "2026-07-07T17:55:56.844Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a8/7bc999ce5d09dfac8961dcda4ed47e173fca2857492f34599b237380f20d/mlx-0.32.0-cp313-cp313-macosx_26_0_arm64.whl", hash = "sha256:4192a2d02014a13a6a1030bf13dfb4e4fe05ec3ffa47678ee37da29111e25cb1", size = 558786, upload-time = "2026-07-07T17:55:58.272Z" }, + { url = "https://files.pythonhosted.org/packages/aa/90/5e49c183f9024f86dc1f038866ccd743aaec1fe412a7e7dc76fec9271f48/mlx-0.32.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2ee79b1f8c2c2a329afc95ece7dce0be798d43f3de771a6370d2b9f9702bbd9a", size = 561387, upload-time = "2026-07-07T17:56:05.177Z" }, + { url = "https://files.pythonhosted.org/packages/dc/33/e3d7f7a18331523762e9968b6716e81514649af81ffd9ba4364e3df95823/mlx-0.32.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:e0db558267bb2d13fac4f85674456adbe0f085c570b9219e03d4e95fdc11c4d0", size = 561389, upload-time = "2026-07-07T17:56:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/bd847d3fed65296573a4bb3399adde6934c0a718813b5636000d7d1b4063/mlx-0.32.0-cp314-cp314-macosx_26_0_arm64.whl", hash = "sha256:23e83c8e74a23156696e9f9905d16a17b7d27b5a596c1bc0f720a98df1c5aadf", size = 561392, upload-time = "2026-07-07T17:56:08.237Z" }, +] + +[[package]] +name = "mlx-lm" +version = "0.31.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "protobuf", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "pyyaml", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "sentencepiece", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "transformers", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/94/9a38d6b0c6fcca995b9136c94eb7da1e9c5165652edf228b96b29960fa7a/mlx_lm-0.31.3.tar.gz", hash = "sha256:61eb0e3ba09444f77f874aff295401d7ccd20b39495cbbce0c782a15474ce733", size = 304318, upload-time = "2026-04-22T07:37:27.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/02/9a67b8e4f87e3e2e5cd7b1ad79304b93c09a0db6af34bee75e6551c06c60/mlx_lm-0.31.3-py3-none-any.whl", hash = "sha256:758cfddf1180053b7613db76fad3d246a331a2a905808e1164a275621fc983b8", size = 408890, upload-time = "2026-04-22T07:37:25.965Z" }, +] + +[[package]] +name = "mlx-metal" +version = "0.32.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/ef/d74ae99cfe9ddb59fd08abd14f47754c3d199291a93756903fa595b31b8a/mlx_metal-0.32.0-py3-none-macosx_14_0_arm64.whl", hash = "sha256:5b64b20ac24b0c401f489de01e8209edc4d372125201f19314e6f39e385322aa", size = 40824649, upload-time = "2026-07-07T17:55:20.7Z" }, + { url = "https://files.pythonhosted.org/packages/d5/38/b96a3de98cfbb009592b3870af46ff63657b192f8d1e06c6c1faea5fbef3/mlx_metal-0.32.0-py3-none-macosx_15_0_arm64.whl", hash = "sha256:1bd94a1ce5b03a0c898771a3e759f0124300c6ab5155127906a1d50b1f3fcf19", size = 40818869, upload-time = "2026-07-07T17:55:25.059Z" }, + { url = "https://files.pythonhosted.org/packages/dc/59/65d32520175379df33f107749193aa94ea9db069167a36a1a100ff689f62/mlx_metal-0.32.0-py3-none-macosx_26_0_arm64.whl", hash = "sha256:3af76a498d84804f66119800499f9d143d7dffb0878a0dd0d7c2846e58565fd7", size = 56511379, upload-time = "2026-07-07T17:55:36.045Z" }, +] + [[package]] name = "more-itertools" version = "11.1.0" @@ -2586,6 +2641,10 @@ ci = [ { name = "pip-licenses" }, { name = "twine" }, ] +mlx = [ + { name = "mlx", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, + { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, +] quality = [ { name = "mutmut" }, ] @@ -2623,6 +2682,10 @@ ci = [ { name = "pip-licenses", specifier = "==5.5.5" }, { name = "twine", specifier = "==7.0.0" }, ] +mlx = [ + { name = "mlx", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'", specifier = "==0.32.0" }, + { name = "mlx-lm", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'", specifier = "==0.31.3" }, +] quality = [{ name = "mutmut", specifier = "==3.7.0" }] [[package]] @@ -3043,6 +3106,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + [[package]] name = "psutil" version = "7.2.2" @@ -4039,6 +4112,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552, upload-time = "2022-05-26T13:35:21.206Z" }, ] +[[package]] +name = "sentencepiece" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/1b/e6c69e4c2026ed575d68dda2847a404468ca7b5fa684bb0b19f71d82d29d/sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e", size = 2180607, upload-time = "2026-07-12T08:38:01.018Z" }, + { url = "https://files.pythonhosted.org/packages/1b/39/3d43a75dd5a22503ca5074d0d37707cabb2e4a71b4bc6e6c61be3643cc7a/sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914", size = 1345667, upload-time = "2026-07-12T08:38:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/f23a2efaa0210b883574001b88fa64e499f798f0848a0b610fb9b384d162/sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0", size = 2184255, upload-time = "2026-07-12T08:38:14.855Z" }, + { url = "https://files.pythonhosted.org/packages/2a/92/3a6ea4a2c6dd9e7062698a5a33534ca0e20844883338ae9c6b9c122c1a9f/sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e", size = 1346997, upload-time = "2026-07-12T08:38:18.499Z" }, + { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, + { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/b3b05095c174d6e80d37d5ddc2f57c2c56237333e7bbd6079cf3243c2a8a/sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8", size = 2188346, upload-time = "2026-07-12T08:38:41.089Z" }, + { url = "https://files.pythonhosted.org/packages/34/db/f9ea1a6844b4fa5dfe2312095cd866a1f724cd0905054ab9d5991778ba50/sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a", size = 1347267, upload-time = "2026-07-12T08:38:44.389Z" }, + { url = "https://files.pythonhosted.org/packages/58/9c/dfc82846460e7a712310f5613f23d8b553cabb4e2e648663c11d8382af56/sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78", size = 2223080, upload-time = "2026-07-12T08:38:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/59/5a/16d51d05360be4cee3ebfe4837c184054c4eed16cabaeb3b039524e9a000/sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5", size = 1361138, upload-time = "2026-07-12T08:38:58.808Z" }, + { url = "https://files.pythonhosted.org/packages/0b/7e/f5df63edb6bcb46c1343cfa5d9192d73a4eb61af2e800d9402efff387523/sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a", size = 2190240, upload-time = "2026-07-12T08:39:08.178Z" }, + { url = "https://files.pythonhosted.org/packages/d1/18/823954c9c90e74eba09fb96752dc37a5555df00d69866cb9406d1725dc7e/sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906", size = 1348056, upload-time = "2026-07-12T08:39:11.744Z" }, + { url = "https://files.pythonhosted.org/packages/84/c4/7afe8c2315b76e46818851a057e50a378a0382aa00b970a1fa444181b6f6/sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151", size = 2223281, upload-time = "2026-07-12T08:39:20.978Z" }, + { url = "https://files.pythonhosted.org/packages/78/52/ffe402b13bce1889228a98dc6cd86ae8afac1112362236be3468be784441/sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b", size = 1361736, upload-time = "2026-07-12T08:39:24.602Z" }, +] + [[package]] name = "setproctitle" version = "1.3.7"