From 985c9e93630e58c530440c11f89bdcfcbefd5336 Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:43:53 -0400 Subject: [PATCH] feat: add safe distributed checkpoint intake and preflight --- .gitignore | 1 + README.md | 105 +- ci/pr-test-policy.json | 8 +- ci/test-quality-policy.json | 12 +- ci/test-risk-map.json | 193 +- .../distributed-checkpoint-intake.md | 189 ++ docs/checkpoints/references.md | 76 + .../schemas/adapter-capability-v1.schema.json | 148 ++ .../artifact-provenance-v1.schema.json | 145 ++ .../checkpoint-descriptor-v1.schema.json | 541 +++++ .../schemas/checkpoint-error-codes-v1.json | 27 + .../checkpoint-error-registry-v1.schema.json | 28 + .../checkpoint-trust-policy-v1.schema.json | 134 ++ .../conversion-manifest-v1.schema.json | 167 ++ ...dapter-claims-tested-without-evidence.json | 21 + .../v1/invalid/generic-offset-field.json | 6 + .../trust-policy-leaks-environment.json | 5 + .../trusted-worker-error-raw-message.json | 10 + .../v1/valid/deferred-adapter-capability.json | 34 + .../v1/valid/trusted-metadata-policy.json | 15 + .../v1/valid/trusted-worker-complete.json | 10 + .../v1/valid/typed-offset-namespaces.json | 144 ++ .../weights-only-conversion-manifest.json | 77 + .../peft-adapter-manifest-v1.schema.json | 61 + .../schemas/support-matrix-v1.schema.json | 95 + .../trusted-worker-message-v1.schema.json | 81 + docs/checkpoints/support-matrix-v1.json | 237 +++ docs/checkpoints/support-runbook.md | 203 ++ docs/distributed-preflight.md | 182 ++ obliteratus/__init__.py | 4 + obliteratus/abliterate.py | 16 +- obliteratus/checkpoint_capabilities.py | 334 +++ obliteratus/checkpoint_errors.py | 176 ++ obliteratus/checkpoint_fixtures.py | 144 ++ obliteratus/checkpoint_fragments.py | 622 ++++++ obliteratus/checkpoint_inspection.py | 1127 ++++++++++ obliteratus/checkpoint_provenance.py | 879 ++++++++ obliteratus/checkpoint_service.py | 29 + obliteratus/checkpoint_writer.py | 1027 +++++++++ obliteratus/cli.py | 180 +- obliteratus/distributed/__init__.py | 46 + obliteratus/distributed/config.py | 498 +++++ obliteratus/distributed/consensus.py | 137 ++ obliteratus/distributed/contracts.py | 481 +++++ obliteratus/distributed/evidence.py | 299 +++ obliteratus/distributed/launcher.py | 375 ++++ obliteratus/distributed/numerical.py | 252 +++ obliteratus/distributed/preflight.py | 1829 +++++++++++++++++ obliteratus/lora_ablation.py | 735 ++++++- obliteratus/run_archive.py | 31 + pyproject.toml | 4 + scripts/check_checkpoint_docs.py | 391 ++++ scripts/generate_checkpoint_fixtures.py | 472 +++++ .../v1/cases/mixed-model-peft/case.json | 161 ++ .../mixed-model-peft/oracles.safetensors | Bin 0 -> 168 bytes .../mixed-model-peft/rank-00000.safetensors | Bin 0 -> 96 bytes .../mixed-model-peft/rank-00001.safetensors | Bin 0 -> 88 bytes .../v1/cases/tp2-pp2-to-single/case.json | 310 +++ .../tp2-pp2-to-single/oracles.safetensors | Bin 0 -> 144 bytes .../tp2-pp2-to-single/rank-00000.safetensors | Bin 0 -> 88 bytes .../tp2-pp2-to-single/rank-00001.safetensors | Bin 0 -> 88 bytes .../tp2-pp2-to-single/rank-00002.safetensors | Bin 0 -> 88 bytes .../tp2-pp2-to-single/rank-00003.safetensors | Bin 0 -> 88 bytes .../v1/cases/world1-complete/case.json | 393 ++++ .../cases/world1-complete/oracles.safetensors | Bin 0 -> 488 bytes .../world1-complete/rank-00000.safetensors | Bin 0 -> 472 bytes .../v1/cases/world2-uneven-1d/case.json | 150 ++ .../world2-uneven-1d/oracles.safetensors | Bin 0 -> 128 bytes .../world2-uneven-1d/rank-00000.safetensors | Bin 0 -> 104 bytes .../world2-uneven-1d/rank-00001.safetensors | Bin 0 -> 104 bytes .../v1/cases/world4-dp-replicas/case.json | 255 +++ .../world4-dp-replicas/oracles.safetensors | Bin 0 -> 84 bytes .../world4-dp-replicas/rank-00000.safetensors | Bin 0 -> 84 bytes .../world4-dp-replicas/rank-00001.safetensors | Bin 0 -> 84 bytes .../world4-dp-replicas/rank-00002.safetensors | Bin 0 -> 84 bytes .../world4-dp-replicas/rank-00003.safetensors | Bin 0 -> 84 bytes .../v1/cases/world4-uneven-2d/case.json | 309 +++ .../world4-uneven-2d/oracles.safetensors | Bin 0 -> 220 bytes .../world4-uneven-2d/rank-00000.safetensors | Bin 0 -> 96 bytes .../world4-uneven-2d/rank-00001.safetensors | Bin 0 -> 104 bytes .../world4-uneven-2d/rank-00002.safetensors | Bin 0 -> 108 bytes .../world4-uneven-2d/rank-00003.safetensors | Bin 0 -> 120 bytes .../v1/fixture-corpus.json | 233 +++ .../v1/negative-cases.json | 108 + tests/test_checkpoint_capabilities.py | 303 +++ tests/test_checkpoint_contract_schemas.py | 101 + tests/test_checkpoint_docs_contracts.py | 108 + tests/test_checkpoint_errors.py | 38 + tests/test_checkpoint_evaluation.py | 218 +- tests/test_checkpoint_fixture_corpus.py | 207 ++ tests/test_checkpoint_fragments.py | 536 +++++ tests/test_checkpoint_inspection.py | 753 +++++++ tests/test_checkpoint_provenance.py | 524 +++++ tests/test_checkpoint_service.py | 29 + tests/test_checkpoint_writer.py | 779 +++++++ tests/test_cli.py | 52 + tests/test_cli_boundaries.py | 70 + tests/test_distributed_contracts.py | 425 ++++ tests/test_distributed_evidence.py | 113 + tests/test_distributed_gloo.py | 871 ++++++++ tests/test_distributed_launcher.py | 392 ++++ tests/test_distributed_preflight.py | 383 ++++ tests/test_package_export_contracts.py | 1 + tests/test_peft_artifacts.py | 374 ++++ tests/test_persistence_pipeline.py | 15 +- tests/test_quant_dequant.py | 14 + tests/test_run_archive.py | 246 ++- uv.lock | 304 +++ 108 files changed, 21726 insertions(+), 92 deletions(-) create mode 100644 docs/checkpoints/distributed-checkpoint-intake.md create mode 100644 docs/checkpoints/references.md create mode 100644 docs/checkpoints/schemas/adapter-capability-v1.schema.json create mode 100644 docs/checkpoints/schemas/artifact-provenance-v1.schema.json create mode 100644 docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json create mode 100644 docs/checkpoints/schemas/checkpoint-error-codes-v1.json create mode 100644 docs/checkpoints/schemas/checkpoint-error-registry-v1.schema.json create mode 100644 docs/checkpoints/schemas/checkpoint-trust-policy-v1.schema.json create mode 100644 docs/checkpoints/schemas/conversion-manifest-v1.schema.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/invalid/adapter-claims-tested-without-evidence.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/invalid/generic-offset-field.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/invalid/trust-policy-leaks-environment.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/invalid/trusted-worker-error-raw-message.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/valid/deferred-adapter-capability.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/valid/trusted-metadata-policy.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/valid/trusted-worker-complete.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/valid/typed-offset-namespaces.json create mode 100644 docs/checkpoints/schemas/fixtures/v1/valid/weights-only-conversion-manifest.json create mode 100644 docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json create mode 100644 docs/checkpoints/schemas/support-matrix-v1.schema.json create mode 100644 docs/checkpoints/schemas/trusted-worker-message-v1.schema.json create mode 100644 docs/checkpoints/support-matrix-v1.json create mode 100644 docs/checkpoints/support-runbook.md create mode 100644 docs/distributed-preflight.md create mode 100644 obliteratus/checkpoint_capabilities.py create mode 100644 obliteratus/checkpoint_errors.py create mode 100644 obliteratus/checkpoint_fixtures.py create mode 100644 obliteratus/checkpoint_fragments.py create mode 100644 obliteratus/checkpoint_inspection.py create mode 100644 obliteratus/checkpoint_provenance.py create mode 100644 obliteratus/checkpoint_service.py create mode 100644 obliteratus/checkpoint_writer.py create mode 100644 obliteratus/distributed/__init__.py create mode 100644 obliteratus/distributed/config.py create mode 100644 obliteratus/distributed/consensus.py create mode 100644 obliteratus/distributed/contracts.py create mode 100644 obliteratus/distributed/evidence.py create mode 100644 obliteratus/distributed/launcher.py create mode 100644 obliteratus/distributed/numerical.py create mode 100644 obliteratus/distributed/preflight.py create mode 100644 scripts/check_checkpoint_docs.py create mode 100644 scripts/generate_checkpoint_fixtures.py create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/rank-00001.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00001.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00002.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00003.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00001.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00001.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00002.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00003.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/case.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/oracles.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00000.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00001.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00002.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00003.safetensors create mode 100644 tests/fixtures/distributed_checkpoints/v1/fixture-corpus.json create mode 100644 tests/fixtures/distributed_checkpoints/v1/negative-cases.json create mode 100644 tests/test_checkpoint_capabilities.py create mode 100644 tests/test_checkpoint_contract_schemas.py create mode 100644 tests/test_checkpoint_docs_contracts.py create mode 100644 tests/test_checkpoint_errors.py create mode 100644 tests/test_checkpoint_fixture_corpus.py create mode 100644 tests/test_checkpoint_fragments.py create mode 100644 tests/test_checkpoint_inspection.py create mode 100644 tests/test_checkpoint_provenance.py create mode 100644 tests/test_checkpoint_service.py create mode 100644 tests/test_checkpoint_writer.py create mode 100644 tests/test_distributed_contracts.py create mode 100644 tests/test_distributed_evidence.py create mode 100644 tests/test_distributed_gloo.py create mode 100644 tests/test_distributed_launcher.py create mode 100644 tests/test_distributed_preflight.py create mode 100644 tests/test_peft_artifacts.py diff --git a/.gitignore b/.gitignore index db7a058..1e03ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ env/ *.pt *.bin *.safetensors +!tests/fixtures/distributed_checkpoints/**/*.safetensors wandb/ outputs/ results/ diff --git a/README.md b/README.md index cb3a7f7..c1077f5 100644 --- a/README.md +++ b/README.md @@ -501,7 +501,7 @@ OBLITERATUS ships with presets for 116 models organized by compute requirement: | **Small** | 4-8 GB | Phi-2 2.7B, Gemma-2 2B, StableLM-2 1.6B | | **Medium** | 8-16 GB | Mistral 7B, Qwen2.5-7B, Gemma-2 9B, Phi-3.5 | | **Large** | 24+ GB | LLaMA-3.1 8B, Qwen2.5-14B, Mistral 24B, DeepSeek-R1 distills | -| **Frontier** | Multi-GPU | DeepSeek-V3.2 685B, Qwen3-235B, GLM-4.7 355B | +| **Frontier** | Multi-device candidates | DeepSeek-V3.2 685B, Qwen3-235B, GLM-4.7 355B | Includes pre-liberated variants (Dolphin, Hermes, WhiteRabbitNeo) for A/B comparison against their chained counterparts. @@ -509,15 +509,56 @@ Includes pre-liberated variants (Dolphin, Hermes, WhiteRabbitNeo) for A/B compar obliteratus models ``` -## Multi-GPU and remote execution +## Single-host multi-device placement and one-host remote execution -OBLITERATUS automatically shards models across multiple GPUs when they don't fit on a single card. It also supports remote execution over SSH, so you can run the pipeline on a GPU server from your laptop. +For compatible Hugging Face models, OBLITERATUS can ask Accelerate to place complete +model modules across multiple GPUs in one process when a model does not fit on one +card. It can also run that one process on one remote SSH host. These features do not +implement rank-partitioned checkpoint intake, distributed collectives, or a +multi-node launcher. See the [checkpoint intake guide](docs/checkpoints/distributed-checkpoint-intake.md) +and [support matrix](docs/checkpoints/support-matrix-v1.json) for the precise boundary. -### How model sharding works +The offline structural inspector can classify checkpoint layout without loading +tensor payloads or invoking producer frameworks: -When you have multiple GPUs, OBLITERATUS uses accelerate's `device_map="auto"` to split the model's layers across all available GPUs. This is **naive pipeline parallelism** — layers are distributed evenly, but only one GPU computes at a time as activations flow sequentially through the layer stack. The other GPUs hold their assigned layers in memory but are idle until their turn. +```bash +obliteratus checkpoint inspect ./checkpoint --json +``` -This means multi-GPU sharding is a **memory solution, not a speed solution**. It lets you run models that don't fit on one GPU, but it won't make small models run faster. In fact, more GPUs can be *slower* due to inter-GPU data transfer overhead at layer boundaries. +This is an evidence and refusal interface, not DCP, Megatron, or DeepSpeed import +support. Producer-backed conversion and multi-host execution remain unavailable. + +The source tree also contains an explicit fixed-membership preflight plus +CPU/Gloo contract and numerical oracles for the planned multi-node runtime. An +external trusted scheduler must start every worker with fixed `torchrun` +membership and `max_restarts=0`; each worker then invokes: + +```bash +obliteratus distributed preflight /absolute/path/profile.json --json +``` + +This command validates the closed local profile, private allowlisted endpoint, +rank and device inventory, exact input/software identities, resource floors, +and shared staging before any model allocation. It never launches peers, uses +SSH, installs software, distributes credentials, loads a model, or activates +from environment variables alone. See the +[distributed preflight contract](docs/distributed-preflight.md). + +The preflight and CPU/Gloo tests prove protocol behavior and placement-aware +arithmetic only. They are not a distributed model loader, GPU/NCCL validation, +physical multi-host evidence, an authenticated or encrypted transport, or a +supported multi-node workflow. + +### How process-local module placement works + +When you have multiple GPUs, OBLITERATUS uses Accelerate's `device_map="auto"` to +place complete model modules across available devices in one process. This is a +capacity-oriented placement mechanism, not rank-based pipeline parallelism: it does +not load rank-local tensor partitions or reconstruct a distributed checkpoint. + +This multi-device placement is a **memory solution, not a speed solution**. It lets +compatible models use aggregate device capacity, but it will not make small models +run faster. More devices can be slower because activations cross device boundaries. ### Offloaded layers and fused MoE experts @@ -538,7 +579,8 @@ obliteratus obliterate bigmodel/200B --gpus 0,1,2,3 obliteratus obliterate meta-llama/Llama-3.1-70B-Instruct --gpus 2,5 ``` -This sets `CUDA_VISIBLE_DEVICES` before CUDA initializes. The model is then sharded across the selected GPUs. +This sets `CUDA_VISIBLE_DEVICES` before CUDA initializes. Accelerate then selects a +process-local placement across the visible devices. ### Precision and quantization @@ -576,8 +618,8 @@ pip install -e ".[quantization]" |-------|-----------|---------|----------|-----------| | `float32` | 4 | 28 GB | 280 GB | 1620 GB | | `float16` / `bfloat16` | 2 | 14 GB | 140 GB | 810 GB | -| `int8` (via `--quantization bitsandbytes-8bit`) | 1 | 7 GB | 70 GB | 405 GB | -| `int4` (via `--quantization bitsandbytes-4bit`) | 0.5 | 3.5 GB | 35 GB | 203 GB | +| `int8` (via `--quantization 8bit`) | 1 | 7 GB | 70 GB | 405 GB | +| `int4` (via `--quantization 4bit`) | 0.5 | 3.5 GB | 35 GB | 203 GB | ```bash # Default: bfloat16 @@ -585,16 +627,20 @@ obliteratus obliterate meta-llama/Llama-3.1-70B-Instruct # 8-bit quantization — fits on fewer GPUs obliteratus obliterate meta-llama/Llama-3.1-70B-Instruct \ - --quantization bitsandbytes-8bit + --quantization 8bit # 4-bit quantization — Llama-405B on 4x A100-80GB obliteratus obliterate meta-llama/Llama-3.1-405B-Instruct \ - --quantization bitsandbytes-4bit --dtype float16 + --quantization 4bit --dtype float16 ``` Quantization roughly halves the GPU count at each step down. A 70B model that needs 3x A100-80GB in bf16 fits on 2 in int8 or 1 in int4. -**FP8 and NVFP4 checkpoints are supported automatically.** No flag needed — the loader detects the format from the checkpoint's `quantization_config`, dequantizes the weights to float (BF16 by default) shard-by-shard, runs the normal pipeline, and saves the output as plain BF16: +**Recognized FP8 and NVFP4 layouts are handled automatically for compatible models.** +No flag is needed for the detected layouts below: the loader dequantizes one file +shard at a time, runs the normal pipeline, and saves plain BF16 weights. This file +sharding is a memory-management technique; it is not support for rank-partitioned +or distributed checkpoint formats. | Format | Schemes detected | |--------|------------------| @@ -648,7 +694,7 @@ Stage breakdown (approximately constant across GPU counts): | Stage | GPT-OSS-120B | DeepSeek-70B | Bottleneck | |-------|-------------|-------------|-----------| | SUMMON (load) | ~11s | ~24s | Disk I/O (model cached locally) | -| PROBE (activations) | ~20s | ~20s | Forward passes through sharded model | +| PROBE (activations) | ~20s | ~20s | Forward passes through a multi-device model | | DISTILL + EXCISE | ~30s | ~30s | SVD + weight projection (CPU-bound) | | VERIFY | ~210s | ~270s | Forward passes on validation prompts | | REBIRTH (save) | ~350s | ~194s | Writing model to disk (234 GB vs 141 GB) | @@ -664,14 +710,13 @@ Key findings: For models that fit on a single GPU with room to spare, the PROBE stage (which runs 1024 forward passes to collect activations) is the main computational bottleneck. Pipeline parallelism doesn't help here — it still processes one prompt at a time through the full layer stack. -True data parallelism (replicating the model and splitting prompts across GPUs) can speed up PROBE, but it requires enough VRAM to hold a full copy of the model on each GPU. An experimental pre-replicated data parallel implementation is available on the `data-parallel-prereplication` branch: +True data parallelism (replicating the model and splitting prompts across GPUs) can +speed up PROBE, but it requires enough VRAM to hold a full copy of the model on each +GPU. The released CLI does not currently provide a supported data-parallel runtime. +The following historical experiment remains useful as research evidence, not as a +branch or command recommendation: -```bash -git checkout data-parallel-prereplication -obliteratus obliterate EleutherAI/pythia-12b --data-parallel -``` - -This deep-copies the model to each GPU once, then distributes prompt batches across replicas using a thread pool. Benchmarks on Pythia 12B (24 GB model, 8x A100-80GB): +Benchmarks on Pythia 12B (24 GB model, 8x A100-80GB): | Mode | PROBE time | Notes | |------|-----------|-------| @@ -682,7 +727,10 @@ Data parallelism becomes more valuable as the prompt count or model size increas ### Remote execution over SSH -Run the full pipeline on a remote GPU node from your local machine. OBLITERATUS handles SSH connection, auto-installs itself on the remote if needed, streams logs in real time, and copies results back when done. +Run the full pipeline as one process on one remote GPU host from your local machine. +OBLITERATUS handles the SSH connection, installs itself on that host if needed, +streams logs in real time, and copies results back when done. It does not coordinate +multiple hosts or provide rendezvous, rank assignment, or distributed collectives. ```bash # Basic remote run @@ -711,7 +759,7 @@ model: remote: host: gpu-node - user: root + user: obliteratus ssh_key: ~/.ssh/id_rsa remote_dir: /tmp/obliteratus_run gpus: "0,1,2,3" # select GPUs on the remote @@ -736,10 +784,10 @@ The remote runner: | Scenario | Recommendation | |----------|---------------| | Model fits on 1 GPU | Use 1 GPU. Adding more won't help and may slow things down. | -| Model almost fits on 1 GPU | Try `--quantization bitsandbytes-8bit` or `bitsandbytes-4bit` to reduce memory. Halving precision roughly halves VRAM. | -| Model fits on 1 GPU, PROBE is slow (many prompts) | Try `data-parallel-prereplication` branch. Only helps if model fits on each GPU with room for activations. | +| Model almost fits on 1 GPU | Try `--quantization 8bit` or `4bit` to reduce memory. Confirm the selected architecture and operation are compatible with quantized loading. | +| Model fits on 1 GPU, PROBE is slow (many prompts) | The released CLI has no supported data-parallel mode; increase batching conservatively and profile the workload. | | Model doesn't fit on 1 GPU | Use `--gpus` with the **minimum** number of GPUs that fits. Run `obliteratus gpu-calc` to find that number. | -| Model needs 4+ GPUs | Pipeline parallel via `device_map="auto"` is the only option. Expect I/O-dominated runtimes for very large models. Consider quantization first — int4 can cut the GPU count by 4x. | +| Model needs 4+ GPUs | For compatible models, process-local `device_map="auto"` may provide capacity. Verify memory headroom and architecture-specific restrictions; this is not distributed checkpoint support. | | Not sure how many GPUs you need | Run `obliteratus gpu-calc --gpu-mem ` for an estimate. | | No local GPUs | Use `--remote user@gpu-node` to run on a remote machine, or use HuggingFace Spaces / Colab. | @@ -849,7 +897,12 @@ Open `docs/index.html` in your browser for a visual interface with: ## Architecture support -Works with any HuggingFace transformer, including: GPT-2, LLaMA, Mistral, Falcon, OPT, BLOOM, Phi, Qwen, Gemma, StableLM, and more. Handles both Conv1D and Linear projections, standard and fused attention, and custom architectures via `trust_remote_code`. +Architecture compatibility is conditional on the model family, projection layout, +loader path, requested operation, and test evidence. The registry includes GPT-2, +LLaMA, Mistral, Falcon, OPT, BLOOM, Phi, Qwen, Gemma, StableLM, and other candidates, +but registry inclusion alone is not proof of support. Consult the +[support matrix](docs/checkpoints/support-matrix-v1.json), and treat custom code +loaded with `trust_remote_code` as an additional trust boundary. ## References diff --git a/ci/pr-test-policy.json b/ci/pr-test-policy.json index 7077141..c780f11 100644 --- a/ci/pr-test-policy.json +++ b/ci/pr-test-policy.json @@ -12,16 +12,22 @@ ".aiwg/**", ".github/workflows/**", "ci/**", + "docs/checkpoints/**", "scripts/check_*.py", + "scripts/generate_checkpoint_fixtures.py", "scripts/jetson_*.py", "scripts/select_pr_tests.py", "scripts/setup_jetson.py", "pyproject.toml", - "uv.lock" + "uv.lock", + "tests/fixtures/distributed_checkpoints/**" ], "infrastructure_tests": [ "tests/test_aiwg_workspace_contracts.py", "tests/test_ci_policy.py", + "tests/test_checkpoint_docs_contracts.py", + "tests/test_checkpoint_contract_schemas.py", + "tests/test_checkpoint_fixture_corpus.py", "tests/test_conditional_gate_scripts.py", "tests/test_jetson_support_tooling.py", "tests/test_pr_test_selection.py", diff --git a/ci/test-quality-policy.json b/ci/test-quality-policy.json index a22fa28..30b1cd7 100644 --- a/ci/test-quality-policy.json +++ b/ci/test-quality-policy.json @@ -22,7 +22,15 @@ "obliteratus/evaluation/advanced_metrics.py", "obliteratus/reporting/report.py", "obliteratus/community.py", - "obliteratus/telemetry.py" + "obliteratus/telemetry.py", + "obliteratus/checkpoint_errors.py", + "obliteratus/checkpoint_capabilities.py", + "obliteratus/checkpoint_fixtures.py", + "obliteratus/checkpoint_fragments.py", + "obliteratus/checkpoint_inspection.py", + "obliteratus/checkpoint_provenance.py", + "obliteratus/checkpoint_service.py", + "obliteratus/checkpoint_writer.py" ], "mature_cpu_scope": { "description": "All source modules except boundaries that intrinsically require a real model runtime, an external service, an interactive UI, or remote/hardware execution.", @@ -86,7 +94,7 @@ { "path": "obliteratus/lora_ablation.py", "boundary": "model-runtime", - "rationale": "Adapter construction and validation require real transformer projections and optional PEFT integration.", + "rationale": "Safe artifact serialization is covered by mandatory CPU tests; the remaining adapter construction and live validation paths require real transformer projections and optional PEFT integration.", "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", "conditional_gate": "model-download-runtime" }, diff --git a/ci/test-risk-map.json b/ci/test-risk-map.json index a2af53f..3fa6ec6 100644 --- a/ci/test-risk-map.json +++ b/ci/test-risk-map.json @@ -92,6 +92,7 @@ "tests/test_offload_surgery.py", "tests/test_persistence_contracts.py", "tests/test_persistence_pipeline.py", + "tests/test_peft_artifacts.py", "tests/test_telemetry.py", "tests/conditional/test_cuda_runtime.py", "tests/conditional/test_jetson_runtime.py" @@ -114,6 +115,38 @@ "tests/test_persistence_contracts.py" ] }, + { + "id": "checkpoint-common-safe-plane", + "owner": "checkpoint intake maintainers", + "description": "Producer-neutral structural inspection, fragment validation, deterministic fixtures, provenance, and transactional canonical writing", + "contract_types": [ + "persistence", + "public-interface", + "reproducibility", + "research-input" + ], + "paths": [ + "obliteratus/checkpoint_capabilities.py", + "obliteratus/checkpoint_errors.py", + "obliteratus/checkpoint_fixtures.py", + "obliteratus/checkpoint_fragments.py", + "obliteratus/checkpoint_inspection.py", + "obliteratus/checkpoint_provenance.py", + "obliteratus/checkpoint_service.py", + "obliteratus/checkpoint_writer.py" + ], + "required_tests": [ + "tests/test_checkpoint_capabilities.py", + "tests/test_checkpoint_contract_schemas.py", + "tests/test_checkpoint_errors.py", + "tests/test_checkpoint_fixture_corpus.py", + "tests/test_checkpoint_fragments.py", + "tests/test_checkpoint_inspection.py", + "tests/test_checkpoint_provenance.py", + "tests/test_checkpoint_service.py", + "tests/test_checkpoint_writer.py" + ] + }, { "id": "configuration-and-selection", "owner": "runtime compatibility maintainers", @@ -154,6 +187,35 @@ "tests/conditional/test_jetson_runtime.py" ] }, + { + "id": "distributed-runtime-contracts", + "owner": "distributed runtime maintainers", + "description": "Strict fixed-membership launch, preflight, evidence, bounded consensus, lifecycle, and placement-aware numerical contracts for the opt-in distributed prototype", + "contract_types": [ + "model-runtime", + "orchestration", + "reproducibility", + "numerical-invariant" + ], + "paths": [ + "obliteratus/distributed/__init__.py", + "obliteratus/distributed/config.py", + "obliteratus/distributed/contracts.py", + "obliteratus/distributed/consensus.py", + "obliteratus/distributed/evidence.py", + "obliteratus/distributed/launcher.py", + "obliteratus/distributed/preflight.py", + "obliteratus/distributed/numerical.py" + ], + "required_tests": [ + "tests/test_distributed_contracts.py", + "tests/test_distributed_evidence.py", + "tests/test_distributed_gloo.py", + "tests/test_distributed_launcher.py", + "tests/test_distributed_preflight.py", + "tests/test_cli.py" + ] + }, { "id": "credential-resolution", "owner": "runtime security maintainers", @@ -197,6 +259,8 @@ "required_tests": [ "tests/test_cli.py", "tests/test_cli_boundaries.py", + "tests/test_checkpoint_service.py", + "tests/test_checkpoint_inspection.py", "tests/test_interactive_contracts.py", "tests/test_local_ui_contracts.py", "tests/test_local_ui_portability.py", @@ -375,13 +439,138 @@ } ], "modules": [ + { + "path": "obliteratus/checkpoint_capabilities.py", + "risk_class": "cpu-contract", + "risk": "closed capability resolution and exact optional-dependency diagnostics without imports or adapter execution", + "required_tests": [ + "tests/test_checkpoint_capabilities.py", + "tests/test_checkpoint_inspection.py", + "tests/test_checkpoint_service.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_errors.py", + "risk_class": "cpu-contract", + "risk": "stable fail-closed error vocabulary and sanitized public diagnostics", + "required_tests": [ + "tests/test_checkpoint_errors.py", + "tests/test_checkpoint_contract_schemas.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_fixtures.py", + "risk_class": "cpu-contract", + "risk": "bounded loading of deterministic project-owned JSON and safetensors fixtures", + "required_tests": [ + "tests/test_checkpoint_fixture_corpus.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_fragments.py", + "risk_class": "cpu-contract", + "risk": "checked geometry, exact coverage, replica agreement, ties, padding, and reconstruction", + "required_tests": [ + "tests/test_checkpoint_fragments.py", + "tests/test_checkpoint_fixture_corpus.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_inspection.py", + "risk_class": "cpu-contract", + "risk": "untrusted local path inventory, bounded inert parsing, classification, and no-reader safety", + "required_tests": [ + "tests/test_checkpoint_inspection.py", + "tests/test_checkpoint_contract_schemas.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_provenance.py", + "risk_class": "cpu-contract", + "risk": "content-addressed lineage, redaction, state truth, and cross-artifact identity", + "required_tests": [ + "tests/test_checkpoint_provenance.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_service.py", + "risk_class": "cpu-contract", + "risk": "public safe-plane API boundary without trusted reader or adapter entrypoints", + "required_tests": [ + "tests/test_checkpoint_service.py", + "tests/test_checkpoint_inspection.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/checkpoint_writer.py", + "risk_class": "cpu-contract", + "risk": "resource admission, deterministic safetensors, independent reload, atomic promotion, and rollback", + "required_tests": [ + "tests/test_checkpoint_writer.py", + "tests/test_checkpoint_atomicity.py", + "tests/test_persistence_contracts.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/distributed/config.py", + "risk_class": "cpu-contract", + "risk": "closed bounded profile parsing, exact policy, endpoint allowlist, and immutable attempt identity", + "required_tests": [ + "tests/test_distributed_launcher.py", + "tests/test_distributed_preflight.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/distributed/evidence.py", + "risk_class": "cpu-contract", + "risk": "allowlist-built redacted evidence, distinct lifecycle receipts, private atomic persistence, and quarantine semantics", + "required_tests": [ + "tests/test_distributed_evidence.py", + "tests/test_distributed_gloo.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/distributed/launcher.py", + "risk_class": "cpu-contract", + "risk": "fixed torchrun membership, pre-connect interface observation, fd-level diagnostic containment, zero restarts, and process-group teardown", + "required_tests": [ + "tests/test_distributed_launcher.py", + "tests/test_cli.py", + "tests/test_distributed_gloo.py" + ], + "conditional_gates": [] + }, + { + "path": "obliteratus/distributed/preflight.py", + "risk_class": "cpu-contract", + "risk": "pre-allocation safetensors, executable-code, topology, observed mount/interface, lifecycle, evidence, and resource admission", + "required_tests": [ + "tests/test_distributed_preflight.py", + "tests/test_distributed_gloo.py", + "tests/test_distributed_evidence.py" + ], + "conditional_gates": [] + }, { "path": "obliteratus/cli.py", "risk_class": "cpu-contract", "risk": "public parsing, validation, dispatch, and local/remote option propagation", "required_tests": [ "tests/test_cli.py", - "tests/test_cli_boundaries.py" + "tests/test_cli_boundaries.py", + "tests/test_distributed_launcher.py", + "tests/test_checkpoint_inspection.py", + "tests/test_checkpoint_service.py" ], "conditional_gates": [] }, @@ -642,6 +831,8 @@ "risk_class": "conditional-runtime", "risk": "optional adapter construction and validation against live projections", "required_tests": [ + "tests/test_peft_artifacts.py", + "tests/test_persistence_pipeline.py", "tests/test_module_imports.py", "tests/conditional/test_model_download_runtime.py" ], diff --git a/docs/checkpoints/distributed-checkpoint-intake.md b/docs/checkpoints/distributed-checkpoint-intake.md new file mode 100644 index 0000000..4e9fba6 --- /dev/null +++ b/docs/checkpoints/distributed-checkpoint-intake.md @@ -0,0 +1,189 @@ +# Checkpoint formats, placement, and safe distributed-checkpoint inspection + +This guide separates capabilities that are often called “sharding” but have +different contracts. The machine-readable source of truth is +[support-matrix-v1.json](support-matrix-v1.json). The current implementation provides bounded, +non-executing structural inspection plus producer-neutral validation and writing. +Producer readers, adapters, exact trusted-reader profiles, trusted payload +execution, and live multi-node model surgery remain unavailable. + +## Short answer + +Current OBLITERATUS can load ordinary Hugging Face-compatible checkpoints and +can use Accelerate to place complete modules across devices visible to one +process on one host. It does not currently reconstruct PyTorch DCP/FSDP, +Megatron, or DeepSpeed rank fragments, and it does not run one surgery job across +multiple hosts. It can now classify the inert structure of those checkpoint +directories without invoking their framework readers. + +No producer-backed conversion is current. The common writer accepts only +already-normalized, validated fragments through its Python API; it does not read +a DCP, Megatron, or DeepSpeed payload. Future explicit offline conversion of one +narrowly qualified producer/model/version case remains Wave 3/4 work. Such a +conversion would not be exact training resume and would not make surgery +multi-node. + +## Terms that must remain separate + +| Term | Meaning | Current OBLITERATUS relevance | +|---|---|---| +| HF file shard | Complete named tensors distributed across files; index maps tensor name to file | Current loader can consume compatible inputs | +| safetensors byte offset | Half-open byte range inside one safetensors data buffer | File-format metadata, not rank placement | +| Accelerate `device_map` | Complete modules placed/offloaded by one process | Current, model-family dependent | +| CPU/disk offload | Process-local backing for complete model parameters | Current, not a distributed checkpoint | +| DCP/FSDP state | Framework-defined distributed state/chunks | Current structural detection only; no reader or adapter | +| Megatron offset | Logical tensor element placement and axis/chunk metadata | Current structural detection only; model-aware mapping remains required | +| DeepSpeed ZeRO/Universal | Partitioned or topology-neutral DeepSpeed state | Current structural detection only; trusted reader/resource gates remain | +| Live multi-node execution | Multiple processes/hosts with rendezvous, collectives, ownership, failure coordination, and save | Not supported; separate research decision | + +The terms in this document and the machine-readable +[schemas](schemas/support-matrix-v1.schema.json) are normative for this feature. Upstream semantics and +qualifications cite the public +[source register](references.md), +including PyTorch DCP/FSDP [R01–R03], Megatron/Bridge [R04–R07], DeepSpeed +[R08–R10], HF/safetensors [R11–R13], Accelerate placement/launch [R14–R15], +and serialization/containment guidance [R27–R35]. + +## Current support matrix + +| Subject | Load/surgery | Structural inspect | Convert to HF | Exact resume | Live multi-node | +|---|---|---|---|---|---| +| Existing compatible HF safetensors | Conditional on model/runtime gates | Conditional, header-only | Already canonical | Out of scope | Out of scope | +| Accelerate `device_map`/offload | Conditional, one process/host | Not applicable | Not applicable | Out of scope | Out of scope | +| PyTorch DCP/FSDP | No payload load | Conditional structural classification | Deferred | Out of scope | Out of scope | +| Megatron distributed state | No payload load | Conditional structural classification | Deferred, model-aware | Out of scope | Out of scope | +| DeepSpeed ZeRO/Universal | No payload load | Conditional structural classification | Deferred | Out of scope | Out of scope | +| PEFT LoRA safetensors | Conditional exact-base Python export | Conditional, header/JSON only | Not a rank-fragment conversion | Out of scope | Out of scope | +| Live multi-node surgery | Preflight only; no model payload | Not applicable | Separate offline concern | Out of scope | Deferred and unqualified | + +“Conditional” means the behavior depends on an exact model architecture, +runtime, kernels, dtype/quantization, memory, and quality gates. It is not a +universal compatibility claim. “Deferred” means planned and unimplemented. + +## Current safe structural inspection + +Run the inspector against one local file or directory: + +```bash +obliteratus checkpoint inspect ./checkpoint --json +``` + +The command inventories regular files without following links, streams digests, +parses bounded JSON and safetensors headers, and emits a strict descriptor with +classification evidence, confidence, resource estimates, and stable blockers. +It does not read tensor payloads, import producer frameworks, unpickle metadata, +initialize a process group, discover plugins, execute remote code, or access the +network. Treat a `conditional` classification as structural evidence only—not a +promise that the checkpoint can be loaded or converted. + +Legacy HF `.bin`/`.pt` files and DCP `.metadata` may be recognized by safe names +and companion structure, but remain opaque and trust-gated. Ambiguous layouts, +links, non-regular files, races, malformed bounded metadata, and resource-limit +violations fail closed. Inspection does not mutate the source or create output. + +The product capability registry is deliberately empty until one producer, +version, model family, and adapter is separately selected and qualified. The +safe plane nevertheless implements the closed-registry dependency diagnostic: +an explicitly supplied exact capability can be identified as a format candidate, +installed distribution metadata is checked without importing the distribution, +and a missing or mismatched dependency reports the exact OBLITERATUS extra, +project version, required distribution versions, and sanitized observed +versions. A candidate becomes an exact match only when independently observed +producer and version evidence also agree. Multiple matching capabilities fail +closed. This metadata-only resolution does not install anything, authorize +trust, invoke a reader, or make the capability a support claim. + +## Existing single-host placement + +For compatible model families, `device_map="auto"` may place complete modules +across CUDA devices visible to the current process and may use CPU/disk offload. +This is a capacity mechanism, not saved checkpoint topology and not rank-based +pipeline parallelism. Upstream Accelerate documents the placement/offload model +[R14](references.md#primary-and-upstream-sources); the project boundary is +confirmed by local code [R21](references.md#project-evidence). + +Qwen hybrid models retain the complete-text-model-on-one-device restriction. +Generic multi-device layer placement is rejected for those paths because the +restriction is a correctness gate, not an unimplemented distributed-checkpoint +feature. + +`--remote` selects one SSH host and launches one OBLITERATUS process there. It +does not coordinate ranks across hosts. + +## Gated intake workflow + +Only step 1 and the producer-neutral portions of steps 4–5 are current. The +remaining actions require separate design, security review, and qualification; +there is no producer-conversion CLI: + +1. **Inspect:** bounded local inventory plus JSON/safetensors-header parsing; + classify format, components, producer evidence, topology facts, state scope, + trust requirement, resources, adapter match, and blockers. +2. **Escalate only when reviewed:** some vendor metadata requires a trusted + reader. Default inspection stops and reports that requirement. Any future + policy must require fresh per-invocation intent plus a strict single-use + source/operation/runtime/isolation/resource-bound record and an exact approved + disposable worker profile. A checksum, familiar local filename, prior scan, + or `weights_only=True` is not trust. This remains unimplemented and + unauthorized. +3. **Adapt:** one exact producer/version/model adapter emits neutral tensor + fragments. Megatron requires a supported Bridge/provider mapping; offsets + alone do not define fused tensor semantics [R04, R07]. +4. **Validate:** prove shape/range coverage, replicas, padding, ties/shared state, + topology, and resource budgets. +5. **Materialize:** the current Python writer accepts already-normalized neutral + fragments, writes bounded safetensors staging, index, configuration, and + conversion manifest, then validates and promotes only on complete success. + It is not a producer reader or adapter. +6. **Load:** pass the canonical path into the unchanged existing HF loader. + +Version 1 intentionally emits model weights only. Model weights do not include +all optimizer, scheduler, RNG/scaler, progress, and data-position state needed +for producer-compatible exact resume [R19–R20](references.md#primary-and-upstream-sources). + +## What to provide with an unsupported-checkpoint report + +Provide only sanitized structural evidence: + +- exact model identifier and immutable revision when shareable; +- producer framework and exact version; +- checkpoint type and normalized relative file tree with sizes and safe SHA-256 + digests, excluding tensor values and sensitive local identifiers; +- saved node/world and TP/PP/DP/CP/EP/ETP/ZeRO topology when known; +- exact metadata field/API meant by “offset”; +- OBLITERATUS commit, command/config, OS, Python, PyTorch, Transformers, + Accelerate, and optional producer versions; +- complete normalized error and first failing stage; +- desired result: inspect, convert, run surgery, export, infer, or resume. + +Do not open an unfamiliar `.pt`, DCP `.metadata`, or vendor checkpoint merely to +collect a report. PyTorch documents serialization trust risks and an upstream +DCP issue identifies pickle use in `.metadata`; a 2026 advisory also shows why +weights-only loading is not a permanent safe-parser boundary [R17–R18, +R27–R29](references.md#security-and-containment-sources). + +## Resource and recovery expectations + +The producer-neutral writer first estimates source/logical/output/temporary +bytes and peak RAM. It enforces actual staged output/temporary bytes before +promotion; actual peak RAM and temporary-byte measurements remain `null` when +the process has not instrumented them, rather than being populated with +estimates. A denied or unknown admission does not start +materialization. Source artifacts remain immutable. Output is written into +sibling staging, validated, and promoted only when complete. Failure or +cancellation does not replace a prior valid output. The existing full-model +REBIRTH scaling bottleneck remains; the common writer API is not a practical +large-model conversion claim. + +DeepSpeed warns that fp32 consolidation can require substantial CPU memory +[R08](references.md#primary-and-upstream-sources). +No general memory multiplier or GPU-count promise is made without exact retained +evidence. + +## Support and escalation + +Use the [support runbook](support-runbook.md) for current triage, evidence +collection, recovery, and escalation. A future `supported` matrix row requires +exact producer/adapter versions, fixture digest, candidate commit, environment, +topology, retained result, and known limits. The offline contract validator is +current; no distributed producer row is promoted to `supported` by this work. diff --git a/docs/checkpoints/references.md b/docs/checkpoints/references.md new file mode 100644 index 0000000..51410d6 --- /dev/null +++ b/docs/checkpoints/references.md @@ -0,0 +1,76 @@ +# Distributed checkpoint intake source register + +**Artifact ID:** RESEARCH-DCI-001 +**Version:** 0.3.0 +**Status:** Reviewed technical evidence +**Accessed:** 2026-09-02 +**Repository baseline:** `5cc43c6e52903497574d80e08dff856028bc47f7` + +This register is the citation authority for the distributed-checkpoint feature +documentation. It records what each source supports and the limits on how the source may +be used. Vendor documentation establishes upstream behavior; it does not by +itself prove OBLITERATUS compatibility. Project support claims require exact-head +tests, immutable fixtures, and retained evidence at an exact candidate commit. + +## Primary and upstream sources + +| ID | Source | Evidence class | Supported use | Required qualification | +|---|---|---|---|---| +| R01 | [PyTorch Distributed Checkpoint API](https://docs.pytorch.org/docs/stable/distributed.checkpoint.html) | Primary project documentation; moderate confidence | DCP save/load planning, multi-rank storage, load-time resharding, preallocated state | DCP documents no general saved-state backward-compatibility guarantee; version-gate adapters | +| R02 | [PyTorch DCP recipe](https://docs.pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html) | Primary project tutorial; moderate confidence | Multi-rank examples and topology-change behavior | Tutorial behavior is illustrative, not a universal format contract | +| R03 | [PyTorch FSDP API](https://docs.pytorch.org/docs/stable/fsdp.html) | Primary project documentation; moderate confidence | Full/local/sharded state-dict distinctions and rank-zero CPU-offload semantics | FSDP state mode is not synonymous with DCP serialization | +| R04 | [Megatron Core sharded-tensor mapping](https://docs.nvidia.com/megatron-core/developer-guide/latest/apidocs/core/core.dist_checkpointing.mapping.html) | Primary vendor API documentation; moderate confidence | `global_offset`, `rank_offsets`, replica identity, element-coordinate semantics | Offsets establish placement, not model-family semantic mapping | +| R05 | [Megatron Core distributed checkpointing](https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/core/dist_checkpointing.html) | Primary vendor documentation; moderate confidence | Model-weight resharding across supported topology changes | Optimizer resharding is format/version-dependent and must be separately qualified | +| R06 | [Megatron Core parallelism guide](https://docs.nvidia.com/megatron-core/developer-guide/latest/user-guide/parallelism-guide.html) | Primary vendor documentation; moderate confidence | TP, PP, DP, CP, EP, ETP, FSDP terminology | Axis products and parameter sharding behavior must follow the producer contract, not inference | +| R07 | [Megatron Bridge AutoBridge](https://docs.nvidia.com/nemo/megatron-bridge/latest/apidocs/bridge/bridge.models.conversion.auto_bridge.html) | Primary vendor API documentation; moderate confidence | Model-aware bidirectional conversion and provider mapping | Bridge availability does not imply every model family or checkpoint version is supported | +| R08 | [DeepSpeed model checkpointing](https://deepspeed.readthedocs.io/en/stable/model-checkpointing.html) | Primary project documentation; moderate confidence | ZeRO-2/3 fp32 consolidation, CPU-memory warning, `safe_serialization` output option | Input consolidation uses framework serialization and belongs behind the explicit trust gate | +| R09 | [DeepSpeed Universal Checkpointing](https://www.deepspeed.ai/tutorials/universal-checkpointing/) | Primary project tutorial; moderate confidence | Topology-neutral DeepSpeed model/optimizer representation for compatible mappings | Universal Checkpointing is neither HF safetensors nor a generic architecture converter | +| R10 | [DeepSpeed `zero_to_fp32.py`](https://github.com/deepspeedai/DeepSpeed/blob/master/deepspeed/utils/zero_to_fp32.py) | Upstream implementation; moderate confidence, version-volatile | Confirms current consolidation path and `weights_only=False` input loading | Pin the exact source revision used by an adapter qualification | +| R11 | [safetensors metadata parsing](https://huggingface.co/docs/safetensors/metadata_parsing) | Primary project documentation; moderate confidence | Header dtype, shape, and half-open `data_offsets` relative to the tensor data buffer | Safe parsing does not establish authenticity, path safety, or provenance | +| R12 | [Hugging Face serialization helpers](https://huggingface.co/docs/huggingface_hub/en/package_reference/serialization) | Primary project documentation; moderate confidence | Sharded safetensors writing, indexes, tied/shared-tensor handling | Atomic publication is an OBLITERATUS responsibility, not an upstream guarantee | +| R13 | [Transformers model loading and sharded checkpoints](https://huggingface.co/docs/transformers/main/models) | Primary project documentation; moderate confidence | Named tensors distributed across files and index-based loading | A file-shard index contains no rank-fragment coordinate contract | +| R14 | [Accelerate big-model inference](https://huggingface.co/docs/accelerate/main/en/concept_guides/big_model_inference) | Primary project documentation; moderate confidence | Single-process module placement and CPU/disk offload via `device_map` | `device_map` is not checkpoint topology or a multi-node launcher | +| R15 | [Accelerate multi-node launch](https://huggingface.co/docs/accelerate/main/en/basic_tutorials/launch) | Primary project documentation; moderate confidence | Per-node machine rank, common rendezvous, launcher invocation on every node | Launch documentation does not prove OBLITERATUS has a distributed runtime | +| R16 | [PEFT checkpoint format](https://huggingface.co/docs/peft/main/developer_guides/checkpoint) | Primary project documentation; moderate confidence | Standard adapter files, adapter-only state, dependency on a base model | Immutable base revision/digest is an OBLITERATUS provenance rule and may be absent upstream | +| R17 | [PyTorch serialization notes](https://docs.pytorch.org/docs/main/notes/serialization.html) and [`torch.load`](https://docs.pytorch.org/docs/stable/generated/torch.load.html) | Primary project documentation; moderate confidence | Serialization trust warning and `weights_only` behavior | `weights_only` narrows risk but does not turn arbitrary input into verified data | +| R18 | [PyTorch issue: DCP `.metadata` uses pickle](https://github.com/pytorch/pytorch/issues/189308) | Upstream issue and source-linked observation; low-to-moderate confidence | Establishes a concrete reason default inspection must not treat `.metadata` as inert | Recheck against the exact PyTorch version before implementing a trusted reader | +| R19 | [Transformers Trainer resume recipes](https://huggingface.co/docs/transformers/main/trainer_recipes) | Primary project documentation; moderate confidence | Resume includes more than model weights, such as optimizer/scheduler/RNG state | Exact state varies by trainer/framework/version | +| R20 | [Accelerate training migration](https://huggingface.co/docs/accelerate/basic_tutorials/migration) | Primary project documentation; moderate confidence | Accelerator state can include model, optimizer, scheduler, RNG, and data position | This does not define a portable cross-framework resume format | + +## Security and containment sources + +| ID | Source | Evidence class | Supported use | Required qualification | +|---|---|---|---|---| +| R27 | [Python `pickle` documentation](https://docs.python.org/3/library/pickle.html) | Primary language documentation; high confidence | Establishes that malicious pickle can execute arbitrary code and untrusted/tampered data must not be unpickled | A signature or digest establishes integrity only under a separately trusted provenance/key decision; it does not make arbitrary objects semantically safe | +| R28 | [PyTorch security policy](https://github.com/pytorch/pytorch/blob/main/SECURITY.md) | Primary project security guidance; high confidence | Treat untrusted models as programs, prefer isolated execution, validate even safer formats, and do not expose distributed primitives to untrusted networks | Security guidance is not proof that a container or any individual loader/profile is safe; qualify the exact runtime and isolation | +| R29 | [PyTorch advisory GHSA-63cw-57p8-fm3p / CVE-2026-24747](https://github.com/pytorch/pytorch/security/advisories/GHSA-63cw-57p8-fm3p) | Primary project advisory; high confidence | Demonstrates code-execution risk in affected `weights_only=True` loading and supports rejecting it as a permanent safe-plane boundary | A patched version fixes the named defect only; future/parser/resource risks and trust requirements remain | +| R30 | [NIST SP 800-190, Application Container Security Guide](https://csrc.nist.gov/pubs/sp/800/190/final) | Primary government security guidance; high confidence | Container-specific threat/mitigation context and the need to secure images, registries, orchestrators, hosts, and runtime configuration | Published in 2017; apply principles to the exact current runtime and do not equate containers with complete sandboxing | +| R31 | [Linux kernel `no_new_privs` documentation](https://docs.kernel.org/userspace-api/no_new_privs.html) | Primary kernel documentation; high confidence | Prevent privilege gains through `execve` and support unprivileged seccomp-filter use | The flag does not prevent all privilege changes or provide filesystem/network/resource isolation by itself | +| R32 | [Linux kernel seccomp-filter documentation](https://docs.kernel.org/userspace-api/seccomp_filter.html) | Primary kernel documentation; high confidence | Reduce the syscall surface of a constrained worker and layer filters after `no_new_privs` | Syscall filtering is one containment layer, not a semantic validator or full sandbox | +| R33 | [Linux kernel cgroup v2 documentation](https://docs.kernel.org/admin-guide/cgroup-v2.html) | Primary kernel documentation; high confidence | Bound and observe worker memory/CPU/process resource use | Controller availability/configuration and kernel behavior must be preflighted and recorded on the exact host/profile | +| R34 | [OWASP Deserialization Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Deserialization_Cheat_Sheet.html) | Primary security-community guidance; moderate-to-high confidence | Prefer pure data formats, avoid native deserialization for untrusted data, and validate before object construction | General guidance; project controls must follow the Python/PyTorch and exact adapter/runtime behavior | +| R35 | [Linux `openat2(2)` manual](https://man7.org/linux/man-pages/man2/openat2.2.html) | Authoritative Linux interface documentation; high confidence | Root-relative path resolution with `RESOLVE_BENEATH`, `RESOLVE_NO_SYMLINKS`, and `RESOLVE_NO_MAGICLINKS` for untrusted paths | Linux-specific and kernel-version-dependent; other platforms need reviewed equivalent semantics or must refuse trusted-reader use | + +## Project evidence + +| ID | Source | Supported use | Limitation | +|---|---|---|---| +| R21 | OBLITERATUS `origin/main` at `5cc43c6e52903497574d80e08dff856028bc47f7`; see `obliteratus/models/loader.py`, `obliteratus/models/offload_surgery.py`, `obliteratus/abliterate.py`, `obliteratus/persistence_contracts.py`, `obliteratus/remote.py`, and `README.md` | Establishes current local loader, process-local placement/offload, complete-state export, atomic helper, one-host remote runner, and documentation boundary | Line references must be refreshed when implementation changes | + +## Claim rules + +1. Use “documents,” “defines,” or “currently implements” for vendor behavior; + do not convert vendor documentation into an OBLITERATUS support claim. +2. Mark project interpretations explicitly, especially the distinction between + `device_map` and checkpoint-rank topology. +3. A `supported` matrix row requires an exact producer version, adapter version, + immutable fixture digest, candidate commit, environment, topology, and retained + result at the exact candidate commit. +4. Archived/versioned documentation remains historical evidence only. Current + contracts use the latest cited primary documentation plus exact-version source. +5. Security claims remain bounded: subprocess isolation and resource limits are + containment controls, not proof that vendor deserialization is safe. +6. `weights_only=True`, a checksum, and a recognized local filename are never + represented as sufficient trust or authenticity controls [R27–R29]. +7. OS controls are cited as exact-profile containment mechanisms, not a portable + universal sandbox claim [R30–R35]. diff --git a/docs/checkpoints/schemas/adapter-capability-v1.schema.json b/docs/checkpoints/schemas/adapter-capability-v1.schema.json new file mode 100644 index 0000000..f47230b --- /dev/null +++ b/docs/checkpoints/schemas/adapter-capability-v1.schema.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/adapter-capability-v1.schema.json", + "title": "OBLITERATUS Checkpoint Adapter Capability v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_id", + "schema_version", + "adapter_id", + "adapter_version", + "contract_version", + "supported_producer_versions", + "tested_producer_versions", + "required_extras", + "formats", + "model_families", + "state_scopes", + "topology_capabilities", + "quantization_capabilities", + "safe_inspection", + "trusted_inspection", + "conversion", + "evidence" + ], + "properties": { + "schema_id": {"const": "obliteratus.adapter-capability"}, + "schema_version": {"const": "1.0.0"}, + "adapter_id": {"$ref": "#/$defs/id"}, + "adapter_version": {"$ref": "#/$defs/version"}, + "contract_version": {"const": "1.0.0"}, + "supported_producer_versions": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/versionBand"} + }, + "tested_producer_versions": { + "type": "array", + "items": {"$ref": "#/$defs/testedVersion"} + }, + "required_extras": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "formats": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "hf_safetensors", + "pytorch_dcp", + "fsdp_state_dict", + "megatron_torch_dist", + "megatron_torch_dcp", + "megatron_fsdp_dtensor", + "deepspeed_zero", + "deepspeed_universal", + "peft_safetensors" + ] + } + }, + "model_families": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "state_scopes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "topology_capabilities": { + "type": "object", + "additionalProperties": false, + "required": ["axes", "saved_to_canonical", "saved_to_changed_topology"], + "properties": { + "axes": { + "type": "array", + "uniqueItems": true, + "items": {"enum": ["nodes", "world", "tp", "pp", "dp", "cp", "ep", "etp", "fsdp", "zero"]} + }, + "saved_to_canonical": {"type": "boolean"}, + "saved_to_changed_topology": {"type": "boolean"} + } + }, + "quantization_capabilities": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["format", "status", "evidence_refs"], + "properties": { + "format": {"$ref": "#/$defs/id"}, + "status": {"enum": ["supported", "refused", "deferred"]}, + "evidence_refs": {"type": "array", "items": {"$ref": "#/$defs/id"}} + } + } + }, + "safe_inspection": {"type": "boolean"}, + "trusted_inspection": {"type": "boolean"}, + "conversion": {"type": "boolean"}, + "evidence": { + "type": "array", + "items": {"$ref": "#/$defs/evidence"} + } + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "version": {"type": "string", "minLength": 1, "maxLength": 128}, + "versionBand": { + "type": "object", + "additionalProperties": false, + "required": ["producer", "specifier", "rationale"], + "properties": { + "producer": {"$ref": "#/$defs/id"}, + "specifier": {"$ref": "#/$defs/version"}, + "rationale": {"type": "string", "minLength": 1, "maxLength": 2048} + } + }, + "testedVersion": { + "type": "object", + "additionalProperties": false, + "required": ["producer", "version", "fixture_digest", "candidate_commit", "environment", "topology"], + "properties": { + "producer": {"$ref": "#/$defs/id"}, + "version": {"$ref": "#/$defs/version"}, + "fixture_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "candidate_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "environment": {"$ref": "#/$defs/id"}, + "topology": {"$ref": "#/$defs/id"} + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "reference", "status"], + "properties": { + "kind": {"enum": ["primary_documentation", "upstream_source", "fixture", "test_result", "review"]}, + "reference": {"$ref": "#/$defs/id"}, + "status": {"enum": ["required", "present", "expired", "missing"]} + } + } + } +} diff --git a/docs/checkpoints/schemas/artifact-provenance-v1.schema.json b/docs/checkpoints/schemas/artifact-provenance-v1.schema.json new file mode 100644 index 0000000..a5ff87b --- /dev/null +++ b/docs/checkpoints/schemas/artifact-provenance-v1.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/artifact-provenance-v1.schema.json", + "title": "OBLITERATUS Artifact Provenance v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_id", + "schema_version", + "artifact_id", + "record_digest", + "sources", + "converter", + "obliteratus_commit", + "configuration_digest", + "tokenizer", + "base_model", + "command", + "environment", + "source_topology", + "lineage", + "input_digests", + "output_digests", + "transformations", + "state", + "adapter", + "dataset", + "training", + "unknowns" + ], + "properties": { + "schema_id": {"const": "obliteratus.artifact-provenance"}, + "schema_version": {"const": "1.0.0"}, + "artifact_id": {"type": "string", "pattern": "^artifact-sha256:[0-9a-f]{64}$"}, + "record_digest": {"$ref": "#/$defs/digest"}, + "sources": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/artifactIdentity"}}, + "converter": {"$ref": "#/$defs/toolIdentity"}, + "obliteratus_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "configuration_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]}, + "tokenizer": {"anyOf": [{"$ref": "#/$defs/artifactIdentity"}, {"type": "null"}]}, + "base_model": {"anyOf": [{"$ref": "#/$defs/artifactIdentity"}, {"type": "null"}]}, + "command": {"type": "array", "maxItems": 256, "items": {"type": "string", "maxLength": 4096}}, + "environment": { + "type": "object", + "additionalProperties": false, + "required": ["packages", "platform", "python"], + "properties": { + "python": {"type": ["string", "null"], "maxLength": 128}, + "platform": {"type": ["string", "null"], "maxLength": 256}, + "packages": {"type": "object", "additionalProperties": {"type": "string", "maxLength": 128}} + } + }, + "source_topology": {"type": "object"}, + "lineage": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/lineageEvent"}}, + "input_digests": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/digest"}}, + "output_digests": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/digest"}}, + "transformations": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "state": { + "type": "object", + "additionalProperties": false, + "required": ["classification", "observed_scopes", "lost_state"], + "properties": { + "classification": {"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]}, + "observed_scopes": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "lost_state": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}} + } + }, + "adapter": {"anyOf": [{"$ref": "#/$defs/adapterIdentity"}, {"type": "null"}]}, + "dataset": {"anyOf": [{"$ref": "#/$defs/datasetIdentity"}, {"type": "null"}]}, + "training": {"anyOf": [{"$ref": "#/$defs/trainingIdentity"}, {"type": "null"}]}, + "unknowns": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1, "maxLength": 1024}} + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "artifactIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "identity", "revision", "digest"], + "properties": { + "kind": {"enum": ["local", "hub", "generated"]}, + "identity": {"$ref": "#/$defs/id"}, + "revision": {"type": ["string", "null"], "maxLength": 256}, + "digest": {"$ref": "#/$defs/digest"} + } + }, + "toolIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "commit"], + "properties": { + "name": {"$ref": "#/$defs/id"}, + "version": {"$ref": "#/$defs/id"}, + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + } + }, + "lineageEvent": { + "type": "object", + "additionalProperties": false, + "required": ["event_id", "event_type", "parent_artifact_ids", "tool", "transformations"], + "properties": { + "event_id": {"$ref": "#/$defs/id"}, + "event_type": {"enum": ["consolidation", "reshard", "pretrain", "full_finetune", "adapter_train", "adapter_merge", "quantization", "dequantization", "surgery"]}, + "parent_artifact_ids": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^artifact-sha256:[0-9a-f]{64}$"}}, + "tool": {"$ref": "#/$defs/id"}, + "transformations": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}} + } + }, + "adapterIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["adapter_type", "base_model", "config_digest", "key_map_digest"], + "properties": { + "adapter_type": {"$ref": "#/$defs/id"}, + "base_model": {"$ref": "#/$defs/artifactIdentity"}, + "config_digest": {"$ref": "#/$defs/digest"}, + "key_map_digest": {"$ref": "#/$defs/digest"} + } + }, + "datasetIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["identifier", "revision", "digest", "split", "subset", "record_count"], + "properties": { + "identifier": {"$ref": "#/$defs/id"}, + "revision": {"type": ["string", "null"], "maxLength": 256}, + "digest": {"$ref": "#/$defs/digest"}, + "split": {"type": ["string", "null"], "maxLength": 256}, + "subset": {"type": ["string", "null"], "maxLength": 256}, + "record_count": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807} + } + }, + "trainingIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["method", "framework", "framework_version", "hyperparameters_digest"], + "properties": { + "method": {"enum": ["pretrain", "full_finetune", "adapter_train", "unknown"]}, + "framework": {"type": ["string", "null"], "maxLength": 256}, + "framework_version": {"type": ["string", "null"], "maxLength": 128}, + "hyperparameters_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]} + } + } + } +} diff --git a/docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json b/docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json new file mode 100644 index 0000000..bb17daf --- /dev/null +++ b/docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json @@ -0,0 +1,541 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/checkpoint-descriptor-v1.schema.json", + "title": "OBLITERATUS Checkpoint Descriptor v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_id", + "schema_version", + "descriptor_id", + "primary_format", + "classification_confidence", + "components", + "producer", + "evidence", + "source_inventory", + "safety", + "state", + "topologies", + "tensor_fragments", + "adapter_resolution", + "conversion_plan", + "resource_estimate", + "support_decision", + "blockers" + ], + "properties": { + "schema_id": {"const": "obliteratus.checkpoint-descriptor"}, + "schema_version": {"const": "1.0.0"}, + "descriptor_id": {"type": "string", "minLength": 1, "maxLength": 160}, + "primary_format": {"$ref": "#/$defs/checkpointFormat"}, + "classification_confidence": { + "enum": ["verified", "declared", "inferred", "unknown"] + }, + "components": { + "type": "array", + "minItems": 1, + "maxItems": 1024, + "items": {"$ref": "#/$defs/component"} + }, + "producer": {"$ref": "#/$defs/producer"}, + "evidence": { + "type": "array", + "maxItems": 100000, + "items": {"$ref": "#/$defs/evidence"} + }, + "source_inventory": {"$ref": "#/$defs/sourceInventory"}, + "safety": {"$ref": "#/$defs/safety"}, + "state": {"$ref": "#/$defs/state"}, + "topologies": { + "type": "array", + "maxItems": 128, + "items": {"$ref": "#/$defs/topology"} + }, + "tensor_fragments": { + "type": "array", + "maxItems": 10000000, + "items": {"$ref": "#/$defs/tensorFragment"} + }, + "adapter_resolution": {"$ref": "#/$defs/adapterResolution"}, + "conversion_plan": {"$ref": "#/$defs/conversionPlan"}, + "resource_estimate": {"$ref": "#/$defs/resourceEstimate"}, + "support_decision": { + "enum": [ + "canonical_hf_ready", + "conversion_supported", + "trusted_inspection_required", + "blocked" + ] + }, + "blockers": { + "type": "array", + "maxItems": 10000, + "items": {"$ref": "#/$defs/blocker"} + } + }, + "$defs": { + "nonNegativeInt64": { + "type": "integer", + "minimum": 0, + "maximum": 9223372036854775807 + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "identifier": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "checkpointFormat": { + "enum": [ + "hf_safetensors", + "hf_pytorch_pickle", + "pytorch_dcp", + "fsdp_state_dict", + "megatron_torch_dist", + "megatron_torch_dcp", + "megatron_fsdp_dtensor", + "deepspeed_zero", + "deepspeed_universal", + "peft_safetensors", + "unknown", + "ambiguous" + ] + }, + "component": { + "type": "object", + "additionalProperties": false, + "required": [ + "component_id", + "kind", + "format", + "producer", + "state_scopes", + "inventory_refs", + "tensor_fragment_refs" + ], + "properties": { + "component_id": {"$ref": "#/$defs/identifier"}, + "kind": { + "enum": [ + "model", + "optimizer", + "scheduler", + "rng", + "scaler", + "progress", + "dataloader", + "peft_adapter", + "configuration", + "tokenizer", + "unknown" + ] + }, + "format": {"$ref": "#/$defs/checkpointFormat"}, + "producer": {"$ref": "#/$defs/producer"}, + "state_scopes": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + }, + "topology_ref": {"type": ["string", "null"], "maxLength": 512}, + "inventory_refs": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + }, + "tensor_fragment_refs": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + } + }, + "producer": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "format_version", "evidence_refs"], + "properties": { + "name": {"type": ["string", "null"], "maxLength": 256}, + "version": {"type": ["string", "null"], "maxLength": 128}, + "format_version": {"type": ["string", "null"], "maxLength": 128}, + "evidence_refs": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "evidence_id", + "subject", + "kind", + "file_ref", + "location", + "confidence", + "verifier" + ], + "properties": { + "evidence_id": {"$ref": "#/$defs/identifier"}, + "subject": {"type": "string", "pattern": "^/"}, + "kind": { + "enum": ["explicit_metadata", "header", "json", "filename", "operator", "derived"] + }, + "file_ref": {"type": ["string", "null"], "maxLength": 512}, + "location": {"type": "string", "maxLength": 1024}, + "confidence": {"enum": ["verified", "declared", "inferred", "unknown"]}, + "verifier": {"type": "string", "maxLength": 256} + } + }, + "sourceInventory": { + "type": "object", + "additionalProperties": false, + "required": ["inventory_id", "files", "total_bytes", "observation_complete"], + "properties": { + "inventory_id": {"$ref": "#/$defs/identifier"}, + "files": { + "type": "array", + "maxItems": 1000000, + "items": {"$ref": "#/$defs/inventoryFile"} + }, + "total_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "observation_complete": {"type": "boolean"} + } + }, + "inventoryFile": { + "type": "object", + "additionalProperties": false, + "required": [ + "file_id", + "relative_path", + "role", + "size_bytes", + "sha256", + "regular_file", + "observation_id" + ], + "properties": { + "file_id": {"$ref": "#/$defs/identifier"}, + "relative_path": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"} + }, + "role": {"type": "string", "minLength": 1, "maxLength": 128}, + "size_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "sha256": {"$ref": "#/$defs/digest"}, + "regular_file": {"const": true}, + "observation_id": {"$ref": "#/$defs/identifier"} + } + }, + "safety": { + "type": "object", + "additionalProperties": false, + "required": [ + "inspection_level", + "trust_required", + "inventory_revalidated", + "unsafe_serialization_findings", + "violations" + ], + "properties": { + "inspection_level": {"enum": ["safe_structure", "trusted_metadata"]}, + "trust_required": {"type": "boolean"}, + "inventory_revalidated": {"type": "boolean"}, + "unsafe_serialization_findings": { + "type": "array", + "items": {"type": "string", "maxLength": 1024} + }, + "violations": { + "type": "array", + "items": {"type": "string", "maxLength": 1024} + } + } + }, + "state": { + "type": "object", + "additionalProperties": false, + "required": ["observed_scopes", "classification"], + "properties": { + "observed_scopes": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + }, + "classification": { + "enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"] + } + } + }, + "fact": { + "type": "object", + "additionalProperties": false, + "required": ["value", "provenance", "evidence_refs"], + "properties": { + "value": {"anyOf": [{"$ref": "#/$defs/nonNegativeInt64"}, {"type": "null"}]}, + "provenance": { + "enum": ["explicit", "filename_inferred", "operator_supplied", "unknown"] + }, + "evidence_refs": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + } + }, + "topology": { + "type": "object", + "additionalProperties": false, + "required": [ + "topology_id", + "kind", + "node_count", + "global_world_size", + "machine_rank", + "global_rank", + "local_rank", + "tp_size", + "pp_size", + "dp_size", + "cp_size", + "ep_size", + "etp_size", + "zero_stage" + ], + "properties": { + "topology_id": {"$ref": "#/$defs/identifier"}, + "kind": {"enum": ["saved", "target"]}, + "node_count": {"$ref": "#/$defs/fact"}, + "global_world_size": {"$ref": "#/$defs/fact"}, + "machine_rank": {"$ref": "#/$defs/fact"}, + "global_rank": {"$ref": "#/$defs/fact"}, + "local_rank": {"$ref": "#/$defs/fact"}, + "tp_size": {"$ref": "#/$defs/fact"}, + "pp_size": {"$ref": "#/$defs/fact"}, + "dp_size": {"$ref": "#/$defs/fact"}, + "cp_size": {"$ref": "#/$defs/fact"}, + "ep_size": {"$ref": "#/$defs/fact"}, + "etp_size": {"$ref": "#/$defs/fact"}, + "zero_stage": {"$ref": "#/$defs/fact"} + } + }, + "tensorFragment": { + "type": "object", + "additionalProperties": false, + "required": [ + "fragment_id", + "component_id", + "fqn", + "role", + "dtype", + "global_shape", + "local_shape", + "element_offset", + "element_extent", + "padding", + "shard_file_id", + "shard_digest_ref", + "replica", + "partition_axes", + "logical_tensor_id", + "storage_locations", + "evidence_refs" + ], + "properties": { + "fragment_id": {"$ref": "#/$defs/identifier"}, + "component_id": {"$ref": "#/$defs/identifier"}, + "fqn": {"$ref": "#/$defs/identifier"}, + "role": {"enum": ["parameter", "persistent_buffer"]}, + "dtype": {"$ref": "#/$defs/identifier"}, + "global_shape": {"$ref": "#/$defs/shape"}, + "local_shape": {"$ref": "#/$defs/shape"}, + "element_offset": {"$ref": "#/$defs/shape"}, + "element_extent": {"$ref": "#/$defs/shape"}, + "padding": {"$ref": "#/$defs/padding"}, + "shard_file_id": {"$ref": "#/$defs/identifier"}, + "shard_digest_ref": {"$ref": "#/$defs/identifier"}, + "fragment_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]}, + "replica": {"$ref": "#/$defs/replica"}, + "partition_axes": { + "type": "array", + "items": {"$ref": "#/$defs/nonNegativeInt64"}, + "uniqueItems": true + }, + "logical_tensor_id": {"$ref": "#/$defs/identifier"}, + "tie_group_id": {"type": ["string", "null"], "maxLength": 512}, + "shared_storage_id": {"type": ["string", "null"], "maxLength": 512}, + "storage_locations": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/storageLocation"} + }, + "evidence_refs": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + } + }, + "shape": { + "type": "array", + "maxItems": 32, + "items": {"$ref": "#/$defs/nonNegativeInt64"} + }, + "padding": { + "type": "object", + "additionalProperties": false, + "required": ["before", "after", "semantic"], + "properties": { + "before": {"$ref": "#/$defs/shape"}, + "after": {"$ref": "#/$defs/shape"}, + "semantic": {"enum": ["none", "producer_declared", "model_mapping_declared"]} + } + }, + "replica": { + "type": "object", + "additionalProperties": false, + "required": ["group_id", "member_index", "member_count"], + "properties": { + "group_id": {"type": ["string", "null"], "maxLength": 512}, + "member_index": {"$ref": "#/$defs/nonNegativeInt64"}, + "member_count": {"$ref": "#/$defs/nonNegativeInt64"} + } + }, + "storageLocation": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "element_offset", "element_extent"], + "properties": { + "kind": {"const": "logical_element_range"}, + "element_offset": {"$ref": "#/$defs/shape"}, + "element_extent": {"$ref": "#/$defs/shape"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "global_offset", "rank_offsets"], + "properties": { + "kind": {"const": "megatron_rank_offsets"}, + "global_offset": {"$ref": "#/$defs/shape"}, + "rank_offsets": { + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + {"$ref": "#/$defs/nonNegativeInt64"}, + {"$ref": "#/$defs/nonNegativeInt64"}, + {"$ref": "#/$defs/nonNegativeInt64"} + ], + "items": false, + "minItems": 3, + "maxItems": 3 + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "begin", "end"], + "properties": { + "kind": {"const": "safetensors_byte_range"}, + "begin": {"$ref": "#/$defs/nonNegativeInt64"}, + "end": {"$ref": "#/$defs/nonNegativeInt64"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "tensor_name", "file_id"], + "properties": { + "kind": {"const": "hf_weight_map"}, + "tensor_name": {"$ref": "#/$defs/identifier"}, + "file_id": {"$ref": "#/$defs/identifier"} + } + } + ] + }, + "adapterResolution": { + "type": "object", + "additionalProperties": false, + "required": ["status", "adapter_id", "adapter_version", "capability_digest", "reason"], + "properties": { + "status": {"enum": ["matched", "not_required", "missing", "ambiguous", "unsupported"]}, + "adapter_id": {"type": ["string", "null"], "maxLength": 256}, + "adapter_version": {"type": ["string", "null"], "maxLength": 128}, + "capability_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]}, + "reason": {"type": "string", "maxLength": 2048} + } + }, + "conversionPlan": { + "type": "object", + "additionalProperties": false, + "required": ["eligible", "target_format", "state_scope", "dropped_scopes"], + "properties": { + "eligible": {"type": "boolean"}, + "target_format": {"const": "hf_safetensors"}, + "state_scope": {"const": "weights_only"}, + "dropped_scopes": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + } + }, + "resourceEstimate": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_bytes", + "logical_bytes", + "output_bytes", + "temporary_bytes", + "peak_ram_bytes", + "peak_vram_bytes", + "file_count", + "tensor_count", + "shard_count", + "assumptions", + "confidence", + "admission" + ], + "properties": { + "source_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "logical_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "output_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "temporary_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "peak_ram_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "peak_vram_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "file_count": {"$ref": "#/$defs/nonNegativeInt64"}, + "tensor_count": {"$ref": "#/$defs/nonNegativeInt64"}, + "shard_count": {"$ref": "#/$defs/nonNegativeInt64"}, + "assumptions": {"type": "array", "items": {"type": "string", "maxLength": 1024}}, + "confidence": {"enum": ["verified", "declared", "inferred", "unknown"]}, + "admission": {"enum": ["admitted", "denied", "unknown"]} + } + }, + "blocker": { + "type": "object", + "additionalProperties": false, + "required": ["code", "category", "phase", "affected_refs", "retryable", "next_action"], + "properties": { + "code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"}, + "category": {"enum": ["cleanup", "concurrency", "evidence", "output", "protocol", "resource", "runtime", "source", "trust", "unsupported", "validation"]}, + "phase": {"enum": ["admission", "classification", "cleanup", "evidence", "materialization", "policy", "preflight", "promotion", "protocol", "reader", "source", "validation"]}, + "affected_refs": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}, + "retryable": {"type": "boolean"}, + "next_action": {"type": "string", "maxLength": 2048} + } + } + } +} diff --git a/docs/checkpoints/schemas/checkpoint-error-codes-v1.json b/docs/checkpoints/schemas/checkpoint-error-codes-v1.json new file mode 100644 index 0000000..41443db --- /dev/null +++ b/docs/checkpoints/schemas/checkpoint-error-codes-v1.json @@ -0,0 +1,27 @@ +{ + "schema_id": "obliteratus.checkpoint-error-registry", + "schema_version": "1.0.0", + "entries": [ + {"code": "DCI_UNSUPPORTED_FORMAT_OR_VERSION", "phase": "classification", "category": "unsupported", "degraded_modes": ["F01"]}, + {"code": "DCI_TRUST_POLICY_REQUIRED", "phase": "policy", "category": "trust", "degraded_modes": ["F02"]}, + {"code": "DCI_TRUST_POLICY_MISMATCH", "phase": "policy", "category": "trust", "degraded_modes": ["F02", "F18"]}, + {"code": "DCI_SOURCE_BOUNDARY_VIOLATION", "phase": "source", "category": "source", "degraded_modes": ["F03"]}, + {"code": "DCI_SOURCE_CHANGED", "phase": "source", "category": "source", "degraded_modes": ["F04"]}, + {"code": "DCI_TRUST_RUNTIME_UNAVAILABLE", "phase": "preflight", "category": "runtime", "degraded_modes": ["F05", "F19"]}, + {"code": "DCI_RUNTIME_IDENTITY_MISMATCH", "phase": "preflight", "category": "runtime", "degraded_modes": ["F06", "F20"]}, + {"code": "DCI_FORBIDDEN_READER_CAPABILITY", "phase": "reader", "category": "runtime", "degraded_modes": ["F07"]}, + {"code": "DCI_RESOURCE_LIMIT", "phase": "reader", "category": "resource", "degraded_modes": ["F08", "F09"]}, + {"code": "DCI_TRUSTED_READER_FAILED", "phase": "reader", "category": "runtime", "degraded_modes": ["F09"]}, + {"code": "DCI_WORKER_PROTOCOL_INVALID", "phase": "protocol", "category": "protocol", "degraded_modes": ["F10"]}, + {"code": "DCI_VALIDATION_FAILED", "phase": "validation", "category": "validation", "degraded_modes": ["F11", "F14"]}, + {"code": "DCI_ADMISSION_DENIED", "phase": "admission", "category": "resource", "degraded_modes": ["F12"]}, + {"code": "DCI_MATERIALIZE_FAILED", "phase": "materialization", "category": "output", "degraded_modes": ["F13"]}, + {"code": "DCI_PROMOTION_FAILED", "phase": "promotion", "category": "output", "degraded_modes": ["F13", "F14"]}, + {"code": "DCI_EVIDENCE_UNAVAILABLE", "phase": "evidence", "category": "evidence", "degraded_modes": ["F15"]}, + {"code": "DCI_DIAGNOSTIC_REDACTION_FAILED", "phase": "evidence", "category": "evidence", "degraded_modes": ["F16"]}, + {"code": "DCI_CLEANUP_INCOMPLETE", "phase": "cleanup", "category": "cleanup", "degraded_modes": ["F17"]}, + {"code": "DCI_CONCURRENT_OPERATION_CONFLICT", "phase": "admission", "category": "concurrency", "degraded_modes": ["F18"]}, + {"code": "DCI_HOST_TRUST_UNSATISFIED", "phase": "preflight", "category": "trust", "degraded_modes": ["F19"]}, + {"code": "DCI_SECURITY_BASELINE_REVOKED", "phase": "preflight", "category": "runtime", "degraded_modes": ["F20"]} + ] +} diff --git a/docs/checkpoints/schemas/checkpoint-error-registry-v1.schema.json b/docs/checkpoints/schemas/checkpoint-error-registry-v1.schema.json new file mode 100644 index 0000000..d766239 --- /dev/null +++ b/docs/checkpoints/schemas/checkpoint-error-registry-v1.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/checkpoint-error-registry-v1.schema.json", + "title": "OBLITERATUS Checkpoint Error Registry v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_id", "schema_version", "entries"], + "properties": { + "schema_id": {"const": "obliteratus.checkpoint-error-registry"}, + "schema_version": {"const": "1.0.0"}, + "entries": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "phase", "category", "degraded_modes"], + "properties": { + "code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"}, + "phase": {"enum": ["classification", "policy", "source", "preflight", "reader", "protocol", "validation", "admission", "materialization", "promotion", "evidence", "cleanup"]}, + "category": {"enum": ["unsupported", "trust", "source", "runtime", "resource", "protocol", "validation", "output", "evidence", "cleanup", "concurrency"]}, + "degraded_modes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^F(?:0[1-9]|1[0-9]|20)$"}} + } + } + } + } +} diff --git a/docs/checkpoints/schemas/checkpoint-trust-policy-v1.schema.json b/docs/checkpoints/schemas/checkpoint-trust-policy-v1.schema.json new file mode 100644 index 0000000..bbef7e6 --- /dev/null +++ b/docs/checkpoints/schemas/checkpoint-trust-policy-v1.schema.json @@ -0,0 +1,134 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/checkpoint-trust-policy-v1.schema.json", + "title": "OBLITERATUS Checkpoint Trust Policy v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_id", "schema_version", "policy_id", "operation_level", "requested_state_scope", "adapter", "source", "runtime", "isolation", "resources", "output", "authorization", "evidence"], + "properties": { + "schema_id": {"const": "obliteratus.checkpoint-trust-policy"}, + "schema_version": {"const": "1.0.0"}, + "policy_id": {"$ref": "#/$defs/id"}, + "operation_level": {"enum": ["trusted_metadata", "trusted_conversion"]}, + "requested_state_scope": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "adapter": { + "type": "object", + "additionalProperties": false, + "required": ["capability_id", "adapter_id", "adapter_version", "adapter_digest"], + "properties": { + "capability_id": {"$ref": "#/$defs/id"}, + "adapter_id": {"$ref": "#/$defs/id"}, + "adapter_version": {"$ref": "#/$defs/id"}, + "adapter_digest": {"$ref": "#/$defs/digest"} + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["root_id", "inventory_digest", "files"], + "properties": { + "root_id": {"$ref": "#/$defs/id"}, + "inventory_digest": {"$ref": "#/$defs/digest"}, + "files": { + "type": "array", + "minItems": 1, + "maxItems": 1000000, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["file_id", "relative_path", "size_bytes", "sha256", "observation_id"], + "properties": { + "file_id": {"$ref": "#/$defs/id"}, + "relative_path": {"type": "string", "minLength": 1, "maxLength": 4096, "not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}}, + "size_bytes": {"$ref": "#/$defs/nonNegativeInt64"}, + "sha256": {"$ref": "#/$defs/digest"}, + "observation_id": {"$ref": "#/$defs/id"} + } + } + } + } + }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": ["framework", "framework_version", "dependency_identity", "runtime_digest"], + "properties": { + "framework": {"$ref": "#/$defs/id"}, + "framework_version": {"$ref": "#/$defs/id"}, + "dependency_identity": {"$ref": "#/$defs/digest"}, + "runtime_digest": {"$ref": "#/$defs/digest"} + } + }, + "isolation": { + "type": "object", + "additionalProperties": false, + "required": ["profile_id", "profile_version", "profile_digest", "required_capabilities"], + "properties": { + "profile_id": {"$ref": "#/$defs/id"}, + "profile_version": {"$ref": "#/$defs/id"}, + "profile_digest": {"$ref": "#/$defs/digest"}, + "required_capabilities": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"enum": ["unprivileged", "no_new_privs", "read_only_source", "root_isolation", "private_scratch", "network_denied", "ipc_isolated", "environment_allowlist", "fd_allowlist", "syscall_policy", "resource_limits", "device_denied", "core_dumps_disabled", "bounded_cleanup"]} + } + } + }, + "resources": { + "type": "object", + "additionalProperties": false, + "required": ["wall_time_seconds", "cpu_seconds", "memory_bytes", "processes", "threads", "open_files", "input_bytes", "header_bytes", "message_bytes", "scratch_bytes", "output_bytes"], + "properties": { + "wall_time_seconds": {"$ref": "#/$defs/positiveInt64"}, + "cpu_seconds": {"$ref": "#/$defs/positiveInt64"}, + "memory_bytes": {"$ref": "#/$defs/positiveInt64"}, + "processes": {"$ref": "#/$defs/positiveInt64"}, + "threads": {"$ref": "#/$defs/positiveInt64"}, + "open_files": {"$ref": "#/$defs/positiveInt64"}, + "input_bytes": {"$ref": "#/$defs/positiveInt64"}, + "header_bytes": {"$ref": "#/$defs/positiveInt64"}, + "message_bytes": {"$ref": "#/$defs/positiveInt64"}, + "scratch_bytes": {"$ref": "#/$defs/positiveInt64"}, + "output_bytes": {"$ref": "#/$defs/nonNegativeInt64"} + } + }, + "output": { + "type": "object", + "additionalProperties": false, + "required": ["root_id", "staging_policy"], + "properties": { + "root_id": {"type": ["string", "null"], "minLength": 1, "maxLength": 512}, + "staging_policy": {"enum": ["none", "private_sibling_atomic"]} + } + }, + "authorization": { + "type": "object", + "additionalProperties": false, + "required": ["event_id", "actor_role", "created_at", "expires_at", "single_use_nonce", "single_use"], + "properties": { + "event_id": {"$ref": "#/$defs/id"}, + "actor_role": {"$ref": "#/$defs/id"}, + "created_at": {"type": "string", "format": "date-time"}, + "expires_at": {"type": "string", "format": "date-time"}, + "single_use_nonce": {"type": "string", "minLength": 32, "maxLength": 256}, + "single_use": {"const": true} + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["candidate_commit", "policy_digest"], + "properties": { + "candidate_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "policy_digest": {"$ref": "#/$defs/digest"} + } + } + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "nonNegativeInt64": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, + "positiveInt64": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807} + } +} diff --git a/docs/checkpoints/schemas/conversion-manifest-v1.schema.json b/docs/checkpoints/schemas/conversion-manifest-v1.schema.json new file mode 100644 index 0000000..0163ce0 --- /dev/null +++ b/docs/checkpoints/schemas/conversion-manifest-v1.schema.json @@ -0,0 +1,167 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/conversion-manifest-v1.schema.json", + "title": "OBLITERATUS Checkpoint Conversion Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_id", + "schema_version", + "manifest_id", + "descriptor", + "source_inventory_digest", + "source_files", + "adapter", + "source_topology", + "state", + "dropped_scopes", + "canonical_output", + "resource_usage", + "validation", + "provenance", + "publication" + ], + "properties": { + "schema_id": {"const": "obliteratus.conversion-manifest"}, + "schema_version": {"const": "1.0.0"}, + "manifest_id": {"$ref": "#/$defs/id"}, + "descriptor": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "digest"], + "properties": { + "schema_version": {"const": "1.0.0"}, + "digest": {"$ref": "#/$defs/digest"} + } + }, + "source_inventory_digest": {"$ref": "#/$defs/digest"}, + "source_files": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/file"} + }, + "adapter": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "capability_digest", "producer", "producer_version"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "version": {"$ref": "#/$defs/id"}, + "capability_digest": {"$ref": "#/$defs/digest"}, + "producer": {"$ref": "#/$defs/id"}, + "producer_version": {"$ref": "#/$defs/id"} + } + }, + "source_topology": {"type": "object"}, + "state": { + "type": "object", + "additionalProperties": false, + "required": ["source_classification", "output_classification", "observed_scopes"], + "properties": { + "source_classification": {"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]}, + "output_classification": {"const": "weights_only"}, + "observed_scopes": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}} + } + }, + "dropped_scopes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["scope", "reason"], + "properties": { + "scope": {"$ref": "#/$defs/id"}, + "reason": {"type": "string", "minLength": 1, "maxLength": 2048} + } + } + }, + "canonical_output": { + "type": "object", + "additionalProperties": false, + "required": ["format", "dtype_policy", "files", "hf_index", "logical_tensor_count", "logical_bytes"], + "properties": { + "format": {"const": "hf_safetensors"}, + "dtype_policy": {"$ref": "#/$defs/id"}, + "files": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/file"}}, + "hf_index": {"type": ["string", "null"], "maxLength": 4096}, + "logical_tensor_count": {"$ref": "#/$defs/int64"}, + "logical_bytes": {"$ref": "#/$defs/int64"} + } + }, + "resource_usage": { + "type": "object", + "additionalProperties": false, + "required": ["estimated_peak_ram_bytes", "actual_peak_ram_bytes", "estimated_temp_bytes", "actual_temp_bytes"], + "properties": { + "estimated_peak_ram_bytes": {"$ref": "#/$defs/int64"}, + "actual_peak_ram_bytes": {"anyOf": [{"$ref": "#/$defs/int64"}, {"type": "null"}]}, + "estimated_temp_bytes": {"$ref": "#/$defs/int64"}, + "actual_temp_bytes": {"anyOf": [{"$ref": "#/$defs/int64"}, {"type": "null"}]} + } + }, + "validation": { + "type": "object", + "additionalProperties": false, + "required": ["coverage", "replicas", "ties", "hashes", "index", "safe_reload", "source_unchanged", "result"], + "properties": { + "coverage": {"type": "boolean"}, + "replicas": {"type": "boolean"}, + "ties": {"type": "boolean"}, + "hashes": {"type": "boolean"}, + "index": {"type": "boolean"}, + "safe_reload": {"type": "boolean"}, + "source_unchanged": {"type": "boolean"}, + "result": {"enum": ["passed", "failed"]} + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["obliteratus_commit", "configuration_digest", "tokenizer_digest", "base_model", "transformation_log", "unknowns"], + "properties": { + "obliteratus_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "configuration_digest": {"type": ["string", "null"], "pattern": "^(sha256:[0-9a-f]{64})?$"}, + "tokenizer_digest": {"type": ["string", "null"], "pattern": "^(sha256:[0-9a-f]{64})?$"}, + "base_model": { + "type": "object", + "additionalProperties": false, + "required": ["identity", "revision", "digest"], + "properties": { + "identity": {"type": ["string", "null"], "maxLength": 512}, + "revision": {"type": ["string", "null"], "maxLength": 256}, + "digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]} + } + }, + "transformation_log": {"type": "array", "items": {"type": "string", "maxLength": 2048}}, + "unknowns": {"type": "array", "items": {"type": "string", "maxLength": 2048}} + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["staging_validated", "promoted", "atomic_strategy", "rollback_result"], + "properties": { + "staging_validated": {"type": "boolean"}, + "promoted": {"type": "boolean"}, + "atomic_strategy": {"$ref": "#/$defs/id"}, + "rollback_result": {"enum": ["not_required", "succeeded", "failed"]} + } + } + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "int64": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, + "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "file": { + "type": "object", + "additionalProperties": false, + "required": ["relative_path", "size_bytes", "sha256"], + "properties": { + "relative_path": {"type": "string", "minLength": 1, "maxLength": 4096, "not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}}, + "size_bytes": {"$ref": "#/$defs/int64"}, + "sha256": {"$ref": "#/$defs/digest"} + } + } + } +} diff --git a/docs/checkpoints/schemas/fixtures/v1/invalid/adapter-claims-tested-without-evidence.json b/docs/checkpoints/schemas/fixtures/v1/invalid/adapter-claims-tested-without-evidence.json new file mode 100644 index 0000000..e4a84d3 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/invalid/adapter-claims-tested-without-evidence.json @@ -0,0 +1,21 @@ +{ + "schema_id": "obliteratus.adapter-capability", + "schema_version": "1.0.0", + "adapter_id": "invalid-adapter", + "adapter_version": "1.0.0", + "contract_version": "1.0.0", + "supported_producer_versions": [], + "tested_producer_versions": [ + {"producer": "unknown", "version": "latest"} + ], + "required_extras": [], + "formats": ["pytorch_dcp"], + "model_families": ["any"], + "state_scopes": ["exact_resume"], + "topology_capabilities": {"axes": [], "saved_to_canonical": true, "saved_to_changed_topology": true}, + "quantization_capabilities": [], + "safe_inspection": true, + "trusted_inspection": true, + "conversion": true, + "evidence": [] +} diff --git a/docs/checkpoints/schemas/fixtures/v1/invalid/generic-offset-field.json b/docs/checkpoints/schemas/fixtures/v1/invalid/generic-offset-field.json new file mode 100644 index 0000000..87657df --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/invalid/generic-offset-field.json @@ -0,0 +1,6 @@ +{ + "schema_id": "obliteratus.checkpoint-descriptor", + "schema_version": "1.0.0", + "descriptor_id": "invalid-generic-offset", + "offset": [0, 4] +} diff --git a/docs/checkpoints/schemas/fixtures/v1/invalid/trust-policy-leaks-environment.json b/docs/checkpoints/schemas/fixtures/v1/invalid/trust-policy-leaks-environment.json new file mode 100644 index 0000000..b5e0100 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/invalid/trust-policy-leaks-environment.json @@ -0,0 +1,5 @@ +{ + "schema_id": "obliteratus.checkpoint-trust-policy", + "schema_version": "1.0.0", + "environment": {"HOME": "/private/path", "TOKEN": "must-not-persist"} +} diff --git a/docs/checkpoints/schemas/fixtures/v1/invalid/trusted-worker-error-raw-message.json b/docs/checkpoints/schemas/fixtures/v1/invalid/trusted-worker-error-raw-message.json new file mode 100644 index 0000000..aa03014 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/invalid/trusted-worker-error-raw-message.json @@ -0,0 +1,10 @@ +{ + "schema_id": "obliteratus.trusted-worker-message", + "schema_version": "1.0.0", + "message_type": "error", + "sequence": 1, + "source_inventory_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "adapter_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "policy_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "payload": {"code": "DCI_TRUSTED_READER_FAILED", "phase": "reader", "category": "runtime", "subject_ids": [], "limit": null, "actual": null, "raw_message": "private path or secret"} +} diff --git a/docs/checkpoints/schemas/fixtures/v1/valid/deferred-adapter-capability.json b/docs/checkpoints/schemas/fixtures/v1/valid/deferred-adapter-capability.json new file mode 100644 index 0000000..f0c36b6 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/valid/deferred-adapter-capability.json @@ -0,0 +1,34 @@ +{ + "schema_id": "obliteratus.adapter-capability", + "schema_version": "1.0.0", + "adapter_id": "megatron-bridge-planned", + "adapter_version": "0.1.0-planned", + "contract_version": "1.0.0", + "supported_producer_versions": [ + { + "producer": "megatron-core", + "specifier": "unselected", + "rationale": "Version band remains blocked on exact fixture and adapter selection." + } + ], + "tested_producer_versions": [], + "required_extras": ["checkpoint-megatron"], + "formats": ["megatron_torch_dist"], + "model_families": ["unselected"], + "state_scopes": ["model_weights"], + "topology_capabilities": { + "axes": ["world", "tp", "pp", "dp"], + "saved_to_canonical": true, + "saved_to_changed_topology": false + }, + "quantization_capabilities": [ + {"format": "native_distributed", "status": "refused", "evidence_refs": []} + ], + "safe_inspection": false, + "trusted_inspection": false, + "conversion": false, + "evidence": [ + {"kind": "primary_documentation", "reference": "R04,R07", "status": "present"}, + {"kind": "fixture", "reference": "exact-version-producer-fixture", "status": "missing"} + ] +} diff --git a/docs/checkpoints/schemas/fixtures/v1/valid/trusted-metadata-policy.json b/docs/checkpoints/schemas/fixtures/v1/valid/trusted-metadata-policy.json new file mode 100644 index 0000000..31658e7 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/valid/trusted-metadata-policy.json @@ -0,0 +1,15 @@ +{ + "schema_id": "obliteratus.checkpoint-trust-policy", + "schema_version": "1.0.0", + "policy_id": "policy-fixture-1", + "operation_level": "trusted_metadata", + "requested_state_scope": ["model_weights"], + "adapter": {"capability_id": "dcp-metadata", "adapter_id": "fixture-reader", "adapter_version": "1.0.0", "adapter_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + "source": {"root_id": "source-1", "inventory_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "files": [{"file_id": "file-1", "relative_path": ".metadata", "size_bytes": 32, "sha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "observation_id": "observation-1"}]}, + "runtime": {"framework": "pytorch", "framework_version": "exact-fixture-version", "dependency_identity": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "runtime_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}, + "isolation": {"profile_id": "disposable-linux-fixture", "profile_version": "1", "profile_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "required_capabilities": ["unprivileged", "no_new_privs", "read_only_source", "network_denied", "resource_limits", "bounded_cleanup"]}, + "resources": {"wall_time_seconds": 30, "cpu_seconds": 20, "memory_bytes": 1073741824, "processes": 8, "threads": 32, "open_files": 128, "input_bytes": 1048576, "header_bytes": 65536, "message_bytes": 1048576, "scratch_bytes": 1048576, "output_bytes": 0}, + "output": {"root_id": null, "staging_policy": "none"}, + "authorization": {"event_id": "authorization-fixture-1", "actor_role": "checkpoint-security-operator", "created_at": "2026-09-02T12:00:00Z", "expires_at": "2026-09-02T12:05:00Z", "single_use_nonce": "0123456789abcdef0123456789abcdef", "single_use": true}, + "evidence": {"candidate_commit": "1111111111111111111111111111111111111111", "policy_digest": "sha256:9999999999999999999999999999999999999999999999999999999999999999"} +} diff --git a/docs/checkpoints/schemas/fixtures/v1/valid/trusted-worker-complete.json b/docs/checkpoints/schemas/fixtures/v1/valid/trusted-worker-complete.json new file mode 100644 index 0000000..d34bc13 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/valid/trusted-worker-complete.json @@ -0,0 +1,10 @@ +{ + "schema_id": "obliteratus.trusted-worker-message", + "schema_version": "1.0.0", + "message_type": "complete", + "sequence": 4, + "source_inventory_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "adapter_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "policy_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "payload": {"record_count": 4, "encoded_bytes": 4096, "transcript_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"} +} diff --git a/docs/checkpoints/schemas/fixtures/v1/valid/typed-offset-namespaces.json b/docs/checkpoints/schemas/fixtures/v1/valid/typed-offset-namespaces.json new file mode 100644 index 0000000..b42f92e --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/valid/typed-offset-namespaces.json @@ -0,0 +1,144 @@ +{ + "schema_id": "obliteratus.checkpoint-descriptor", + "schema_version": "1.0.0", + "descriptor_id": "fixture-typed-offset-namespaces", + "primary_format": "megatron_torch_dist", + "classification_confidence": "verified", + "components": [ + { + "component_id": "model", + "kind": "model", + "format": "megatron_torch_dist", + "producer": { + "name": "megatron-core", + "version": "fixture-version", + "format_version": "torch_dist", + "evidence_refs": ["ev-producer"] + }, + "state_scopes": ["model_weights"], + "topology_ref": "saved-topology", + "inventory_refs": ["shard-0"], + "tensor_fragment_refs": ["fragment-0"] + } + ], + "producer": { + "name": "megatron-core", + "version": "fixture-version", + "format_version": "torch_dist", + "evidence_refs": ["ev-producer"] + }, + "evidence": [ + { + "evidence_id": "ev-producer", + "subject": "/producer", + "kind": "explicit_metadata", + "file_ref": "shard-0", + "location": "fixture metadata producer", + "confidence": "verified", + "verifier": "neutral-fixture-generator-v1" + } + ], + "source_inventory": { + "inventory_id": "inventory-0", + "files": [ + { + "file_id": "shard-0", + "relative_path": "rank-0/model.safetensors", + "role": "model_fragment", + "size_bytes": 16, + "sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "regular_file": true, + "observation_id": "dev-1:ino-1:size-16:mtime-0" + } + ], + "total_bytes": 16, + "observation_complete": true + }, + "safety": { + "inspection_level": "trusted_metadata", + "trust_required": true, + "inventory_revalidated": true, + "unsafe_serialization_findings": [], + "violations": [] + }, + "state": { + "observed_scopes": ["model_weights"], + "classification": "weights_only" + }, + "topologies": [ + { + "topology_id": "saved-topology", + "kind": "saved", + "node_count": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "global_world_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "machine_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "global_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "local_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "tp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "pp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "dp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "cp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "ep_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]}, + "etp_size": {"value": null, "provenance": "unknown", "evidence_refs": []}, + "zero_stage": {"value": null, "provenance": "unknown", "evidence_refs": []} + } + ], + "tensor_fragments": [ + { + "fragment_id": "fragment-0", + "component_id": "model", + "fqn": "layer.weight", + "role": "parameter", + "dtype": "F32", + "global_shape": [2, 2], + "local_shape": [2, 2], + "element_offset": [0, 0], + "element_extent": [2, 2], + "padding": {"before": [0, 0], "after": [0, 0], "semantic": "none"}, + "shard_file_id": "shard-0", + "shard_digest_ref": "shard-0", + "fragment_digest": null, + "replica": {"group_id": null, "member_index": 0, "member_count": 1}, + "partition_axes": [], + "logical_tensor_id": "layer.weight", + "tie_group_id": null, + "shared_storage_id": null, + "storage_locations": [ + {"kind": "logical_element_range", "element_offset": [0, 0], "element_extent": [2, 2]}, + {"kind": "megatron_rank_offsets", "global_offset": [0, 0], "rank_offsets": [[0, 0, 1]]}, + {"kind": "safetensors_byte_range", "begin": 0, "end": 16}, + {"kind": "hf_weight_map", "tensor_name": "layer.weight", "file_id": "shard-0"} + ], + "evidence_refs": ["ev-producer"] + } + ], + "adapter_resolution": { + "status": "matched", + "adapter_id": "fixture-megatron", + "adapter_version": "0.0.0-fixture", + "capability_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "reason": "Neutral schema fixture only; not a product support claim." + }, + "conversion_plan": { + "eligible": true, + "target_format": "hf_safetensors", + "state_scope": "weights_only", + "dropped_scopes": [] + }, + "resource_estimate": { + "source_bytes": 16, + "logical_bytes": 16, + "output_bytes": 16, + "temporary_bytes": 32, + "peak_ram_bytes": 16, + "peak_vram_bytes": 0, + "file_count": 1, + "tensor_count": 1, + "shard_count": 1, + "assumptions": ["neutral fixture"], + "confidence": "verified", + "admission": "admitted" + }, + "support_decision": "conversion_supported", + "blockers": [] +} diff --git a/docs/checkpoints/schemas/fixtures/v1/valid/weights-only-conversion-manifest.json b/docs/checkpoints/schemas/fixtures/v1/valid/weights-only-conversion-manifest.json new file mode 100644 index 0000000..bb3fd31 --- /dev/null +++ b/docs/checkpoints/schemas/fixtures/v1/valid/weights-only-conversion-manifest.json @@ -0,0 +1,77 @@ +{ + "schema_id": "obliteratus.conversion-manifest", + "schema_version": "1.0.0", + "manifest_id": "fixture-manifest-weights-only", + "descriptor": { + "schema_version": "1.0.0", + "digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + }, + "source_inventory_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "source_files": [ + { + "relative_path": "rank-0/model.safetensors", + "size_bytes": 16, + "sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "adapter": { + "id": "fixture-adapter", + "version": "0.0.0-fixture", + "capability_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "producer": "fixture", + "producer_version": "0" + }, + "source_topology": {"topology_id": "saved-topology"}, + "state": { + "source_classification": "model_and_optimizer", + "output_classification": "weights_only", + "observed_scopes": ["model_weights", "optimizer"] + }, + "dropped_scopes": [ + {"scope": "optimizer", "reason": "Version 1 canonical output is model weights only."} + ], + "canonical_output": { + "format": "hf_safetensors", + "dtype_policy": "preserve", + "files": [ + { + "relative_path": "model.safetensors", + "size_bytes": 16, + "sha256": "sha256:4444444444444444444444444444444444444444444444444444444444444444" + } + ], + "hf_index": null, + "logical_tensor_count": 1, + "logical_bytes": 16 + }, + "resource_usage": { + "estimated_peak_ram_bytes": 16, + "actual_peak_ram_bytes": 16, + "estimated_temp_bytes": 32, + "actual_temp_bytes": 16 + }, + "validation": { + "coverage": true, + "replicas": true, + "ties": true, + "hashes": true, + "index": true, + "safe_reload": true, + "source_unchanged": true, + "result": "passed" + }, + "provenance": { + "obliteratus_commit": "e39f908832405ccad89cb2a5111e7c2576741d94", + "configuration_digest": null, + "tokenizer_digest": null, + "base_model": {"identity": null, "revision": null, "digest": null}, + "transformation_log": ["neutral fixture canonicalization"], + "unknowns": ["base model identity intentionally absent in neutral fixture"] + }, + "publication": { + "staging_validated": true, + "promoted": true, + "atomic_strategy": "sibling-staging-and-rename", + "rollback_result": "not_required" + } +} diff --git a/docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json b/docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json new file mode 100644 index 0000000..f5ca342 --- /dev/null +++ b/docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/peft-adapter-manifest-v1.schema.json", + "title": "OBLITERATUS PEFT Adapter Manifest v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_id", "schema_version", "adapter_name", "adapter_type", "adapter_format_version", "base_model", "rank", "alpha", "scaling", "dropout", "bias", "modules_to_save", "target_modules", "tie_policy", "merged", "key_map_digest", "model_card_digest", "key_map"], + "properties": { + "schema_id": {"const": "obliteratus.peft-adapter-manifest"}, + "schema_version": {"const": "1.0.0"}, + "adapter_name": {"$ref": "#/$defs/id"}, + "adapter_type": {"const": "lora"}, + "adapter_format_version": {"const": "peft-lora-v1"}, + "base_model": {"$ref": "#/$defs/baseModel"}, + "rank": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}, + "alpha": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}, + "scaling": {"type": "number", "exclusiveMinimum": 0}, + "dropout": {"type": "number", "minimum": 0, "exclusiveMaximum": 1}, + "bias": {"const": "none"}, + "modules_to_save": {"type": "array", "maxItems": 0}, + "target_modules": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "tie_policy": {"const": "base_model_declared"}, + "merged": {"const": false}, + "key_map_digest": {"$ref": "#/$defs/digest"}, + "model_card_digest": {"$ref": "#/$defs/digest"}, + "key_map": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/keyMap"}} + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "shape": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}}, + "baseModel": { + "type": "object", + "additionalProperties": false, + "required": ["repo_id", "revision", "weights_digest", "tokenizer_digest", "vocab_size", "architecture", "tied_embeddings"], + "properties": { + "repo_id": {"$ref": "#/$defs/id"}, + "revision": {"type": "string", "pattern": "^[0-9a-f]{40,64}$"}, + "weights_digest": {"$ref": "#/$defs/digest"}, + "tokenizer_digest": {"$ref": "#/$defs/digest"}, + "vocab_size": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}, + "architecture": {"$ref": "#/$defs/id"}, + "tied_embeddings": {"type": "boolean"} + } + }, + "keyMap": { + "type": "object", + "additionalProperties": false, + "required": ["module_name", "target_module", "lora_A_key", "lora_B_key", "lora_A_shape", "lora_B_shape", "rank"], + "properties": { + "module_name": {"$ref": "#/$defs/id"}, + "target_module": {"$ref": "#/$defs/id"}, + "lora_A_key": {"$ref": "#/$defs/id"}, + "lora_B_key": {"$ref": "#/$defs/id"}, + "lora_A_shape": {"$ref": "#/$defs/shape"}, + "lora_B_shape": {"$ref": "#/$defs/shape"}, + "rank": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807} + } + } + } +} diff --git a/docs/checkpoints/schemas/support-matrix-v1.schema.json b/docs/checkpoints/schemas/support-matrix-v1.schema.json new file mode 100644 index 0000000..5c5e5ed --- /dev/null +++ b/docs/checkpoints/schemas/support-matrix-v1.schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/support-matrix-v1.schema.json", + "title": "OBLITERATUS Checkpoint and Runtime Support Matrix v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_id", "schema_version", "generated_from", "status_vocabulary", "rows"], + "properties": { + "schema_id": {"const": "obliteratus.checkpoint-support-matrix"}, + "schema_version": {"const": "1.0.0"}, + "generated_from": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "status_vocabulary": { + "type": "array", + "const": ["supported", "conditional", "deferred", "out_of_scope"] + }, + "rows": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/row"} + } + }, + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "status": { + "type": "object", + "additionalProperties": false, + "required": ["value", "basis"], + "properties": { + "value": {"enum": ["supported", "conditional", "deferred", "out_of_scope"]}, + "basis": {"type": "string", "minLength": 1, "maxLength": 2048} + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["references", "candidate_commit", "fixture_digest", "environment", "topology", "retained_result"], + "properties": { + "references": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/id"}}, + "candidate_commit": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}$"}, + "fixture_digest": {"type": ["string", "null"], "pattern": "^sha256:[0-9a-f]{64}$"}, + "environment": {"type": ["string", "null"], "maxLength": 1024}, + "topology": {"type": ["string", "null"], "maxLength": 1024}, + "retained_result": {"type": ["string", "null"], "maxLength": 2048} + } + }, + "row": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "subject", + "format", + "producer_versions", + "adapter", + "model_mapping", + "state_scopes", + "safety_level", + "optional_extras", + "capabilities", + "canonical_output", + "evidence", + "limits" + ], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$"}, + "subject": {"$ref": "#/$defs/id"}, + "format": {"$ref": "#/$defs/id"}, + "producer_versions": {"type": "array", "items": {"$ref": "#/$defs/id"}}, + "adapter": {"type": ["string", "null"], "maxLength": 512}, + "model_mapping": {"type": "string", "maxLength": 2048}, + "state_scopes": {"type": "array", "items": {"$ref": "#/$defs/id"}}, + "safety_level": {"enum": ["ordinary_hf_load", "safe_structure", "trusted_metadata", "not_applicable"]}, + "optional_extras": {"type": "array", "items": {"$ref": "#/$defs/id"}}, + "capabilities": { + "type": "object", + "additionalProperties": false, + "required": ["detect", "safe_inspect", "trusted_inspect", "weights_canonicalize", "topology_reshard", "surgery", "exact_resume", "live_multi_node"], + "properties": { + "detect": {"$ref": "#/$defs/status"}, + "safe_inspect": {"$ref": "#/$defs/status"}, + "trusted_inspect": {"$ref": "#/$defs/status"}, + "weights_canonicalize": {"$ref": "#/$defs/status"}, + "topology_reshard": {"$ref": "#/$defs/status"}, + "surgery": {"$ref": "#/$defs/status"}, + "exact_resume": {"$ref": "#/$defs/status"}, + "live_multi_node": {"$ref": "#/$defs/status"} + } + }, + "canonical_output": {"type": ["string", "null"], "maxLength": 512}, + "evidence": {"$ref": "#/$defs/evidence"}, + "limits": {"type": "array", "minItems": 1, "items": {"type": "string", "maxLength": 2048}} + } + } + } +} diff --git a/docs/checkpoints/schemas/trusted-worker-message-v1.schema.json b/docs/checkpoints/schemas/trusted-worker-message-v1.schema.json new file mode 100644 index 0000000..a0b8e61 --- /dev/null +++ b/docs/checkpoints/schemas/trusted-worker-message-v1.schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://obliteratus.dev/schemas/trusted-worker-message-v1.schema.json", + "title": "OBLITERATUS Trusted Worker Message v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_id", "schema_version", "message_type", "sequence", "source_inventory_digest", "adapter_digest", "policy_digest", "payload"], + "properties": { + "schema_id": {"const": "obliteratus.trusted-worker-message"}, + "schema_version": {"const": "1.0.0"}, + "message_type": {"enum": ["metadata_record", "tensor_fragment_record", "error", "complete"]}, + "sequence": {"type": "integer", "minimum": 0, "maximum": 100000000}, + "source_inventory_digest": {"$ref": "#/$defs/digest"}, + "adapter_digest": {"$ref": "#/$defs/digest"}, + "policy_digest": {"$ref": "#/$defs/digest"}, + "payload": {"type": "object"} + }, + "allOf": [ + {"if": {"properties": {"message_type": {"const": "metadata_record"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/metadata"}}}}, + {"if": {"properties": {"message_type": {"const": "tensor_fragment_record"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/fragment"}}}}, + {"if": {"properties": {"message_type": {"const": "error"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/error"}}}}, + {"if": {"properties": {"message_type": {"const": "complete"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/complete"}}}} + ], + "$defs": { + "id": {"type": "string", "minLength": 1, "maxLength": 512}, + "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "metadata": { + "type": "object", + "additionalProperties": false, + "required": ["record_id", "record_type", "subject", "value", "evidence_refs"], + "properties": { + "record_id": {"$ref": "#/$defs/id"}, + "record_type": {"enum": ["producer", "topology", "state_scope", "tensor_metadata"]}, + "subject": {"type": "string", "pattern": "^/", "maxLength": 1024}, + "value": {"type": ["string", "integer", "boolean", "null"], "maxLength": 4096}, + "evidence_refs": {"type": "array", "maxItems": 1024, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}} + } + }, + "fragment": { + "type": "object", + "additionalProperties": false, + "required": ["fragment_id", "tensor_name", "dtype", "shape", "source_file_id", "source_byte_offset", "source_byte_length", "source_digest", "logical_axis", "logical_start", "logical_end"], + "properties": { + "fragment_id": {"$ref": "#/$defs/id"}, + "tensor_name": {"$ref": "#/$defs/id"}, + "dtype": {"$ref": "#/$defs/id"}, + "shape": {"type": "array", "maxItems": 64, "items": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}}, + "source_file_id": {"$ref": "#/$defs/id"}, + "source_byte_offset": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, + "source_byte_length": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}, + "source_digest": {"$ref": "#/$defs/digest"}, + "logical_axis": {"type": ["integer", "null"], "minimum": 0, "maximum": 63}, + "logical_start": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, + "logical_end": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807} + } + }, + "error": { + "type": "object", + "additionalProperties": false, + "required": ["code", "phase", "category", "subject_ids", "limit", "actual"], + "properties": { + "code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"}, + "phase": {"enum": ["policy", "preflight", "reader", "protocol", "validation", "materialization", "promotion", "evidence", "cleanup"]}, + "category": {"enum": ["unsupported", "trust", "source", "runtime", "resource", "protocol", "validation", "output", "evidence", "cleanup"]}, + "subject_ids": {"type": "array", "maxItems": 128, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}, + "limit": {"type": ["integer", "null"], "minimum": 0}, + "actual": {"type": ["integer", "null"], "minimum": 0} + } + }, + "complete": { + "type": "object", + "additionalProperties": false, + "required": ["record_count", "encoded_bytes", "transcript_digest"], + "properties": { + "record_count": {"type": "integer", "minimum": 0, "maximum": 100000000}, + "encoded_bytes": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}, + "transcript_digest": {"$ref": "#/$defs/digest"} + } + } + } +} diff --git a/docs/checkpoints/support-matrix-v1.json b/docs/checkpoints/support-matrix-v1.json new file mode 100644 index 0000000..b9ae833 --- /dev/null +++ b/docs/checkpoints/support-matrix-v1.json @@ -0,0 +1,237 @@ +{ + "schema_id": "obliteratus.checkpoint-support-matrix", + "schema_version": "1.0.0", + "generated_from": "5cc43c6e52903497574d80e08dff856028bc47f7", + "status_vocabulary": ["supported", "conditional", "deferred", "out_of_scope"], + "rows": [ + { + "id": "hf-safetensors-existing-loader", + "subject": "Existing Hugging Face-compatible safetensors input", + "format": "hf_safetensors", + "producer_versions": ["Transformers/Hugging Face compatible; exact model-family behavior varies"], + "adapter": null, + "model_mapping": "Existing Transformers model class and OBLITERATUS architecture gates", + "state_scopes": ["model_weights"], + "safety_level": "ordinary_hf_load", + "optional_extras": [], + "capabilities": { + "detect": {"value": "conditional", "basis": "The standalone inspector classifies bounded HF safetensors structure without loading payloads; model compatibility is separate."}, + "safe_inspect": {"value": "conditional", "basis": "Current bounded JSON and safetensors-header inspection emits structural evidence only."}, + "trusted_inspect": {"value": "out_of_scope", "basis": "Ordinary supported HF safetensors do not need the planned vendor metadata path."}, + "weights_canonicalize": {"value": "out_of_scope", "basis": "Input is already the planned canonical format."}, + "topology_reshard": {"value": "out_of_scope", "basis": "HF file indexes do not encode rank-fragment topology."}, + "surgery": {"value": "conditional", "basis": "Current behavior depends on model architecture, dtype, quantization, kernels, memory, and quality gates."}, + "exact_resume": {"value": "out_of_scope", "basis": "OBLITERATUS is not a training-resume system."}, + "live_multi_node": {"value": "out_of_scope", "basis": "The current loader/runner has no multi-host process topology."} + }, + "canonical_output": "Hugging Face-compatible safetensors from the current save path", + "evidence": { + "references": ["R11", "R12", "R13", "R21"], + "candidate_commit": "5cc43c6e52903497574d80e08dff856028bc47f7", + "fixture_digest": null, + "environment": "Local baseline source and existing repository tests; not a universal model-family qualification", + "topology": "One OBLITERATUS process; visible devices may receive complete modules", + "retained_result": null + }, + "limits": [ + "A registry entry is not proof of a successful model run.", + "Architecture-specific restrictions, including Qwen hybrid complete-model placement, still apply.", + "Structural inspection does not establish model-load, surgery, or producer compatibility." + ] + }, + { + "id": "accelerate-device-map-process-local", + "subject": "Accelerate device_map placement and CPU/disk offload", + "format": "runtime_placement_not_checkpoint_format", + "producer_versions": ["Existing project dependency lock"], + "adapter": null, + "model_mapping": "Complete modules placed across devices visible to one process", + "state_scopes": ["runtime_model_placement"], + "safety_level": "not_applicable", + "optional_extras": [], + "capabilities": { + "detect": {"value": "out_of_scope", "basis": "This is runtime placement, not an input format."}, + "safe_inspect": {"value": "out_of_scope", "basis": "This is runtime placement, not checkpoint inspection."}, + "trusted_inspect": {"value": "out_of_scope", "basis": "This is runtime placement, not checkpoint inspection."}, + "weights_canonicalize": {"value": "out_of_scope", "basis": "Placement does not reconstruct rank fragments."}, + "topology_reshard": {"value": "out_of_scope", "basis": "device_map has no saved rank-fragment contract."}, + "surgery": {"value": "conditional", "basis": "Current process-local path depends on model-family compatibility and resource headroom."}, + "exact_resume": {"value": "out_of_scope", "basis": "No training state is restored."}, + "live_multi_node": {"value": "out_of_scope", "basis": "One process and one host; device placement is not a launcher."} + }, + "canonical_output": null, + "evidence": { + "references": ["R14", "R21"], + "candidate_commit": "5cc43c6e52903497574d80e08dff856028bc47f7", + "fixture_digest": null, + "environment": "Existing OBLITERATUS single-process implementation", + "topology": "Single host, single OBLITERATUS process, one or more visible devices", + "retained_result": null + }, + "limits": [ + "Not rank-based pipeline parallelism.", + "Not distributed checkpoint intake.", + "Qwen hybrid models reject generic layer placement across devices." + ] + }, + { + "id": "pytorch-dcp-fsdp-planned", + "subject": "PyTorch DCP and FSDP model-weight intake", + "format": "pytorch_dcp_or_fsdp_state", + "producer_versions": ["Unselected; must be exact-version qualified"], + "adapter": "Deferred exact-version PyTorch adapter", + "model_mapping": "Requires a predeclared target model state schema", + "state_scopes": ["model_weights"], + "safety_level": "safe_structure", + "optional_extras": ["checkpoint-pytorch-planned"], + "capabilities": { + "detect": {"value": "conditional", "basis": "Current bounded inventory recognizes DCP/FSDP marker structure without reading DCP metadata."}, + "safe_inspect": {"value": "conditional", "basis": "Current inventory-only classification keeps DCP metadata opaque and trust-gated."}, + "trusted_inspect": {"value": "deferred", "basis": "Blocked on security-owner acceptance, exact-profile controls, and an exact PyTorch version."}, + "weights_canonicalize": {"value": "deferred", "basis": "No exact-version adapter has been qualified."}, + "topology_reshard": {"value": "deferred", "basis": "Only exact model-weight cases with retained fixture evidence may qualify."}, + "surgery": {"value": "deferred", "basis": "Requires successful canonical output then existing OBLITERATUS gates."}, + "exact_resume": {"value": "out_of_scope", "basis": "Version 1 emits weights only."}, + "live_multi_node": {"value": "out_of_scope", "basis": "Checkpoint intake is not distributed execution."} + }, + "canonical_output": "Planned HF safetensors plus conversion manifest", + "evidence": { + "references": ["R01", "R02", "R03", "R17", "R18", "R27", "R28", "R29"], + "candidate_commit": null, + "fixture_digest": null, + "environment": null, + "topology": null, + "retained_result": null + }, + "limits": ["No producer reader, adapter, conversion implementation, or qualified version band exists.", "Structural classification alone is not payload compatibility.", "Exact resume and arbitrary stateful/planner objects are excluded."] + }, + { + "id": "megatron-bridge-planned", + "subject": "Megatron distributed model-weight intake through a model-aware Bridge", + "format": "megatron_torch_dist_with_newer_forms_explicitly_deferred", + "producer_versions": ["Unselected; must be exact-version and format qualified"], + "adapter": "Deferred model-aware Megatron Bridge adapter", + "model_mapping": "Requires supported Megatron Bridge/provider mapping and reference configuration", + "state_scopes": ["model_weights"], + "safety_level": "safe_structure", + "optional_extras": ["checkpoint-megatron-planned"], + "capabilities": { + "detect": {"value": "conditional", "basis": "Current bounded inventory recognizes declared Megatron torch_dist marker structure without a vendor import."}, + "safe_inspect": {"value": "conditional", "basis": "Current inspection reports bounded structural facts only and does not interpret model semantics."}, + "trusted_inspect": {"value": "deferred", "basis": "Blocked on security-owner acceptance, exact-profile controls, and an exact vendor stack."}, + "weights_canonicalize": {"value": "deferred", "basis": "Blocked on common infrastructure and model-aware mapping."}, + "topology_reshard": {"value": "deferred", "basis": "Must be proven for exact model-weight topology and format."}, + "surgery": {"value": "deferred", "basis": "Requires canonical output and existing model-family gates."}, + "exact_resume": {"value": "out_of_scope", "basis": "Optimizer/resume compatibility is version/format-specific and outside v1."}, + "live_multi_node": {"value": "out_of_scope", "basis": "Bridge conversion is not an OBLITERATUS distributed runtime."} + }, + "canonical_output": "Planned HF safetensors plus conversion manifest", + "evidence": { + "references": ["R04", "R05", "R06", "R07", "R27", "R28"], + "candidate_commit": null, + "fixture_digest": null, + "environment": null, + "topology": null, + "retained_result": null + }, + "limits": ["Offsets alone never authorize generic QKV/MLP/expert reconstruction.", "No producer reader, adapter, conversion implementation, or qualified model/version exists.", "Structural classification alone is not payload compatibility."] + }, + { + "id": "deepspeed-zero-universal-planned", + "subject": "DeepSpeed ZeRO or Universal model-weight intake", + "format": "deepspeed_zero_or_universal", + "producer_versions": ["Unselected; must be exact-version qualified"], + "adapter": "Deferred exact-version DeepSpeed adapter", + "model_mapping": "Official consolidation or Universal mapping for an exact compatible case", + "state_scopes": ["model_weights"], + "safety_level": "safe_structure", + "optional_extras": ["checkpoint-deepspeed-planned"], + "capabilities": { + "detect": {"value": "conditional", "basis": "Current bounded inventory recognizes declared ZeRO or Universal marker structure without a DeepSpeed import."}, + "safe_inspect": {"value": "conditional", "basis": "Current inspection reports bounded structural facts only and keeps framework serialization opaque."}, + "trusted_inspect": {"value": "deferred", "basis": "Blocked on security-owner acceptance and exact-profile controls; official consolidation reads framework serialization."}, + "weights_canonicalize": {"value": "deferred", "basis": "No adapter has passed the resource and trust gates."}, + "topology_reshard": {"value": "deferred", "basis": "Universal behavior must be proven for the exact mapping/version."}, + "surgery": {"value": "deferred", "basis": "Requires canonical output and existing model gates."}, + "exact_resume": {"value": "out_of_scope", "basis": "Version 1 emits weights only."}, + "live_multi_node": {"value": "out_of_scope", "basis": "Checkpoint consolidation is not live OBLITERATUS execution."} + }, + "canonical_output": "Planned HF safetensors plus conversion manifest", + "evidence": { + "references": ["R08", "R09", "R10", "R17", "R27", "R28"], + "candidate_commit": null, + "fixture_digest": null, + "environment": null, + "topology": null, + "retained_result": null + }, + "limits": ["No producer reader, adapter, conversion implementation, or qualified version band exists.", "Structural classification alone is not payload compatibility.", "Input is trust-gated and peak CPU RAM/disk must be admitted."] + }, + { + "id": "peft-lora-safe-artifacts", + "subject": "PEFT LoRA safetensors inspection and exact-base artifact export", + "format": "peft_lora_safetensors", + "producer_versions": ["PEFT format contract; live compatibility remains version-conditional"], + "adapter": null, + "model_mapping": "Exact base revision, weights/tokenizer digests, architecture, vocabulary, ties, and full target-module paths are required", + "state_scopes": ["adapter_weights"], + "safety_level": "safe_structure", + "optional_extras": [], + "capabilities": { + "detect": {"value": "conditional", "basis": "Current inspector recognizes adapter_config.json with safetensors structure without loading payloads."}, + "safe_inspect": {"value": "conditional", "basis": "Current bounded JSON/header inspection preserves adapter and base identity evidence."}, + "trusted_inspect": {"value": "out_of_scope", "basis": "Canonical PEFT safetensors artifacts do not require a vendor metadata reader."}, + "weights_canonicalize": {"value": "out_of_scope", "basis": "LoRA artifact export is not distributed rank-fragment canonicalization."}, + "topology_reshard": {"value": "out_of_scope", "basis": "PEFT adapter tensors do not establish distributed checkpoint topology."}, + "surgery": {"value": "conditional", "basis": "The Python exporter writes canonical PEFT files only when exact base identity is supplied; the default pipeline otherwise writes a truthfully unsupported safe artifact."}, + "exact_resume": {"value": "out_of_scope", "basis": "Adapter artifacts do not contain complete training-resume state."}, + "live_multi_node": {"value": "out_of_scope", "basis": "Adapter export is an offline artifact operation."} + }, + "canonical_output": "adapter_model.safetensors, adapter_config.json, adapter_manifest.json, provenance, and model card", + "evidence": { + "references": ["R16", "R27"], + "candidate_commit": null, + "fixture_digest": null, + "environment": "Mandatory CPU artifact tests plus a conditional upstream PEFT loader check; not a universal base-model qualification", + "topology": "Offline single-process adapter artifact export", + "retained_result": null + }, + "limits": [ + "Exact base and tokenizer identity are mandatory for a canonical claim.", + "The default pipeline does not invent missing identity and emits a safely serialized unsupported artifact instead.", + "Live base-model behavior remains conditional on the exact PEFT, Transformers, model, and runtime versions." + ] + }, + { + "id": "live-multi-node-surgery-research", + "subject": "Live multi-node OBLITERATUS surgery", + "format": "runtime_not_checkpoint_format", + "producer_versions": [], + "adapter": null, + "model_mapping": "Fixed-membership preflight only; model execution mapping is unimplemented", + "state_scopes": ["runtime_execution"], + "safety_level": "not_applicable", + "optional_extras": [], + "capabilities": { + "detect": {"value": "out_of_scope", "basis": "Runtime capability is not a source format."}, + "safe_inspect": {"value": "out_of_scope", "basis": "Runtime capability is not checkpoint inspection."}, + "trusted_inspect": {"value": "out_of_scope", "basis": "Runtime capability is not checkpoint inspection."}, + "weights_canonicalize": {"value": "out_of_scope", "basis": "Offline conversion is a separate subsystem."}, + "topology_reshard": {"value": "out_of_scope", "basis": "Runtime ownership does not establish checkpoint conversion."}, + "surgery": {"value": "deferred", "basis": "Preflight and CPU/Gloo protocol contracts are available; model loading and surgery payloads are not."}, + "exact_resume": {"value": "out_of_scope", "basis": "No training-resume product is planned."}, + "live_multi_node": {"value": "deferred", "basis": "Candidate preflight only; no exact physical profile, multi-host qualification, or supported model workflow exists."} + }, + "canonical_output": null, + "evidence": { + "references": ["R06", "R15", "R21"], + "candidate_commit": null, + "fixture_digest": null, + "environment": null, + "topology": null, + "retained_result": null + }, + "limits": ["Current --remote runs one OBLITERATUS process on one SSH host.", "Canonicalizing a distributed checkpoint does not make surgery distributed.", "The distributed command validates fixed membership and exits before model allocation."] + } + ] +} diff --git a/docs/checkpoints/support-runbook.md b/docs/checkpoints/support-runbook.md new file mode 100644 index 0000000..07d15ce --- /dev/null +++ b/docs/checkpoints/support-runbook.md @@ -0,0 +1,203 @@ +# Support runbook: checkpoints, placement, and distributed-state reports + +**Artifact ID:** SUPPORT-DCI-001 +**Version:** 0.3.0 +**Status:** Wave 2 safe structural inspection available; producer conversion deferred +**Owner:** OBLITERATUS maintainers +**Tracking:** Use the repository issue and pull-request workflow; include sanitized evidence only. + +## 1. Service overview + +This runbook supports the current HF-compatible loader, process-local device +placement/offload, one-host remote runner, and reports involving distributed +checkpoint formats. The bounded structural inspector is current; producer +readers, adapters, trusted execution, and producer-backed conversion are not. +See the [guide](distributed-checkpoint-intake.md) and [support +matrix](support-matrix-v1.json). + +## 2. Escalation ownership + +| Report class | Primary owner | Escalate when | +|---|---|---| +| Model-capacity / “multi node offset” report | Distributed-runtime triage | Workflow exceeds one qualified host or exact model/topology is unknown | +| Descriptor/format/topology contract | Architecture | New format or overloaded term appears | +| Trusted-reader or filesystem boundary | Security | Report requires vendor/Python metadata reader | +| Reconstruction correctness | Test and data | Gap/overlap/replica/tie/padding behavior is unclear | +| Writer/recovery/resources | Persistence | ENOSPC, partial staging, or whole-state memory risk | +| Producer adapter | Adapter owner | Exact supported version/model mapping is identified | +| Provenance/PEFT | Data | Base/config/tokenizer/adapter identity is incomplete | +| Support claim | Release and documentation | A matrix status or public claim would change | +| Live multi-node runtime | Architecture and security | Reporter needs multiple participating hosts/processes | + +## 3. Operational readiness checklist + +- [ ] Record exact OBLITERATUS commit and installed package versions. +- [ ] Classify the request using the glossary before suggesting remediation. +- [ ] Run `obliteratus checkpoint inspect SOURCE --json` only when the source can + be handled under the local structural-inspection policy. +- [ ] Confirm whether the input is ordinary HF, DCP/FSDP, Megatron, DeepSpeed, + PEFT, or ambiguous. +- [ ] Confirm whether the goal is loading, conversion, surgery, export, + inference, or exact resume. +- [ ] Check the machine support row and its limitations. +- [ ] Do not describe a deferred/planned row as available. +- [ ] Do not cross the trusted-reader boundary during triage; no exact profile, + residual-risk acceptance, reader, adapter, or payload execution is part + of the current capability. +- [ ] Link evidence to the exact public issue or pull request and candidate commit. + +## 4. Monitoring and alerts + +There is no persistent intake service in version 1. For inspection, monitor the +CLI exit status and descriptor blockers; stable `DCI_*` codes identify the first +failed structural phase. For current model runs, monitor existing CLI logs, +stage transitions, host RAM/disk/VRAM, source and output paths, and validation +results. Alert thresholds and producer conversion metrics are not claimed. + +## 5. Common scenarios + +### A. “Model needs multi node offset” + +1. Route the complaint primarily to live model-capacity triage; record that + this remains an interpretation, + not a reporter-defined standard term. +2. Request the exact model/revision, workflow stage, current memory/topology, + desired host count, failure output, and sanitized environment evidence. +3. Do not infer a checkpoint producer or adapter from the phrase. Treat a DCP, + Megatron, or ZeRO/UCP artifact as a separate request that requires independent + exact producer evidence. +4. Treat an explicit affine/reference-mean activation offset hypothesis as a + separate algorithm question. +5. State that current model support remains single-process placement/offload or + one process on one remote host; the distributed command is preflight only. + +### B. HF model does not fit on one device + +1. Confirm model-family placement restrictions and current CLI help. +2. For compatible families, use existing single-process placement/offload or a + supported quantization mode with sufficient headroom. +3. Do not call `device_map` rank sharding or multi-node execution. +4. Preserve the Qwen hybrid one-device restriction. + +### C. DCP, Megatron, or DeepSpeed directory supplied today + +1. State that no current OBLITERATUS producer adapter is qualified. +2. Run the structural inspector only; a successful classification is not a load + or conversion result. +3. Do not invoke an unfamiliar vendor reader as a diagnostic shortcut. +4. Collect only sanitized structural evidence from the descriptor. +5. Record the deferred adapter capability and offer external producer-supported + conversion only as an operator-controlled workaround. + +### D. Conversion or save runs out of RAM/disk + +1. Preserve the source and any prior valid output. +2. Record stage, normalized error, host resource totals/free capacity, source + logical size estimate, and staging status without private local identifiers. +3. Do not promote or reuse incomplete staging as valid output. +4. Classify full-state export pressure separately from common-writer failures. + +### E. Request for exact training resume + +Explain that the planned v1 output is model weights only. Exact resume normally +requires producer-specific optimizer, scheduler, RNG/scaler, progress, and data- +position state [R19–R20](references.md#primary-and-upstream-sources). + +### F. Trusted metadata is requested + +1. State that the planned trusted-reader path is not currently implemented or + accepted for production use. +2. Do not treat locality, filenames, prior structural inspection, a checksum, + or `weights_only=True` as trust [R27–R29](references.md#security-and-containment-sources). +3. When implemented, require fresh explicit intent plus an exact single-use + source/operation/runtime/isolation/resource-bound policy. +4. Refuse when any isolation/runtime capability is unavailable; never suggest a + weaker subprocess/container fallback or integrity override. +5. Prefer a producer-side safetensors export performed in the operator's already + trusted environment when an approved reader profile is unavailable. +6. Require security review of the exact source/runtime/profile/fixture decision; + keep adapter rows deferred until the complete evidence boundary passes. + +## 6. Troubleshooting procedure + +1. Capture `git rev-parse HEAD` and `python3 --version`. +2. Capture current CLI syntax with `python3 -m obliteratus --help`; do not rely on + examples that the parser rejects. +3. Identify the first failing stage: structural inventory/probe, current load, + placement, pristine check, surgery, save, reload, or a gated intake phase. +4. Compare the report to the support matrix and glossary. +5. Check architecture/model restrictions before changing placement. +6. Record normalized relative paths or opaque IDs, file sizes, safe digests, + versions, topology facts/provenance, and resource estimates. +7. Reproduce only with project-owned or explicitly approved fixtures. +8. Escalate to the owner table with exact evidence and a no-mutation statement. + +## 7. Recovery and rollback + +Current producer-neutral writes use sibling staging, validate all outputs, +recheck source identity, then promote. Existing model runs retain their current +persistence behavior. + +- If inspection fails: no output should exist. +- If an exact registered capability reports a missing or incompatible optional + dependency: retain its `adapter_resolution.reason` diagnostic, install only + the named exact reviewed extra/version in the intended disposable profile, + and retry from an unchanged source. A dependency match does not satisfy the + separate trust-policy or profile-approval gates. +- If materialization fails: source/prior output remain unchanged; staging is not + success. +- If promotion validation fails: the common writer restores the prior + destination and removes owned staging; never overwrite evidence silently. +- If trust, source identity, runtime, containment, redaction, evidence, cleanup, + or security-baseline validation fails: promote nothing, allow no bypass, and + create a fresh attempt only after the cause is corrected. +- If an adapter support regression appears: remove/defer its static support row + and registry entry; ordinary HF loading remains available. + +## 8. Change management + +- Contract/schema changes require architecture and traceability review plus a + new schema version when incompatible. +- Adapter changes require exact producer-version fixtures and retained evidence. +- Trusted-reader/profile changes require an exact threat model, degraded-mode + review, security tests, and explicit residual-risk acceptance. +- Support status changes require release approval and offline contract validation. +- Live multi-node claims require architecture and security approval plus exact + multi-host evidence. +- Delivery follows repository PR, signed-commit, and CI policy. + +## 9. Communication templates + +### Unsupported format + +> OBLITERATUS identified this as `{format}`, which is `{status}` in support +> matrix v1. No source or prior output was changed. The blocking contract is +> `{code}` at `{phase}`. Continue with `{sanitized next action}`. + +### Evidence pending + +> The upstream framework documents this capability, but OBLITERATUS has no +> exact-version/model/topology qualification at the candidate commit. The row +> remains deferred until the required evidence passes. + +### Conversion versus runtime + +> Converting rank-sharded model weights into HF safetensors is an offline input +> step. It does not make the OBLITERATUS surgery process multi-node; that runtime +> remains a separate, unsupported capability. + +## 10. Post-incident activities + +- Preserve descriptor/manifest and test result digests. +- Record exact commit, versions, topology, resource admission/actuals, failure + code/phase, source-immutability result, and output-promotion result. +- Add a minimal project-owned regression fixture. +- Update risk, traceability, support matrix, and runbook if the contract changed. +- Never turn one successful case into an unqualified universal support claim. + +## 11. Runbook maintenance + +Review with each adapter version-band change, schema version, release candidate, +and incident. Security owns trust-boundary text; architecture owns terminology; +test/release own evidence status; support owns scenario clarity. Offline checks +must continue validating local links, CLI examples, and support-matrix contracts. diff --git a/docs/distributed-preflight.md b/docs/distributed-preflight.md new file mode 100644 index 0000000..fc18e27 --- /dev/null +++ b/docs/distributed-preflight.md @@ -0,0 +1,182 @@ +# Distributed preflight contract + +`obliteratus distributed preflight PROFILE.json` is the only distributed entry +point currently implemented. It consumes a worker group that an external +trusted scheduler has already launched. It does not launch `torchrun`, SSH to a +peer, provision hosts, install packages, relay credentials, load model weights, +or perform surgery. + +The command is an admission gate, not a supported multi-node workflow. Native +Llama tensor-parallel loading, distributed PROBE/DISTILL/EXCISE/VERIFY, export, +and physical two-host qualification remain unimplemented and unqualified. +NCCL, Gloo, and rendezvous authentication or encryption are not claimed. + +## Invocation boundary + +The scheduler must provide all of the following to every rank: + +- `RANK`, `LOCAL_RANK`, `WORLD_SIZE`, and `LOCAL_WORLD_SIZE`; +- `GROUP_RANK`, `ROLE_RANK`, and `ROLE_WORLD_SIZE`; +- `MASTER_ADDR` and `MASTER_PORT`; +- `TORCHELASTIC_RUN_ID`, `TORCHELASTIC_RESTART_COUNT=0`, and + `TORCHELASTIC_MAX_RESTARTS=0`; +- a fresh `OBLITERATUS_RUN_ID` distinct from the rendezvous ID; and +- exact `GLOO_SOCKET_IFNAME` and `NCCL_SOCKET_IFNAME` values matching the + reviewed profile. + +Ordinary `run`, `obliterate`, and one-host SSH commands never inspect these +variables to infer distributed intent. The distributed command accepts only +the local profile path and optional `--json`; secrets and operational overrides +are not CLI inputs. + +## Closed profile schema + +The profile is bounded UTF-8 JSON. Every section and field is required; unknown +or duplicate fields fail closed. The following uses non-operational example +values and is not an accepted physical-host profile: + +```json +{ + "schema_version": 1, + "run": { + "run_id": "11111111111111111111111111111111", + "rendezvous_id": "22222222222222222222222222222222", + "world_size": 2, + "local_world_size": 1 + }, + "identity": { + "source_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "model_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "tokenizer_digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "commit_sha": "dddddddddddddddddddddddddddddddddddddddd", + "code_digest": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "topology": { + "tensor_parallel_size": 2, + "coordinator_rank": 0, + "placement_plan_digest": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "dimension_divisors": [4096, 11008] + }, + "network": { + "master_addr": "10.10.0.10", + "master_port": 29500, + "interface": "eth0", + "allowed_master_cidrs": ["10.10.0.0/24"] + }, + "source": {"path": "/srv/immutable/model"}, + "staging": { + "path": "/srv/obliteratus/staging", + "storage_digest": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "resources": { + "min_free_device_memory_bytes": 8589934592, + "min_free_host_memory_bytes": 17179869184, + "min_free_staging_bytes": 107374182400, + "max_source_files": 10000, + "max_source_bytes": 1099511627776, + "max_source_file_bytes": 274877906944 + }, + "timeouts": { + "source_seconds": 1800, + "init_seconds": 60, + "collective_seconds": 60, + "teardown_seconds": 15 + }, + "software": { + "python": "3.12.11", + "platform": "Linux-6.18.0-x86_64-with-glibc2.42", + "machine": "x86_64", + "torch": "2.13.0", + "transformers": "5.15.0", + "accelerate": "1.12.0", + "safetensors": "0.7.0", + "cuda": "13.0", + "nccl": "2.28.3", + "driver": "13000" + }, + "execution": { + "device_kind": "cuda", + "device_name": "REVIEWED-EXACT-GPU-NAME", + "compute_capability": "REVIEWED-EXACT-CAPABILITY", + "evidence_tier": "candidate_preflight", + "allowed_environment_keys": ["HOME", "LANG", "PATH", "CUDA_VISIBLE_DEVICES"], + "local_files_only": true, + "trust_remote_code": false, + "allow_runtime_install": false, + "allow_plugins": false, + "allow_compilation": false, + "allow_adapters": false, + "allow_quantization": false + }, + "evidence": {"path": "/srv/obliteratus/staging/11111111111111111111111111111111/preflight.json"} +} +``` + +The source tree must be an immutable, local, non-symlink directory containing +a structurally valid canonical Hugging Face safetensors checkpoint, immutable +tokenizer metadata, and only closed inert file types. The existing bounded +safe-structure inspector validates safetensors headers and rejects adapters, +pickle formats, aliases, links, and mutable files without deserializing tensor +payloads. Content hashes are streamed without loading tensor data. The +three source digests are canonical hashes of the complete file inventory, the +safetensors subset, and tokenizer-named files respectively. + +`commit_sha` binds Git history while `code_digest` independently binds the +Python files actually available to the process, including dirty-tree changes. +`storage_digest` is not a label: each worker derives it from the exact Linux +mountinfo record backing the staging path. The selected interface must own an +address inside the private allowlist, and the coordinator's master address must +be bound to that interface. All inherited environment keys must appear in the +closed allowlist; credential-, cloud-, token-, and proxy-shaped keys can never +be allowlisted. + +## Admission and failure behavior + +Before returning `preflighted`, every rank must agree on the run, config, +source, model, tokenizer, commit, software, storage, placement, topology, and +network-interface identities. Global devices and per-host local ranks must be +unique. Device, host RAM, and staging capacity must meet the exact integer +floors. Rank zero exclusively reserves `staging/` and every rank must +observe an atomic shared marker there. + +The selected interface must own an address inside the private allowlist before +Gloo initialization is attempted; the coordinator must also own the configured +numeric rendezvous address on that interface. All local preflight probes are +bounded by the smaller configured source/collective deadline. Source traversal +and structural inspection nest under that same process-level wall-clock timer, +including payload hashing and revalidation. A runtime that cannot enforce that +deadline refuses the attempt. + +Any missing rank, disagreement, timeout, backend error, rank loss, or uncertain +cleanup fails the whole attempt. Restarts and in-memory resume are forbidden; +retry requires fresh run and rendezvous IDs and a complete new preflight. +Cleanup uncertainty is `quarantined`. + +Evidence is bounded, mode `0600`, atomically created, and never overwritten. +The path is fixed to `staging//preflight.json`. While Gloo is live, +every rank reads and votes on one canonical `PREFLIGHTED/PREPARED` lifecycle +record. This record has a distinct stage-message schema and cannot be decoded +as terminal success evidence. After bounded group destruction, each rank +creates a private `PREFLIGHTED/COMMITTED` teardown acknowledgement; rank zero +publishes terminal success only after validating the complete fixed-rank set. +Failures emit `ABORTING` and then `ABORTED` or `QUARANTINED` lifecycle receipts. +Missing or invalid receipts produce `LMS_CLEANUP_INCOMPLETE`, never success. + +An FD-level guard is active across backend initialization, collectives, and +teardown. Raw native/backend stderr is discarded; any emitted bytes fail the +attempt as `LMS_DIAGNOSTIC_REDACTION_FAILED`. Terminal evidence contains only +allowlisted state, counts, stable error codes, the mandatory `protocol_cpu` or +`candidate_preflight` scope label, and opaque digests—not raw endpoints, +hostnames, device identifiers, paths, environment mappings, exceptions, +prompts, tensors, or credentials. + +If process-group destruction exceeds its explicit bound, the affected +externally launched worker exits with fixed status `70` before the FD guard is +restored. A Python teardown thread is never allowed to outlive containment. +Missing teardown acknowledgement then forces coordinator-side +`LMS_CLEANUP_INCOMPLETE` and `QUARANTINED`; torchrun restart remains disabled. + +The implementation and tests define a fail-closed candidate boundary only. No +exact physical-host profile, residual-risk acceptance, authenticated or encrypted +transport, GPU/NCCL qualification, model payload, or production support claim is +included. diff --git a/obliteratus/__init__.py b/obliteratus/__init__.py index 6248a92..ac1c0da 100644 --- a/obliteratus/__init__.py +++ b/obliteratus/__init__.py @@ -23,6 +23,7 @@ __all__ = [ "get_watchtower", "AutoObliterator", "RunArchive", + "CheckpointService", ] @@ -84,4 +85,7 @@ def __getattr__(name): if name == "RunArchive": from obliteratus.run_archive import RunArchive return RunArchive + if name == "CheckpointService": + from obliteratus.checkpoint_service import CheckpointService + return CheckpointService raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/obliteratus/abliterate.py b/obliteratus/abliterate.py index 9e68724..6daaff9 100644 --- a/obliteratus/abliterate.py +++ b/obliteratus/abliterate.py @@ -7760,9 +7760,19 @@ class AbliterationPipeline: ) if self._lora_adapters: - from obliteratus.lora_ablation import save_lora_adapters - adapter_path = save_lora_adapters(self._lora_adapters, checkpoint_dir) - self.log(f"Saved LoRA adapters to {adapter_path}") + from obliteratus.lora_ablation import save_unsupported_obliteratus_adapters + adapter_path = save_unsupported_obliteratus_adapters( + self._lora_adapters, + checkpoint_dir, + reason=( + "This run did not retain an exact base-model commit, weights digest, " + "tokenizer digest, vocabulary, and architecture identity." + ), + ) + self.log( + "Saved a safe, explicitly unsupported OBLITERATUS adapter artifact to " + f"{adapter_path}; no PEFT compatibility claim was made." + ) def _rebirth(self) -> Path: """Atomically save the abliterated model with comprehensive metadata.""" diff --git a/obliteratus/checkpoint_capabilities.py b/obliteratus/checkpoint_capabilities.py new file mode 100644 index 0000000..b8d8b41 --- /dev/null +++ b/obliteratus/checkpoint_capabilities.py @@ -0,0 +1,334 @@ +"""Closed, inert checkpoint capability and dependency diagnostics. + +This module resolves declarative capability metadata only. It never imports an +optional producer framework, discovers plugins, reads checkpoint payloads, or +authorizes a trusted operation. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from hashlib import sha256 +from importlib import metadata +from typing import Any + +from obliteratus import __version__ + + +_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +_VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.+!-]*$") +_TRUST_REQUIRED_FORMATS = frozenset( + { + "hf_pytorch_pickle", + "pytorch_dcp", + "fsdp_state_dict", + "megatron_torch_dist", + "megatron_torch_dcp", + "megatron_fsdp_dtensor", + "deepspeed_zero", + "deepspeed_universal", + } +) + + +def _require_identifier(value: str, field: str) -> None: + if ( + not isinstance(value, str) + or len(value) > 64 + or _IDENTIFIER.fullmatch(value) is None + ): + raise ValueError(f"{field} must be a portable package identifier") + + +def _require_version(value: str, field: str) -> None: + if ( + not isinstance(value, str) + or len(value) > 64 + or not any(character.isdigit() for character in value) + or _VERSION.fullmatch(value) is None + ): + raise ValueError(f"{field} must be an exact portable version") + + +@dataclass(frozen=True, order=True) +class ExactDependency: + """One distribution and the only version accepted by a capability.""" + + distribution: str + version: str + + def __post_init__(self) -> None: + _require_identifier(self.distribution, "distribution") + _require_version(self.version, "version") + + @property + def requirement(self) -> str: + return f"{self.distribution}=={self.version}" + + +@dataclass(frozen=True) +class AdapterCapability: + """Inert metadata for one future exact adapter contract. + + Possessing this record does not provide an adapter, reader, trust profile, + or authorization. Product registrations remain empty until a separately + approved adapter supplies an exact record. + """ + + adapter_id: str + adapter_version: str + producer: str + producer_version: str + formats: tuple[str, ...] + required_extras: tuple[str, ...] + required_dependencies: tuple[ExactDependency, ...] + + def __post_init__(self) -> None: + if any( + type(value) is not tuple + for value in (self.formats, self.required_extras, self.required_dependencies) + ): + raise TypeError("capability collections must be immutable tuples") + _require_identifier(self.adapter_id, "adapter_id") + _require_version(self.adapter_version, "adapter_version") + _require_identifier(self.producer, "producer") + _require_version(self.producer_version, "producer_version") + if not self.formats: + raise ValueError("formats must not be empty") + if not self.required_extras: + raise ValueError("required_extras must not be empty") + if not self.required_dependencies: + raise ValueError("required_dependencies must not be empty") + if len(self.formats) > 8: + raise ValueError("formats exceeds the bounded capability limit") + if len(self.required_extras) > 4: + raise ValueError("required_extras exceeds the bounded capability limit") + if len(self.required_dependencies) > 4: + raise ValueError("required_dependencies exceeds the bounded capability limit") + for checkpoint_format in self.formats: + _require_identifier(checkpoint_format, "format") + if checkpoint_format not in _TRUST_REQUIRED_FORMATS: + raise ValueError("capabilities may target only trust-required formats") + for extra in self.required_extras: + _require_identifier(extra, "required_extra") + if len(set(self.formats)) != len(self.formats): + raise ValueError("formats must be unique") + if len(set(self.required_extras)) != len(self.required_extras): + raise ValueError("required_extras must be unique") + if not all(isinstance(item, ExactDependency) for item in self.required_dependencies): + raise TypeError("required_dependencies must contain ExactDependency records") + distributions = [item.distribution for item in self.required_dependencies] + if len(set(distributions)) != len(distributions): + raise ValueError("required dependency distributions must be unique") + + @property + def capability_digest(self) -> str: + record = { + "adapter_id": self.adapter_id, + "adapter_version": self.adapter_version, + "producer": self.producer, + "producer_version": self.producer_version, + "formats": sorted(self.formats), + "required_extras": sorted(self.required_extras), + "required_dependencies": [ + {"distribution": item.distribution, "version": item.version} + for item in sorted(self.required_dependencies) + ], + } + payload = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() + return f"sha256:{sha256(payload).hexdigest()}" + + +@dataclass(frozen=True) +class AdapterResolution: + """Descriptor-compatible result of an inert registry lookup.""" + + status: str + adapter_id: str | None + adapter_version: str | None + capability_digest: str | None + reason: str + dependency_unavailable: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "adapter_id": self.adapter_id, + "adapter_version": self.adapter_version, + "capability_digest": self.capability_digest, + "reason": self.reason, + } + + +def _installed_version(distribution: str) -> str | None: + try: + return metadata.version(distribution) + except metadata.PackageNotFoundError: + return None + + +def _render_observed(value: str | None) -> str: + if value is None: + return "" + if ( + not isinstance(value, str) + or len(value) > 64 + or not any(character.isdigit() for character in value) + or _VERSION.fullmatch(value) is None + ): + return "" + return value + + +@dataclass(frozen=True) +class AdapterRegistry: + """A closed table; resolution performs no plugin or package imports.""" + + capabilities: tuple[AdapterCapability, ...] = () + + def __post_init__(self) -> None: + if type(self.capabilities) is not tuple: + raise TypeError("capabilities must be an immutable tuple") + if not all(isinstance(item, AdapterCapability) for item in self.capabilities): + raise TypeError("capabilities must contain AdapterCapability records") + if len(self.capabilities) > 16: + raise ValueError("adapter registry exceeds the bounded capability limit") + identifiers = [item.adapter_id for item in self.capabilities] + if len(set(identifiers)) != len(identifiers): + raise ValueError("adapter capability identifiers must be unique") + + def resolve( + self, + checkpoint_format: str, + *, + version_provider: Callable[[str], str | None] = _installed_version, + project_name: str = "obliteratus", + project_version: str = __version__, + observed_producer: str | None = None, + observed_producer_version: str | None = None, + ) -> AdapterResolution: + """Resolve one format and report exact dependency state without importing it.""" + _require_identifier(checkpoint_format, "checkpoint_format") + _require_identifier(project_name, "project_name") + _require_version(project_version, "project_version") + if (observed_producer is None) != (observed_producer_version is None): + raise ValueError("observed producer and version must be provided together") + if observed_producer is not None and observed_producer_version is not None: + _require_identifier(observed_producer, "observed_producer") + _require_version(observed_producer_version, "observed_producer_version") + format_matches = tuple( + sorted( + ( + capability + for capability in self.capabilities + if checkpoint_format in capability.formats + ), + key=lambda capability: capability.adapter_id, + ) + ) + if not format_matches: + return AdapterResolution( + status="missing", + adapter_id=None, + adapter_version=None, + capability_digest=None, + reason=( + "No approved exact capability is registered; structural recognition " + "does not select an adapter or dependency set." + ), + ) + source_identity_verified = observed_producer is not None + matches = ( + tuple( + capability + for capability in format_matches + if capability.producer == observed_producer + and capability.producer_version == observed_producer_version + ) + if source_identity_verified + else format_matches + ) + if not matches: + return AdapterResolution( + status="missing", + adapter_id=None, + adapter_version=None, + capability_digest=None, + reason=( + "No exact capability matches the observed producer/version; " + "do not install or select a format-only candidate." + ), + ) + if len(matches) > 1: + identifiers = ",".join(item.adapter_id for item in matches) + return AdapterResolution( + status="ambiguous", + adapter_id=None, + adapter_version=None, + capability_digest=None, + reason=f"Multiple exact capabilities match: {identifiers}.", + ) + + capability = matches[0] + observed = { + dependency.distribution: version_provider(dependency.distribution) + for dependency in sorted(capability.required_dependencies) + } + unavailable = [ + dependency + for dependency in sorted(capability.required_dependencies) + if observed[dependency.distribution] != dependency.version + ] + if unavailable: + extras = ",".join(sorted(capability.required_extras)) + install_extra = f"{project_name}[{extras}]=={project_version}" + requirements = ",".join( + item.requirement for item in sorted(capability.required_dependencies) + ) + observed_versions = ",".join( + f"{item.distribution}={_render_observed(observed[item.distribution])}" + for item in sorted(capability.required_dependencies) + ) + return AdapterResolution( + status="missing", + adapter_id=capability.adapter_id, + adapter_version=capability.adapter_version, + capability_digest=capability.capability_digest, + reason=( + ( + "source_identity=verified; " + if source_identity_verified + else "source_identity=unverified; " + ) + + "dependency_status=missing_or_incompatible; " + f"install_extra={install_extra}; required_versions={requirements}; " + f"observed_versions={observed_versions}" + ), + dependency_unavailable=True, + ) + if not source_identity_verified: + return AdapterResolution( + status="missing", + adapter_id=capability.adapter_id, + adapter_version=capability.adapter_version, + capability_digest=capability.capability_digest, + reason=( + "source_identity=unverified; exact producer/version evidence is " + "required before this format-only capability candidate can match." + ), + ) + return AdapterResolution( + status="matched", + adapter_id=capability.adapter_id, + adapter_version=capability.adapter_version, + capability_digest=capability.capability_digest, + reason="Exact capability dependencies are present; trust authorization is still required.", + ) + + +def registry_from(capabilities: Iterable[AdapterCapability]) -> AdapterRegistry: + """Construct a deterministic closed registry from explicit records.""" + return AdapterRegistry(tuple(sorted(capabilities, key=lambda item: item.adapter_id))) diff --git a/obliteratus/checkpoint_errors.py b/obliteratus/checkpoint_errors.py new file mode 100644 index 0000000..b5abfef --- /dev/null +++ b/obliteratus/checkpoint_errors.py @@ -0,0 +1,176 @@ +"""Stable, sanitized failures for checkpoint inspection and conversion contracts.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + + +_ERROR_CONTRACTS = { + "DCI_UNSUPPORTED_FORMAT_OR_VERSION": ( + "unsupported", + "classification", + False, + "Provide an identified supported format or a safe canonical export.", + ), + "DCI_TRUST_POLICY_REQUIRED": ( + "trust", + "policy", + False, + "Use structural inspection or obtain separate approval for an exact trust profile.", + ), + "DCI_TRUST_POLICY_MISMATCH": ( + "trust", + "policy", + False, + "Use the exact approved policy identity or stop the trusted operation.", + ), + "DCI_SOURCE_BOUNDARY_VIOLATION": ( + "source", + "source", + False, + "Repair the immutable local source boundary and inspect it again.", + ), + "DCI_SOURCE_CHANGED": ( + "source", + "source", + True, + "Retry with an immutable source snapshot.", + ), + "DCI_TRUST_RUNTIME_UNAVAILABLE": ( + "runtime", + "preflight", + True, + "Provision the exact approved disposable runtime before retrying.", + ), + "DCI_RUNTIME_IDENTITY_MISMATCH": ( + "runtime", + "preflight", + False, + "Use the exact approved runtime identity and verify it again.", + ), + "DCI_FORBIDDEN_READER_CAPABILITY": ( + "runtime", + "reader", + False, + "Remove the forbidden capability; do not weaken the reader policy.", + ), + "DCI_RESOURCE_LIMIT": ( + "resource", + "reader", + False, + "Reduce the source or raise a reviewed explicit safety limit.", + ), + "DCI_TRUSTED_READER_FAILED": ( + "runtime", + "reader", + True, + "Retain the failure evidence and retry only in the exact approved runtime.", + ), + "DCI_WORKER_PROTOCOL_INVALID": ( + "protocol", + "protocol", + False, + "Reject the worker result and correct the versioned protocol implementation.", + ), + "DCI_VALIDATION_FAILED": ( + "validation", + "validation", + False, + "Correct the producer-neutral records and validate them again.", + ), + "DCI_ADMISSION_DENIED": ( + "resource", + "admission", + True, + "Select sufficient capacity or lower an explicit output bound.", + ), + "DCI_MATERIALIZE_FAILED": ( + "output", + "materialization", + True, + "Correct the output failure and retry from the unchanged source.", + ), + "DCI_PROMOTION_FAILED": ( + "output", + "promotion", + True, + "Correct the promotion failure and retry from the unchanged source.", + ), + "DCI_EVIDENCE_UNAVAILABLE": ( + "evidence", + "evidence", + False, + "Regenerate complete evidence without bypassing validation.", + ), + "DCI_DIAGNOSTIC_REDACTION_FAILED": ( + "evidence", + "evidence", + False, + "Suppress the diagnostic and repair redaction before disclosure.", + ), + "DCI_CLEANUP_INCOMPLETE": ( + "cleanup", + "cleanup", + True, + "Quarantine the owned scratch path and complete verified cleanup.", + ), + "DCI_CONCURRENT_OPERATION_CONFLICT": ( + "concurrency", + "admission", + True, + "Wait for the owning operation or select a distinct destination.", + ), + "DCI_HOST_TRUST_UNSATISFIED": ( + "trust", + "preflight", + False, + "Move the operation to an approved isolated host.", + ), + "DCI_SECURITY_BASELINE_REVOKED": ( + "runtime", + "preflight", + False, + "Stop and obtain a newly approved security baseline.", + ), +} + + +class CheckpointContractError(ValueError): + """A fail-closed checkpoint refusal with a stable public contract.""" + + def __init__( + self, + code: str, + *, + detail: str, + affected_refs: Iterable[str] = (), + ) -> None: + try: + category, phase, retryable, next_action = _ERROR_CONTRACTS[code] + except KeyError as error: # pragma: no cover - programmer error + raise ValueError(f"Unknown checkpoint error code: {code}") from error + references = tuple(sorted({str(reference) for reference in affected_refs})) + self.code = code + self.category = category + self.phase = phase + self.retryable = retryable + self.next_action = next_action + self.detail = detail + self.affected_refs = references + super().__init__(f"{code}: {detail}") + + def to_blocker(self) -> dict[str, Any]: + """Return the strict blocker shape accepted by descriptor contract v1.""" + return { + "code": self.code, + "category": self.category, + "phase": self.phase, + "affected_refs": list(self.affected_refs), + "retryable": self.retryable, + "next_action": self.next_action, + } + + def to_diagnostic(self) -> dict[str, Any]: + """Return a deterministic diagnostic without exception chains or local paths.""" + return {**self.to_blocker(), "detail": self.detail} diff --git a/obliteratus/checkpoint_fixtures.py b/obliteratus/checkpoint_fixtures.py new file mode 100644 index 0000000..e555956 --- /dev/null +++ b/obliteratus/checkpoint_fixtures.py @@ -0,0 +1,144 @@ +"""Strict loader for the tiny project-owned neutral checkpoint fixtures.""" + +from __future__ import annotations + +import json +import stat +from dataclasses import dataclass +from pathlib import Path + +import torch +from safetensors.torch import load_file + +from obliteratus.checkpoint_fragments import Padding, Replica, TensorFragment + + +_MAX_FIXTURE_JSON_BYTES = 256 * 1024 + + +@dataclass(frozen=True) +class TensorOracle: + """Independent expected logical value stored separately from fragments.""" + + shape: tuple[int, ...] + dtype: str + sha256: str + values: torch.Tensor + + +@dataclass(frozen=True) +class FixtureCase: + """One bounded fixture case ready for neutral validation.""" + + case_id: str + fragments: tuple[TensorFragment, ...] + tensor_oracles: dict[str, TensorOracle] + expected_manifest_digest: str + + +def _regular_file(path: Path) -> None: + mode = path.lstat().st_mode + if not stat.S_ISREG(mode): + raise ValueError(f"fixture artifact is not a regular file: {path.name}") + + +def _load_json(path: Path) -> dict: + _regular_file(path) + size = path.stat().st_size + if size > _MAX_FIXTURE_JSON_BYTES: + raise ValueError(f"fixture JSON exceeds {_MAX_FIXTURE_JSON_BYTES} bytes: {path.name}") + value = json.loads(path.read_bytes().decode("utf-8")) + if not isinstance(value, dict): + raise ValueError(f"fixture JSON is not an object: {path.name}") + return value + + +def _require_safe_name(value: object, field: str) -> str: + if not isinstance(value, str) or not value or Path(value).name != value: + raise ValueError(f"unsafe fixture {field}") + return value + + +def load_fixture_case(case_root: Path | str) -> FixtureCase: + """Load one generated JSON+safetensors fixture without pickle or network paths.""" + root = Path(case_root) + if root.is_symlink() or not root.is_dir(): + raise ValueError("fixture case root must be a non-symlink directory") + record = _load_json(root / "case.json") + if record.get("schema_id") != "obliteratus.checkpoint-fixture-case": + raise ValueError("unsupported fixture case schema") + tensor_files: dict[str, dict[str, torch.Tensor]] = {} + fragments: list[TensorFragment] = [] + fragment_records = record.get("fragments") + if not isinstance(fragment_records, list) or len(fragment_records) > 64: + raise ValueError("fixture fragment list is invalid or too large") + for item in fragment_records: + if not isinstance(item, dict): + raise ValueError("fixture fragment must be an object") + payload_file = _require_safe_name(item["payload_file"], "payload file") + payload_key = _require_safe_name(item["payload_key"], "payload key") + if payload_file not in tensor_files: + payload_path = root / payload_file + _regular_file(payload_path) + tensor_files[payload_file] = load_file(payload_path, device="cpu") + try: + payload = tensor_files[payload_file][payload_key] + except KeyError as error: + raise ValueError("fixture payload key is missing") from error + padding = item["padding"] + replica = item["replica"] + fragments.append( + TensorFragment( + fragment_id=item["fragment_id"], + component_id=item["component_id"], + fqn=item["fqn"], + role=item["role"], + dtype=item["dtype"], + global_shape=tuple(item["global_shape"]), + local_shape=tuple(item["local_shape"]), + element_offset=tuple(item["element_offset"]), + element_extent=tuple(item["element_extent"]), + padding=Padding( + before=tuple(padding["before"]), + after=tuple(padding["after"]), + semantic=padding["semantic"], + ), + shard_file_id=item["shard_file_id"], + shard_digest_ref=item["shard_digest_ref"], + fragment_digest=item["fragment_digest"], + replica=Replica( + group_id=replica["group_id"], + member_index=replica["member_index"], + member_count=replica["member_count"], + ), + partition_axes=tuple(item["partition_axes"]), + logical_tensor_id=item["logical_tensor_id"], + tie_group_id=item["tie_group_id"], + shared_storage_id=item["shared_storage_id"], + topology_coordinates=tuple( + (kind, coordinate) for kind, coordinate in item["topology_coordinates"] + ), + evidence_refs=tuple(item["evidence_refs"]), + payload=payload, + ) + ) + oracle_file = _require_safe_name(record["oracle_file"], "oracle file") + oracle_path = root / oracle_file + _regular_file(oracle_path) + oracle_values = load_file(oracle_path, device="cpu") + tensor_oracles: dict[str, TensorOracle] = {} + for item in record["oracles"]: + logical_tensor_id = item["logical_tensor_id"] + payload_key = _require_safe_name(item["payload_key"], "oracle payload key") + tensor_oracles[logical_tensor_id] = TensorOracle( + shape=tuple(item["shape"]), + dtype=item["dtype"], + sha256=item["sha256"], + values=oracle_values[payload_key], + ) + return FixtureCase( + case_id=record["case_id"], + fragments=tuple(fragments), + tensor_oracles=tensor_oracles, + expected_manifest_digest=record["expected_manifest_digest"], + ) diff --git a/obliteratus/checkpoint_fragments.py b/obliteratus/checkpoint_fragments.py new file mode 100644 index 0000000..ea7f0d4 --- /dev/null +++ b/obliteratus/checkpoint_fragments.py @@ -0,0 +1,622 @@ +"""Producer-neutral tensor fragment validation and bounded reconstruction.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from hashlib import sha256 +from typing import Literal + +import torch + +from obliteratus.checkpoint_errors import CheckpointContractError + + +_INT64_MAX = (1 << 63) - 1 +_MAX_IDENTIFIER_BYTES = 4096 +_ROLES = frozenset({"parameter", "persistent_buffer"}) +_PADDING_SEMANTICS = frozenset({"none", "producer_declared", "model_mapping_declared"}) +_SAFE_DTYPES = frozenset( + { + "bool", + "uint8", + "uint16", + "uint32", + "uint64", + "int8", + "int16", + "int32", + "int64", + "float16", + "bfloat16", + "float32", + "float64", + "float8_e4m3fn", + "float8_e5m2", + "complex64", + "complex128", + } +) + + +@dataclass(frozen=True) +class FragmentLimits: + """Explicit CPU-safe limits applied before combinatorial validation work.""" + + max_fragments: int = 4096 + max_dimensions: int = 32 + max_elements_per_tensor: int = _INT64_MAX + max_overlap_checks: int = 2_000_000 + + def __post_init__(self) -> None: + for name, value in vars(self).items(): + if type(value) is not int or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + + +@dataclass(frozen=True) +class Padding: + """Declared non-logical values surrounding a fragment payload.""" + + before: tuple[int, ...] + after: tuple[int, ...] + semantic: Literal["none", "producer_declared", "model_mapping_declared"] = "none" + + @classmethod + def zeros(cls, dimensions: int) -> Padding: + return cls(before=(0,) * dimensions, after=(0,) * dimensions) + + +@dataclass(frozen=True) +class Replica: + """An explicit replica declaration; absent grouping means unique content.""" + + group_id: str | None + member_index: int + member_count: int + + @classmethod + def unique(cls) -> Replica: + return cls(group_id=None, member_index=0, member_count=1) + + +@dataclass(frozen=True) +class TensorFragment: + """Typed logical geometry plus an optional already-normalized CPU payload.""" + + fragment_id: str + component_id: str + fqn: str + role: Literal["parameter", "persistent_buffer"] + dtype: str + global_shape: tuple[int, ...] + local_shape: tuple[int, ...] + element_offset: tuple[int, ...] + element_extent: tuple[int, ...] + padding: Padding + shard_file_id: str + shard_digest_ref: str + fragment_digest: str | None + replica: Replica + partition_axes: tuple[int, ...] + logical_tensor_id: str + tie_group_id: str | None + shared_storage_id: str | None + topology_coordinates: tuple[tuple[str, int], ...] + evidence_refs: tuple[str, ...] + payload: torch.Tensor | None = None + + def manifest_record(self) -> dict[str, object]: + """Return deterministic neutral metadata; tensor bytes are never embedded.""" + return { + "fragment_id": self.fragment_id, + "component_id": self.component_id, + "fqn": self.fqn, + "role": self.role, + "dtype": self.dtype, + "global_shape": list(self.global_shape), + "local_shape": list(self.local_shape), + "element_offset": list(self.element_offset), + "element_extent": list(self.element_extent), + "padding": { + "before": list(self.padding.before), + "after": list(self.padding.after), + "semantic": self.padding.semantic, + }, + "shard_file_id": self.shard_file_id, + "shard_digest_ref": self.shard_digest_ref, + "fragment_digest": self.fragment_digest, + "replica": { + "group_id": self.replica.group_id, + "member_index": self.replica.member_index, + "member_count": self.replica.member_count, + }, + "partition_axes": list(self.partition_axes), + "logical_tensor_id": self.logical_tensor_id, + "tie_group_id": self.tie_group_id, + "shared_storage_id": self.shared_storage_id, + "topology_coordinates": [list(item) for item in sorted(self.topology_coordinates)], + "evidence_refs": sorted(self.evidence_refs), + } + + +@dataclass(frozen=True) +class ValidatedLogicalTensor: + """One exactly covered logical tensor after explicit replica deduplication.""" + + logical_tensor_id: str + fqn: str + dtype: str + global_shape: tuple[int, ...] + fragments: tuple[TensorFragment, ...] + replica_members: tuple[tuple[str, ...], ...] + partition_axes: tuple[int, ...] + topology_coordinates: tuple[tuple[str, int], ...] + tie_group_id: str | None + + +@dataclass(frozen=True) +class ValidationResult: + """Deterministic result used by independent oracles and the canonical writer.""" + + logical_tensors: tuple[ValidatedLogicalTensor, ...] + manifest_digest: str + logical_elements: int + + def get(self, logical_tensor_id: str) -> ValidatedLogicalTensor: + for tensor in self.logical_tensors: + if tensor.logical_tensor_id == logical_tensor_id: + return tensor + raise KeyError(logical_tensor_id) + + +def _refuse(detail: str, *references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail=detail, + affected_refs=references, + ) + + +def _resource(detail: str, *references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_RESOURCE_LIMIT", + detail=detail, + affected_refs=references, + ) + + +def _checked_shape( + value: tuple[int, ...], + *, + name: str, + dimensions: int | None, + limits: FragmentLimits, + reference: str, +) -> tuple[int, ...]: + if not isinstance(value, tuple): + raise _refuse("shape_type_invalid", reference) + if len(value) > limits.max_dimensions: + raise _resource("max_dimensions", reference) + if dimensions is not None and len(value) != dimensions: + raise _refuse("dimension_mismatch", reference) + for item in value: + if type(item) is not int: + raise _refuse("integer_type_invalid", reference) + if item < 0: + raise _refuse("negative_integer", reference) + if item > _INT64_MAX: + raise _refuse("integer_overflow", reference) + return value + + +def _checked_add(left: int, right: int, reference: str) -> int: + if left > _INT64_MAX - right: + raise _refuse("integer_overflow", reference) + return left + right + + +def _checked_product(shape: tuple[int, ...], reference: str) -> int: + result = 1 + for item in shape: + if item and result > _INT64_MAX // item: + raise _refuse("integer_overflow", reference) + result *= item + return result + + +def _valid_text(value: object) -> bool: + if not isinstance(value, str) or not value: + return False + try: + return len(value.encode("utf-8")) <= _MAX_IDENTIFIER_BYTES + except UnicodeEncodeError: + return False + + +def _valid_optional_text(value: object) -> bool: + return value is None or _valid_text(value) + + +def _payload_view(fragment: TensorFragment) -> torch.Tensor | None: + payload = fragment.payload + if payload is None: + return None + if not isinstance(payload, torch.Tensor): + raise _refuse("payload_type_invalid", fragment.logical_tensor_id, fragment.fragment_id) + if payload.device.type != "cpu": + raise _refuse("payload_device_not_cpu", fragment.logical_tensor_id, fragment.fragment_id) + if payload.layout != torch.strided or payload.is_quantized: + raise _refuse("payload_layout_unsupported", fragment.logical_tensor_id, fragment.fragment_id) + if tuple(payload.shape) != fragment.local_shape: + raise _refuse("payload_shape_mismatch", fragment.logical_tensor_id, fragment.fragment_id) + if str(payload.dtype).removeprefix("torch.") != fragment.dtype: + raise _refuse("payload_dtype_mismatch", fragment.logical_tensor_id, fragment.fragment_id) + if not fragment.global_shape: + return payload + slices = tuple( + slice(before, before + extent) + for before, extent in zip( + fragment.padding.before, + fragment.element_extent, + strict=True, + ) + ) + return payload[slices] + + +def _tensor_digest(tensor: torch.Tensor) -> str: + raw = tensor.detach().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + return f"sha256:{sha256(raw).hexdigest()}" + + +def _observed_digest(fragment: TensorFragment) -> str | None: + payload = _payload_view(fragment) + if payload is None: + return fragment.fragment_digest + actual = _tensor_digest(payload) + if fragment.fragment_digest is not None and fragment.fragment_digest != actual: + raise _refuse( + "fragment_digest_mismatch", + fragment.logical_tensor_id, + fragment.fragment_id, + ) + return actual + + +def _validate_fragment(fragment: TensorFragment, limits: FragmentLimits) -> None: + reference = fragment.logical_tensor_id + identifiers = ( + fragment.fragment_id, + fragment.component_id, + fragment.fqn, + fragment.dtype, + fragment.shard_file_id, + fragment.shard_digest_ref, + fragment.logical_tensor_id, + ) + if any(not _valid_text(value) for value in identifiers): + raise _refuse("identifier_invalid", reference) + if fragment.dtype not in _SAFE_DTYPES: + raise _refuse("dtype_unsupported", reference, fragment.fragment_id) + if not _valid_optional_text(fragment.tie_group_id) or not _valid_optional_text( + fragment.shared_storage_id + ): + raise _refuse("identifier_invalid", reference, fragment.fragment_id) + if fragment.fragment_digest is not None and ( + not isinstance(fragment.fragment_digest, str) + or not fragment.fragment_digest.startswith("sha256:") + or len(fragment.fragment_digest) != 71 + or any( + character not in "0123456789abcdef" + for character in fragment.fragment_digest[7:] + ) + ): + raise _refuse("fragment_digest_invalid", reference, fragment.fragment_id) + if not isinstance(fragment.padding, Padding): + raise _refuse("padding_type_invalid", reference, fragment.fragment_id) + if not isinstance(fragment.replica, Replica): + raise _refuse("replica_type_invalid", reference, fragment.fragment_id) + if fragment.role not in _ROLES: + raise _refuse("role_invalid", reference, fragment.fragment_id) + dimensions = len( + _checked_shape( + fragment.global_shape, + name="global_shape", + dimensions=None, + limits=limits, + reference=reference, + ) + ) + local = _checked_shape( + fragment.local_shape, + name="local_shape", + dimensions=dimensions, + limits=limits, + reference=reference, + ) + offset = _checked_shape( + fragment.element_offset, + name="element_offset", + dimensions=dimensions, + limits=limits, + reference=reference, + ) + extent = _checked_shape( + fragment.element_extent, + name="element_extent", + dimensions=dimensions, + limits=limits, + reference=reference, + ) + before = _checked_shape( + fragment.padding.before, + name="padding.before", + dimensions=dimensions, + limits=limits, + reference=reference, + ) + after = _checked_shape( + fragment.padding.after, + name="padding.after", + dimensions=dimensions, + limits=limits, + reference=reference, + ) + if fragment.padding.semantic not in _PADDING_SEMANTICS: + raise _refuse("padding_semantic_invalid", reference, fragment.fragment_id) + if fragment.padding.semantic == "none" and any((*before, *after)): + raise _refuse("undeclared_padding", reference, fragment.fragment_id) + for axis in range(dimensions): + logical_end = _checked_add(offset[axis], extent[axis], reference) + if logical_end > fragment.global_shape[axis]: + raise _refuse("fragment_out_of_bounds", reference, fragment.fragment_id) + padded = _checked_add(_checked_add(before[axis], extent[axis], reference), after[axis], reference) + if padded != local[axis]: + raise _refuse("padding_shape_mismatch", reference, fragment.fragment_id) + global_elements = _checked_product(fragment.global_shape, reference) + if global_elements > limits.max_elements_per_tensor: + raise _resource("max_elements_per_tensor", reference) + if not isinstance(fragment.partition_axes, tuple): + raise _refuse("partition_axis_type_invalid", reference, fragment.fragment_id) + if len(set(fragment.partition_axes)) != len(fragment.partition_axes): + raise _refuse("partition_axis_duplicate", reference, fragment.fragment_id) + for axis in fragment.partition_axes: + if type(axis) is not int or axis < 0 or axis >= dimensions: + raise _refuse("partition_axis_out_of_bounds", reference, fragment.fragment_id) + if fragment.replica.group_id is None: + if fragment.replica != Replica.unique(): + raise _refuse("replica_declaration_invalid", reference, fragment.fragment_id) + else: + if ( + not _valid_text(fragment.replica.group_id) + or type(fragment.replica.member_count) is not int + or fragment.replica.member_count < 2 + or fragment.replica.member_count > _INT64_MAX + or type(fragment.replica.member_index) is not int + ): + raise _refuse("replica_declaration_invalid", reference, fragment.fragment_id) + if not 0 <= fragment.replica.member_index < fragment.replica.member_count: + raise _refuse("replica_member_out_of_bounds", reference, fragment.fragment_id) + if not isinstance(fragment.topology_coordinates, tuple) or any( + not isinstance(item, tuple) + or len(item) != 2 + or not _valid_text(item[0]) + or type(item[1]) is not int + or item[1] < 0 + or item[1] > _INT64_MAX + for item in fragment.topology_coordinates + ): + raise _refuse("topology_coordinate_invalid", reference, fragment.fragment_id) + if len({kind for kind, _ in fragment.topology_coordinates}) != len( + fragment.topology_coordinates + ): + raise _refuse("topology_coordinate_duplicate", reference, fragment.fragment_id) + if not isinstance(fragment.evidence_refs, tuple) or any( + not _valid_text(item) for item in fragment.evidence_refs + ): + raise _refuse("evidence_ref_invalid", reference, fragment.fragment_id) + _observed_digest(fragment) + + +def _replica_metadata(fragment: TensorFragment) -> tuple[object, ...]: + return ( + fragment.component_id, + fragment.fqn, + fragment.role, + fragment.dtype, + fragment.global_shape, + fragment.local_shape, + fragment.element_offset, + fragment.element_extent, + fragment.padding, + fragment.partition_axes, + fragment.logical_tensor_id, + fragment.tie_group_id, + fragment.shared_storage_id, + ) + + +def _deduplicate_replicas( + fragments: list[TensorFragment], +) -> tuple[list[TensorFragment], tuple[tuple[str, ...], ...]]: + representatives = [item for item in fragments if item.replica.group_id is None] + groups: dict[tuple[object, ...], list[TensorFragment]] = {} + for item in fragments: + if item.replica.group_id is None: + continue + key = ( + item.logical_tensor_id, + item.replica.group_id, + item.element_offset, + item.element_extent, + ) + groups.setdefault(key, []).append(item) + memberships: list[tuple[str, ...]] = [] + for group in groups.values(): + ordered = sorted(group, key=lambda item: (item.replica.member_index, item.fragment_id)) + expected_count = ordered[0].replica.member_count + indices = [item.replica.member_index for item in ordered] + if any(item.replica.member_count != expected_count for item in ordered): + raise _refuse("replica_count_mismatch", ordered[0].logical_tensor_id) + if indices != list(range(expected_count)): + raise _refuse("replica_members_missing", ordered[0].logical_tensor_id) + if len({_replica_metadata(item) for item in ordered}) != 1: + raise _refuse("replica_metadata_mismatch", ordered[0].logical_tensor_id) + digests = [_observed_digest(item) for item in ordered] + if any(digest is None for digest in digests): + raise _refuse("replica_digest_unavailable", ordered[0].logical_tensor_id) + if len(set(digests)) != 1: + raise _refuse("replica_digest_mismatch", ordered[0].logical_tensor_id) + representative = min(ordered, key=lambda item: item.fragment_id) + representatives.append(representative) + memberships.append(tuple(sorted(item.fragment_id for item in ordered))) + return representatives, tuple(sorted(memberships)) + + +def _fragments_overlap(left: TensorFragment, right: TensorFragment) -> bool: + if not left.global_shape: + return True + return all( + left_start < right_start + right_size and right_start < left_start + left_size + for left_start, left_size, right_start, right_size in zip( + left.element_offset, + left.element_extent, + right.element_offset, + right.element_extent, + strict=True, + ) + ) + + +def _logical_tensor( + logical_tensor_id: str, + fragments: list[TensorFragment], + limits: FragmentLimits, +) -> ValidatedLogicalTensor: + first = fragments[0] + identity = {(item.fqn, item.dtype, item.global_shape, item.tie_group_id) for item in fragments} + if len(identity) != 1: + raise _refuse("logical_tensor_metadata_mismatch", logical_tensor_id) + representatives, replica_members = _deduplicate_replicas(fragments) + representatives.sort( + key=lambda item: (item.element_offset, item.element_extent, item.fragment_id) + ) + checks = len(representatives) * (len(representatives) - 1) // 2 + if checks > limits.max_overlap_checks: + raise _resource("max_overlap_checks", logical_tensor_id) + for index, left in enumerate(representatives): + for right in representatives[index + 1 :]: + if _fragments_overlap(left, right): + raise _refuse("coverage_overlap", logical_tensor_id) + global_elements = _checked_product(first.global_shape, logical_tensor_id) + if global_elements > limits.max_elements_per_tensor: + raise _resource("max_elements_per_tensor", logical_tensor_id) + covered = sum(_checked_product(item.element_extent, logical_tensor_id) for item in representatives) + if covered != global_elements: + raise _refuse("coverage_gap" if covered < global_elements else "coverage_mismatch", logical_tensor_id) + if global_elements == 0 and len(representatives) != 1: + raise _refuse("zero_tensor_representation_ambiguous", logical_tensor_id) + return ValidatedLogicalTensor( + logical_tensor_id=logical_tensor_id, + fqn=first.fqn, + dtype=first.dtype, + global_shape=first.global_shape, + fragments=tuple(representatives), + replica_members=replica_members, + partition_axes=tuple(sorted({axis for item in fragments for axis in item.partition_axes})), + topology_coordinates=tuple( + sorted({coordinate for item in fragments for coordinate in item.topology_coordinates}) + ), + tie_group_id=first.tie_group_id, + ) + + +def reconstruct_logical_tensor( + result: ValidationResult, + logical_tensor_id: str, +) -> torch.Tensor: + """Materialize one validated tensor from already-normalized CPU payloads.""" + logical = result.get(logical_tensor_id) + first_payload = _payload_view(logical.fragments[0]) + if first_payload is None: + raise _refuse("payload_unavailable", logical_tensor_id) + output = torch.empty(logical.global_shape, dtype=first_payload.dtype, device="cpu") + for fragment in logical.fragments: + payload = _payload_view(fragment) + if payload is None: + raise _refuse("payload_unavailable", logical_tensor_id, fragment.fragment_id) + if not logical.global_shape: + output.copy_(payload) + continue + destination = tuple( + slice(offset, offset + extent) + for offset, extent in zip( + fragment.element_offset, + fragment.element_extent, + strict=True, + ) + ) + output[destination].copy_(payload) + return output + + +def _validate_ties(result: ValidationResult) -> None: + groups: dict[str, list[ValidatedLogicalTensor]] = {} + for logical in result.logical_tensors: + if logical.tie_group_id is not None: + groups.setdefault(logical.tie_group_id, []).append(logical) + for tensors in groups.values(): + if len(tensors) < 2: + raise _refuse("tie_group_member_missing", tensors[0].logical_tensor_id) + references = tuple(sorted(item.logical_tensor_id for item in tensors)) + if len({(item.dtype, item.global_shape) for item in tensors}) != 1: + raise _refuse("tie_group_metadata_mismatch", *references) + if all(all(fragment.payload is not None for fragment in item.fragments) for item in tensors): + digests = { + _tensor_digest(reconstruct_logical_tensor(result, item.logical_tensor_id)) + for item in tensors + } + if len(digests) != 1: + raise _refuse("tie_group_content_mismatch", *references) + + +def validate_fragments( + fragments: list[TensorFragment] | tuple[TensorFragment, ...], + *, + limits: FragmentLimits | None = None, +) -> ValidationResult: + """Validate exact logical coverage with explicit, agreement-checked replicas.""" + active_limits = limits or FragmentLimits() + if not isinstance(fragments, (list, tuple)) or not fragments: + raise _refuse("fragment_set_empty") + if len(fragments) > active_limits.max_fragments: + raise _resource("max_fragments") + if any(not isinstance(item, TensorFragment) for item in fragments): + raise _refuse("fragment_type_invalid") + fragment_ids = [item.fragment_id for item in fragments] + if len(set(fragment_ids)) != len(fragment_ids): + raise _refuse("fragment_id_duplicate") + grouped: dict[str, list[TensorFragment]] = {} + for fragment in fragments: + _validate_fragment(fragment, active_limits) + grouped.setdefault(fragment.logical_tensor_id, []).append(fragment) + logical_tensors = tuple( + _logical_tensor(logical_id, grouped[logical_id], active_limits) + for logical_id in sorted(grouped) + ) + records = [item.manifest_record() for item in sorted(fragments, key=lambda item: item.fragment_id)] + manifest_payload = json.dumps( + records, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + result = ValidationResult( + logical_tensors=logical_tensors, + manifest_digest=f"sha256:{sha256(manifest_payload).hexdigest()}", + logical_elements=sum(_checked_product(item.global_shape, item.logical_tensor_id) for item in logical_tensors), + ) + _validate_ties(result) + return result diff --git a/obliteratus/checkpoint_inspection.py b/obliteratus/checkpoint_inspection.py new file mode 100644 index 0000000..925839b --- /dev/null +++ b/obliteratus/checkpoint_inspection.py @@ -0,0 +1,1127 @@ +"""Bounded, offline, structure-only checkpoint inventory and classification.""" + +from __future__ import annotations + +import json +import os +import stat +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path, PurePosixPath +from typing import Any + +from obliteratus.checkpoint_capabilities import AdapterRegistry +from obliteratus.checkpoint_errors import CheckpointContractError + + +_INT64_MAX = (1 << 63) - 1 +_VERIFIER = "obliteratus.safe-structure-v1" +_DTYPE_BYTES = { + "BOOL": 1, + "U8": 1, + "I8": 1, + "F8_E4M3": 1, + "F8_E5M2": 1, + "I16": 2, + "U16": 2, + "F16": 2, + "BF16": 2, + "I32": 4, + "U32": 4, + "F32": 4, + "C64": 8, + "I64": 8, + "U64": 8, + "F64": 8, + "C128": 16, +} + + +@dataclass(frozen=True) +class InspectionLimits: + """Explicit bounds for untrusted local structural inspection.""" + + max_files: int = 100_000 + max_directories: int = 10_000 + max_total_bytes: int = 1 << 40 + max_json_bytes: int = 8 << 20 + max_safetensors_header_bytes: int = 64 << 20 + max_tensors: int = 2_000_000 + hash_chunk_bytes: int = 1 << 20 + + def __post_init__(self) -> None: + for name, value in vars(self).items(): + if type(value) is not int or value <= 0: + raise ValueError(f"{name} must be a positive integer") + + +@dataclass(frozen=True) +class _Fingerprint: + device: int + inode: int + mode: int + size: int + modified_ns: int + + @classmethod + def from_stat(cls, value: os.stat_result) -> _Fingerprint: + return cls( + device=value.st_dev, + inode=value.st_ino, + mode=value.st_mode, + size=value.st_size, + modified_ns=value.st_mtime_ns, + ) + + +@dataclass(frozen=True) +class _ObservedFile: + file_id: str + relative_path: str + path: Path + fingerprint: _Fingerprint + sha256: str + role: str + + +@dataclass(frozen=True) +class _ObservedDirectory: + relative_path: str + path: Path + fingerprint: _Fingerprint + + +@dataclass(frozen=True) +class _HeaderSummary: + tensor_names: tuple[str, ...] + tensor_count: int + logical_bytes: int + + +class _DuplicateJsonKey(ValueError): + pass + + +@dataclass(frozen=True) +class CheckpointInspection: + """A strict descriptor produced without invoking any checkpoint reader.""" + + descriptor_id: str + primary_format: str + support_decision: str + _descriptor: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Return a detached JSON-compatible descriptor.""" + return json.loads(json.dumps(self._descriptor, sort_keys=True, allow_nan=False)) + + def to_json(self) -> str: + """Return deterministic JSON suitable for CLI and retained evidence.""" + return json.dumps( + self._descriptor, + indent=2, + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + ) + "\n" + + +def _boundary(detail: str, *references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail=detail, + affected_refs=references, + ) + + +def _changed(*references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_SOURCE_CHANGED", + detail="source_changed", + affected_refs=references, + ) + + +def _resource(detail: str, *references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_RESOURCE_LIMIT", + detail=detail, + affected_refs=references, + ) + + +def _validation(detail: str, *references: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail=detail, + affected_refs=references, + ) + + +def _safe_relative(path: Path, root: Path, *, root_is_file: bool) -> str: + relative = path.name if root_is_file else path.relative_to(root).as_posix() + pure = PurePosixPath(relative) + if ( + not relative + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + or len(relative) > 4096 + ): + raise _boundary("path_traversal") + return relative + + +def _role(relative_path: str) -> str: + name = PurePosixPath(relative_path).name + if name.endswith(".safetensors"): + return "tensor_payload" + if name.endswith(".index.json"): + return "weight_index" + if name in {"config.json", "adapter_config.json", "fsdp_metadata.json"}: + return "configuration" + if name in {"metadata.json", ".metadata", "universal_checkpoint_info.json"}: + return "producer_metadata" + if name.endswith((".bin", ".pt", ".pth", ".distcp")): + return "unsafe_serialization_or_shard" + return "other" + + +def _open_flags() -> int: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + return flags + + +def _matches(descriptor: int, expected: _Fingerprint) -> bool: + return _Fingerprint.from_stat(os.fstat(descriptor)) == expected + + +def _open_observed(path: Path, reference: str) -> int: + try: + return os.open(path, _open_flags()) + except OSError as error: + raise _changed(reference) from error + + +def _hash_regular_file( + path: Path, + expected: _Fingerprint, + limits: InspectionLimits, +) -> str: + """Hash one already-bounded regular file while detecting replacement or mutation.""" + descriptor = _open_observed(path, path.name) + digest = sha256() + try: + if not _matches(descriptor, expected): + raise _changed(path.name) + while True: + chunk = os.read(descriptor, limits.hash_chunk_bytes) + if not chunk: + break + digest.update(chunk) + if not _matches(descriptor, expected): + raise _changed(path.name) + finally: + os.close(descriptor) + try: + if _Fingerprint.from_stat(path.lstat()) != expected: + raise _changed(path.name) + except FileNotFoundError as error: + raise _changed(path.name) from error + return f"sha256:{digest.hexdigest()}" + + +def _collect_paths( + source: Path, + limits: InspectionLimits, +) -> tuple[ + list[tuple[Path, str, _Fingerprint]], + tuple[_ObservedDirectory, ...], + _Fingerprint, +]: + try: + root_stat = source.lstat() + except FileNotFoundError as error: + raise _boundary("source_missing") from error + if stat.S_ISLNK(root_stat.st_mode): + raise _boundary("source_symlink") + try: + if source.absolute() != source.resolve(strict=True): + raise _boundary("source_symlink") + except FileNotFoundError as error: + raise _changed("source") from error + root_fingerprint = _Fingerprint.from_stat(root_stat) + root_is_file = stat.S_ISREG(root_stat.st_mode) + if root_is_file: + if root_stat.st_size < 0 or root_stat.st_size > _INT64_MAX: + raise _resource("file_size") + if root_stat.st_size > limits.max_total_bytes: + raise _resource("max_total_bytes") + return [(source, source.name, root_fingerprint)], (), root_fingerprint + if not stat.S_ISDIR(root_stat.st_mode): + raise _boundary("source_special_file") + pending = [(source, ".", root_fingerprint)] + directory_count = 0 + directories: list[_ObservedDirectory] = [] + result: list[tuple[Path, str, _Fingerprint]] = [] + total_bytes = 0 + while pending: + directory, relative_directory, expected_directory = pending.pop() + directory_count += 1 + if directory_count > limits.max_directories: + raise _resource("max_directories") + try: + current_directory = _Fingerprint.from_stat(directory.lstat()) + if current_directory != expected_directory or not stat.S_ISDIR( + current_directory.mode + ): + raise _changed(relative_directory) + with os.scandir(directory) as iterator: + entries = sorted(iterator, key=lambda entry: entry.name) + except OSError as error: + raise _boundary("source_unreadable") from error + directories.append( + _ObservedDirectory(relative_directory, directory, expected_directory) + ) + for entry in entries: + path = Path(entry.path) + try: + observed = entry.stat(follow_symlinks=False) + except FileNotFoundError as error: + raise _changed(entry.name) from error + mode = observed.st_mode + relative = _safe_relative(path, source, root_is_file=False) + if stat.S_ISLNK(mode): + raise _boundary("source_symlink", relative) + if stat.S_ISDIR(mode): + pending.append((path, relative, _Fingerprint.from_stat(observed))) + continue + if not stat.S_ISREG(mode): + raise _boundary("source_special_file", relative) + if len(result) >= limits.max_files: + raise _resource("max_files") + if observed.st_size < 0 or observed.st_size > _INT64_MAX: + raise _resource("file_size") + if total_bytes > limits.max_total_bytes - observed.st_size: + raise _resource("max_total_bytes") + total_bytes += observed.st_size + result.append((path, relative, _Fingerprint.from_stat(observed))) + try: + if _Fingerprint.from_stat(directory.lstat()) != expected_directory: + raise _changed(relative_directory) + except FileNotFoundError as error: + raise _changed(relative_directory) from error + return ( + sorted(result, key=lambda item: item[1]), + tuple(sorted(directories, key=lambda item: item.relative_path)), + root_fingerprint, + ) + + +def _inventory( + source: Path, + limits: InspectionLimits, +) -> tuple[tuple[_ObservedFile, ...], tuple[_ObservedDirectory, ...], _Fingerprint]: + paths, directories, root_fingerprint = _collect_paths(source, limits) + observed_files = [] + for index, (path, relative, fingerprint) in enumerate(paths, start=1): + digest = _hash_regular_file(path, fingerprint, limits) + observed_files.append( + _ObservedFile( + file_id=f"file-{index:06d}", + relative_path=relative, + path=path, + fingerprint=fingerprint, + sha256=digest, + role=_role(relative), + ) + ) + return tuple(observed_files), directories, root_fingerprint + + +def _read_bounded(file: _ObservedFile, maximum: int, limit_name: str) -> bytes: + if file.fingerprint.size > maximum: + raise _resource(limit_name, file.file_id) + descriptor = _open_observed(file.path, file.file_id) + try: + if not _matches(descriptor, file.fingerprint): + raise _changed(file.file_id) + chunks: list[bytes] = [] + remaining = file.fingerprint.size + while remaining: + chunk = os.read(descriptor, min(remaining, 1 << 20)) + if not chunk: + raise _changed(file.file_id) + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1) or not _matches(descriptor, file.fingerprint): + raise _changed(file.file_id) + return b"".join(chunks) + finally: + os.close(descriptor) + + +def _read_json(file: _ObservedFile, limits: InspectionLimits) -> dict[str, Any]: + payload = _read_bounded(file, limits.max_json_bytes, "max_json_bytes") + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_reject_duplicate_json_keys, + ) + except (UnicodeError, json.JSONDecodeError, _DuplicateJsonKey) as error: + raise _validation("json_invalid", file.file_id) from error + if not isinstance(value, dict): + raise _validation("json_object_required", file.file_id) + return value + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _DuplicateJsonKey(key) + result[key] = value + return result + + +def _checked_tensor_bytes(dtype: str, shape: object, reference: str) -> int: + if dtype not in _DTYPE_BYTES or not isinstance(shape, list) or len(shape) > 32: + raise _validation("safetensors_header_invalid", reference) + elements = 1 + for dimension in shape: + if type(dimension) is not int or dimension < 0 or dimension > _INT64_MAX: + raise _validation("safetensors_header_invalid", reference) + if dimension and elements > _INT64_MAX // dimension: + raise _validation("integer_overflow", reference) + elements *= dimension + width = _DTYPE_BYTES[dtype] + if elements and elements > _INT64_MAX // width: + raise _validation("integer_overflow", reference) + return elements * width + + +def _safetensors_header(file: _ObservedFile, limits: InspectionLimits) -> _HeaderSummary: + if file.fingerprint.size < 8: + raise _validation("safetensors_truncated", file.file_id) + descriptor = _open_observed(file.path, file.file_id) + try: + if not _matches(descriptor, file.fingerprint): + raise _changed(file.file_id) + length_bytes = os.read(descriptor, 8) + if len(length_bytes) != 8: + raise _validation("safetensors_truncated", file.file_id) + header_length = int.from_bytes(length_bytes, "little") + if header_length > limits.max_safetensors_header_bytes: + raise _resource("max_safetensors_header_bytes", file.file_id) + if header_length > file.fingerprint.size - 8: + raise _validation("safetensors_truncated", file.file_id) + header_bytes = b"" + while len(header_bytes) < header_length: + chunk = os.read(descriptor, header_length - len(header_bytes)) + if not chunk: + raise _validation("safetensors_truncated", file.file_id) + header_bytes += chunk + if not _matches(descriptor, file.fingerprint): + raise _changed(file.file_id) + finally: + os.close(descriptor) + try: + header = json.loads( + header_bytes.decode("utf-8"), + object_pairs_hook=_reject_duplicate_json_keys, + ) + except (UnicodeError, json.JSONDecodeError, _DuplicateJsonKey) as error: + raise _validation("safetensors_header_invalid", file.file_id) from error + if not isinstance(header, dict): + raise _validation("safetensors_header_invalid", file.file_id) + entries = [(name, value) for name, value in header.items() if name != "__metadata__"] + if len(entries) > limits.max_tensors: + raise _resource("max_tensors", file.file_id) + names: list[str] = [] + ranges: list[tuple[int, int]] = [] + logical_bytes = 0 + payload_bytes = file.fingerprint.size - 8 - header_length + for name, value in entries: + if not isinstance(name, str) or not name or not isinstance(value, dict): + raise _validation("safetensors_header_invalid", file.file_id) + offsets = value.get("data_offsets") + if ( + not isinstance(offsets, list) + or len(offsets) != 2 + or any(type(item) is not int or item < 0 for item in offsets) + ): + raise _validation("safetensors_header_invalid", file.file_id) + begin, end = offsets + dtype = value.get("dtype") + if not isinstance(dtype, str): + raise _validation("safetensors_header_invalid", file.file_id) + expected_bytes = _checked_tensor_bytes(dtype, value.get("shape"), file.file_id) + if begin > end or end > payload_bytes or end - begin != expected_bytes: + raise _validation("safetensors_range_invalid", file.file_id) + names.append(name) + ranges.append((begin, end)) + logical_bytes += expected_bytes + if logical_bytes > _INT64_MAX: + raise _validation("integer_overflow", file.file_id) + ranges.sort() + if any(left_end > right_start for (_, left_end), (right_start, _) in zip(ranges, ranges[1:])): + raise _validation("safetensors_range_overlap", file.file_id) + cursor = 0 + for begin, end in ranges: + if begin != cursor: + raise _validation("safetensors_range_gap", file.file_id) + cursor = end + if cursor != payload_bytes: + raise _validation("safetensors_range_gap", file.file_id) + return _HeaderSummary(tuple(sorted(names)), len(names), logical_bytes) + + +def _revalidate( + source: Path, + root_fingerprint: _Fingerprint, + files: tuple[_ObservedFile, ...], + directories: tuple[_ObservedDirectory, ...], + limits: InspectionLimits, +) -> None: + try: + if _Fingerprint.from_stat(source.lstat()) != root_fingerprint: + raise _changed("source") + for file in files: + if _Fingerprint.from_stat(file.path.lstat()) != file.fingerprint: + raise _changed(file.file_id) + if _hash_regular_file(file.path, file.fingerprint, limits) != file.sha256: + raise _changed(file.file_id) + for directory in directories: + if _Fingerprint.from_stat(directory.path.lstat()) != directory.fingerprint: + raise _changed(directory.relative_path) + except FileNotFoundError as error: + raise _changed("source") from error + paths, observed_directories, observed_root = _collect_paths(source, limits) + expected_paths = tuple( + (file.relative_path, file.fingerprint) for file in files + ) + actual_paths = tuple((relative, fingerprint) for _, relative, fingerprint in paths) + if observed_root != root_fingerprint or actual_paths != expected_paths: + raise _changed("source") + if tuple( + (directory.relative_path, directory.fingerprint) + for directory in observed_directories + ) != tuple( + (directory.relative_path, directory.fingerprint) for directory in directories + ): + raise _changed("source") + + +def _evidence( + evidence_id: str, + *, + subject: str, + kind: str, + file: _ObservedFile | None, + confidence: str, + location: str, +) -> dict[str, Any]: + return { + "evidence_id": evidence_id, + "subject": subject, + "kind": kind, + "file_ref": file.file_id if file is not None else None, + "location": location, + "confidence": confidence, + "verifier": _VERIFIER, + } + + +def _producer(name: str | None, evidence_refs: list[str]) -> dict[str, Any]: + return { + "name": name, + "version": None, + "format_version": None, + "evidence_refs": evidence_refs, + } + + +def _component( + component_id: str, + kind: str, + checkpoint_format: str, + producer_name: str | None, + scopes: list[str], + files: list[_ObservedFile], + evidence_refs: list[str], +) -> dict[str, Any]: + return { + "component_id": component_id, + "kind": kind, + "format": checkpoint_format, + "producer": _producer(producer_name, evidence_refs), + "state_scopes": scopes, + "topology_ref": None, + "inventory_refs": sorted(file.file_id for file in files), + "tensor_fragment_refs": [], + } + + +def _file_name_map(files: tuple[_ObservedFile, ...]) -> dict[str, _ObservedFile]: + result: dict[str, _ObservedFile] = {} + for item in files: + name = PurePosixPath(item.relative_path).name + if name in result: + raise _validation("duplicate_basename", result[name].file_id, item.file_id) + result[name] = item + return result + + +def _format_components( + files: tuple[_ObservedFile, ...], + limits: InspectionLimits, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[CheckpointContractError], int, int]: + by_name = _file_name_map(files) + names = set(by_name) + evidence: list[dict[str, Any]] = [] + errors: list[CheckpointContractError] = [] + components: list[dict[str, Any]] = [] + logical_bytes = 0 + tensor_count = 0 + + hf_files: list[_ObservedFile] = [] + hf_index = by_name.get("model.safetensors.index.json") + if hf_index is not None: + hf_files.append(hf_index) + if "model.safetensors" in names: + errors.append( + _validation( + "hf_layout_collision", + hf_index.file_id, + by_name["model.safetensors"].file_id, + ) + ) + try: + index = _read_json(hf_index, limits) + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict) or not weight_map or len(weight_map) > limits.max_tensors: + raise _validation("hf_weight_map_invalid", hf_index.file_id) + referenced_names: set[str] = set() + for tensor_name, shard_name in weight_map.items(): + if ( + not isinstance(tensor_name, str) + or not tensor_name + or not isinstance(shard_name, str) + or PurePosixPath(shard_name).name != shard_name + ): + raise _validation("hf_weight_map_invalid", hf_index.file_id) + referenced_names.add(shard_name) + shard_candidates = { + name for name in names if name.startswith("model-") and name.endswith(".safetensors") + } + if referenced_names != shard_candidates or not referenced_names <= names: + raise _validation("hf_weight_map_shard_mismatch", hf_index.file_id) + header_names: dict[str, set[str]] = {} + for shard_name in sorted(referenced_names): + shard = by_name[shard_name] + hf_files.append(shard) + summary = _safetensors_header(shard, limits) + header_names[shard_name] = set(summary.tensor_names) + tensor_count += summary.tensor_count + logical_bytes += summary.logical_bytes + if tensor_count > limits.max_tensors: + raise _resource("max_tensors", hf_index.file_id) + mapped_names = { + shard_name: { + tensor_name + for tensor_name, mapped_shard in weight_map.items() + if mapped_shard == shard_name + } + for shard_name in referenced_names + } + if any( + header_names[shard_name] != mapped_names[shard_name] + for shard_name in referenced_names + ): + raise _validation("hf_weight_map_tensor_mismatch", hf_index.file_id) + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + elif "model.safetensors" in names: + model_file = by_name["model.safetensors"] + hf_files.append(model_file) + try: + summary = _safetensors_header(model_file, limits) + tensor_count += summary.tensor_count + logical_bytes += summary.logical_bytes + if tensor_count > limits.max_tensors: + raise _resource("max_tensors", model_file.file_id) + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + if hf_files: + evidence_id = "evidence-hf-safetensors" + evidence.append( + _evidence( + evidence_id, + subject="/components/hf_safetensors", + kind="header", + file=hf_files[0], + confidence="verified" if not errors else "inferred", + location=hf_files[0].relative_path, + ) + ) + components.append( + _component( + "component-model", + "model", + "hf_safetensors", + "Hugging Face", + ["model_weights"], + hf_files, + [evidence_id], + ) + ) + + peft_files: list[_ObservedFile] = [] + if "adapter_model.safetensors" in names and "adapter_config.json" in names: + peft_files = [by_name["adapter_model.safetensors"], by_name["adapter_config.json"]] + try: + summary = _safetensors_header(peft_files[0], limits) + _read_json(peft_files[1], limits) + tensor_count += summary.tensor_count + logical_bytes += summary.logical_bytes + if tensor_count > limits.max_tensors: + raise _resource("max_tensors", peft_files[0].file_id) + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + evidence_id = "evidence-peft-safetensors" + evidence.append( + _evidence( + evidence_id, + subject="/components/peft_adapter", + kind="header", + file=peft_files[0], + confidence="verified" if not errors else "inferred", + location=peft_files[0].relative_path, + ) + ) + components.append( + _component( + "component-peft-adapter", + "peft_adapter", + "peft_safetensors", + "PEFT", + ["adapter_weights"], + peft_files, + [evidence_id], + ) + ) + + legacy_names = sorted( + name for name in names if name == "pytorch_model.bin" or name == "pytorch_model.bin.index.json" + ) + if legacy_names: + selected = [by_name[name] for name in legacy_names] + evidence_id = "evidence-hf-pickle-filename" + evidence.append( + _evidence( + evidence_id, + subject="/components/hf_pytorch_pickle", + kind="filename", + file=selected[0], + confidence="inferred", + location=selected[0].relative_path, + ) + ) + components.append( + _component( + "component-model-pickle", + "model", + "hf_pytorch_pickle", + "Hugging Face", + ["model_weights"], + selected, + [evidence_id], + ) + ) + + fsdp_file = by_name.get("fsdp_metadata.json") + fsdp = False + if fsdp_file is not None: + try: + fsdp_record = _read_json(fsdp_file, limits) + fsdp = fsdp_record.get("state_dict_type") in { + "SHARDED_STATE_DICT", + "LOCAL_STATE_DICT", + "FULL_STATE_DICT", + } + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + dcp_files = [item for item in files if PurePosixPath(item.relative_path).name == ".metadata" or item.relative_path.endswith(".distcp")] + if fsdp: + assert fsdp_file is not None + selected = sorted({*dcp_files, fsdp_file}, key=lambda item: item.relative_path) + format_name, component_id = "fsdp_state_dict", "component-fsdp" + producer_name = "PyTorch FSDP" + elif dcp_files: + selected = dcp_files + format_name, component_id = "pytorch_dcp", "component-pytorch-dcp" + producer_name = "PyTorch" + else: + selected = [] + format_name = component_id = producer_name = "" + if selected: + evidence_id = f"evidence-{format_name}" + evidence.append( + _evidence( + evidence_id, + subject=f"/components/{format_name}", + kind="filename" if not fsdp else "json", + file=selected[0], + confidence="declared" if fsdp else "inferred", + location=selected[0].relative_path, + ) + ) + components.append( + _component( + component_id, + "unknown", + format_name, + producer_name, + ["unknown_state"], + selected, + [evidence_id], + ) + ) + + metadata_file = by_name.get("metadata.json") + megatron = False + if metadata_file is not None: + try: + metadata = _read_json(metadata_file, limits) + megatron = metadata.get("sharded_backend") == "torch_dist" + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + if megatron: + assert metadata_file is not None + selected = [metadata_file] + [by_name[name] for name in sorted(names) if name == "common.pt"] + evidence_id = "evidence-megatron-torch-dist" + evidence.append( + _evidence( + evidence_id, + subject="/components/megatron_torch_dist", + kind="json", + file=metadata_file, + confidence="declared", + location="metadata.json#/sharded_backend", + ) + ) + components.append( + _component( + "component-megatron", + "unknown", + "megatron_torch_dist", + "Megatron-Core", + ["unknown_state"], + selected, + [evidence_id], + ) + ) + + universal = by_name.get("universal_checkpoint_info.json") + zero_files = [item for item in files if "zero_pp_rank_" in PurePosixPath(item.relative_path).name] + if universal is not None: + try: + _read_json(universal, limits) + except CheckpointContractError as error: + if error.code == "DCI_RESOURCE_LIMIT": + raise + errors.append(error) + selected = [universal, *zero_files] + ds_format = "deepspeed_universal" + elif zero_files: + selected = zero_files + ds_format = "deepspeed_zero" + else: + selected = [] + ds_format = "" + if selected: + evidence_id = f"evidence-{ds_format}" + evidence.append( + _evidence( + evidence_id, + subject=f"/components/{ds_format}", + kind="json" if universal is not None else "filename", + file=selected[0], + confidence="declared" if universal is not None else "inferred", + location=selected[0].relative_path, + ) + ) + components.append( + _component( + "component-deepspeed", + "unknown", + ds_format, + "DeepSpeed", + ["unknown_state"], + selected, + [evidence_id], + ) + ) + return components, evidence, errors, tensor_count, logical_bytes + + +def _primary(components: list[dict[str, Any]], errors: list[CheckpointContractError]) -> tuple[str, str]: + formats = [component["format"] for component in components] + if not formats: + return "unknown", "unknown" + if any(error.detail.endswith("layout_collision") for error in errors): + return "ambiguous", "inferred" + if len(formats) > 1: + return "ambiguous", "inferred" + if errors: + return formats[0], "inferred" + if formats[0] in {"hf_safetensors", "peft_safetensors"}: + return formats[0], "verified" + if formats[0] in {"fsdp_state_dict", "megatron_torch_dist", "deepspeed_universal"}: + return formats[0], "declared" + return formats[0], "inferred" + + +def _classification_blocker(primary_format: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_UNSUPPORTED_FORMAT_OR_VERSION", + detail="ambiguous_layout" if primary_format == "ambiguous" else "unknown_layout", + affected_refs=("source",), + ) + + +def _trust_blocker(components: list[dict[str, Any]]) -> CheckpointContractError: + return CheckpointContractError( + "DCI_TRUST_POLICY_REQUIRED", + detail="payload_or_vendor_metadata_required", + affected_refs=(component["component_id"] for component in components), + ) + + +def _descriptor_digest(value: object) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + return sha256(payload).hexdigest() + + +def _inspect_checkpoint( + source: Path | str, + *, + limits: InspectionLimits | None = None, + adapter_registry: AdapterRegistry | None = None, +) -> CheckpointInspection: + """Inspect one local source using only inert filenames, JSON, and safetensors headers.""" + active_limits = limits or InspectionLimits() + active_registry = adapter_registry or AdapterRegistry() + source_path = Path(source) + files, directories, root_fingerprint = _inventory(source_path, active_limits) + components, evidence, probe_errors, tensor_count, logical_bytes = _format_components( + files, + active_limits, + ) + primary_format, confidence = _primary(components, probe_errors) + if not components: + evidence.append( + _evidence( + "evidence-unknown-layout", + subject="/primary_format", + kind="derived", + file=None, + confidence="unknown", + location="bounded inventory contains no recognized structural signature", + ) + ) + components = [ + _component( + "component-unknown", + "unknown", + "unknown", + None, + ["unknown_state"], + list(files), + ["evidence-unknown-layout"], + ) + ] + formats = {component["format"] for component in components} + trusted_formats = { + "hf_pytorch_pickle", + "pytorch_dcp", + "fsdp_state_dict", + "megatron_torch_dist", + "megatron_torch_dcp", + "megatron_fsdp_dtensor", + "deepspeed_zero", + "deepspeed_universal", + } + trust_required = bool(formats & trusted_formats) + blockers = list(probe_errors) + if primary_format in {"unknown", "ambiguous"}: + blockers.append(_classification_blocker(primary_format)) + elif trust_required: + blockers.append(_trust_blocker(components)) + if primary_format == "ambiguous": + adapter_resolution = { + "status": "ambiguous", + "adapter_id": None, + "adapter_version": None, + "capability_digest": None, + "reason": "Ambiguous structure cannot select a capability.", + } + elif trust_required: + resolved_adapter = active_registry.resolve(primary_format) + adapter_resolution = resolved_adapter.to_dict() + if adapter_resolution["status"] == "ambiguous": + blockers.append(_classification_blocker("ambiguous")) + elif resolved_adapter.dependency_unavailable: + blockers.append( + CheckpointContractError( + "DCI_TRUST_RUNTIME_UNAVAILABLE", + detail="optional_dependency_missing_or_incompatible", + affected_refs=(str(adapter_resolution["adapter_id"]),), + ) + ) + else: + adapter_resolution = { + "status": "not_required" if len(formats) == 1 else "unsupported", + "adapter_id": None, + "adapter_version": None, + "capability_digest": None, + "reason": ( + "Canonical safetensors needs no producer adapter." + if len(formats) == 1 + else "No producer capability applies to this structure." + ), + } + if blockers: + support_decision = ( + "trusted_inspection_required" + if ( + trust_required + and len(formats) == 1 + and adapter_resolution["status"] != "ambiguous" + ) + else "blocked" + ) + else: + support_decision = "canonical_hf_ready" + scopes = sorted({scope for component in components for scope in component["state_scopes"]}) + state_classification = ( + "weights_only" + if set(scopes) <= {"model_weights", "adapter_weights"} and "unknown_state" not in scopes + else "unknown" + ) + unsafe_findings = [] + if formats & trusted_formats: + unsafe_findings.append("unsafe or opaque serialization requires a separately approved reader") + inventory_records = [ + { + "file_id": file.file_id, + "relative_path": file.relative_path, + "role": file.role, + "size_bytes": file.fingerprint.size, + "sha256": file.sha256, + "regular_file": True, + "observation_id": f"observation-{_descriptor_digest([file.relative_path, file.fingerprint.size, file.sha256])[:24]}", + } + for file in files + ] + total_source_bytes = sum(file.fingerprint.size for file in files) + inventory_core = { + "files": inventory_records, + "total_bytes": total_source_bytes, + "observation_complete": True, + } + inventory_id = f"inventory-{_descriptor_digest(inventory_core)[:32]}" + _revalidate(source_path, root_fingerprint, files, directories, active_limits) + unique_evidence = {item["evidence_id"]: item for item in evidence} + producer_names = {component["producer"]["name"] for component in components} + producer_name = next(iter(producer_names)) if len(producer_names) == 1 else None + producer_evidence = sorted(unique_evidence) + descriptor: dict[str, Any] = { + "schema_id": "obliteratus.checkpoint-descriptor", + "schema_version": "1.0.0", + "descriptor_id": "pending", + "primary_format": primary_format, + "classification_confidence": confidence, + "components": sorted(components, key=lambda item: item["component_id"]), + "producer": _producer(producer_name, producer_evidence), + "evidence": [unique_evidence[key] for key in sorted(unique_evidence)], + "source_inventory": {"inventory_id": inventory_id, **inventory_core}, + "safety": { + "inspection_level": "safe_structure", + "trust_required": trust_required, + "inventory_revalidated": True, + "unsafe_serialization_findings": unsafe_findings, + "violations": [error.detail for error in probe_errors], + }, + "state": { + "observed_scopes": scopes, + "classification": state_classification, + }, + "topologies": [], + "tensor_fragments": [], + "adapter_resolution": adapter_resolution, + "conversion_plan": { + "eligible": False, + "target_format": "hf_safetensors", + "state_scope": "weights_only", + "dropped_scopes": sorted(scope for scope in scopes if scope not in {"model_weights", "adapter_weights"}), + }, + "resource_estimate": { + "source_bytes": total_source_bytes, + "logical_bytes": logical_bytes, + "output_bytes": logical_bytes, + "temporary_bytes": 0, + "peak_ram_bytes": min( + total_source_bytes, + max(active_limits.max_json_bytes, active_limits.max_safetensors_header_bytes), + ), + "peak_vram_bytes": 0, + "file_count": len(files), + "tensor_count": tensor_count, + "shard_count": sum(1 for file in files if file.relative_path.endswith((".safetensors", ".distcp", ".bin", ".pt"))), + "assumptions": [ + "Structure-only estimates do not authorize payload loading or conversion." + ], + "confidence": "verified" if tensor_count and not probe_errors else "unknown", + "admission": "unknown", + }, + "support_decision": support_decision, + "blockers": [error.to_blocker() for error in blockers], + } + descriptor_id = f"descriptor-{_descriptor_digest({key: value for key, value in descriptor.items() if key != 'descriptor_id'})[:32]}" + descriptor["descriptor_id"] = descriptor_id + return CheckpointInspection( + descriptor_id=descriptor_id, + primary_format=primary_format, + support_decision=support_decision, + _descriptor=descriptor, + ) + + +def inspect_checkpoint( + source: Path | str, + *, + limits: InspectionLimits | None = None, + adapter_registry: AdapterRegistry | None = None, +) -> CheckpointInspection: + """Inspect one local source and expose only stable fail-closed errors.""" + try: + return _inspect_checkpoint( + source, + limits=limits, + adapter_registry=adapter_registry, + ) + except CheckpointContractError: + raise + except OSError as error: + raise _changed("source") from error diff --git a/obliteratus/checkpoint_provenance.py b/obliteratus/checkpoint_provenance.py new file mode 100644 index 0000000..bcce291 --- /dev/null +++ b/obliteratus/checkpoint_provenance.py @@ -0,0 +1,879 @@ +"""Canonical, content-addressed provenance for checkpoint-derived artifacts.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Any, Mapping, Sequence + + +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_COMMIT = re.compile(r"^[0-9a-f]{40}$") +_ARTIFACT_ID = re.compile(r"^artifact-sha256:[0-9a-f]{64}$") +_SECRET_KEY = re.compile( + r"(?:^|[_-])(?:token|secret|password|credential|api[-_]?key)(?:$|[_-])", + re.I, +) +_PROMPT_KEY = re.compile(r"(?:prompt|harmful|harmless|instruction|conversation)", re.I) +_SECRET_VALUE = re.compile(r"(?i)(?:hf_[a-z0-9]{12,}|bearer\s+[a-z0-9._~+/-]{12,})") +_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]") +_MAX_COLLECTION_ITEMS = 4096 +_MAX_COMMAND_ARGUMENTS = 256 +_MAX_NESTING_DEPTH = 16 +_LINEAGE_TYPES = frozenset( + { + "consolidation", + "reshard", + "pretrain", + "full_finetune", + "adapter_train", + "adapter_merge", + "quantization", + "dequantization", + "surgery", + } +) +_EXACT_RESUME_SCOPES = frozenset( + { + "model_weights", + "optimizer_state", + "scheduler_state", + "rng_state", + "dataloader_state", + "framework_state", + } +) + + +def _require_digest(value: str, field: str) -> None: + if not isinstance(value, str) or not _DIGEST.fullmatch(value): + raise ValueError(f"{field} must be a sha256 digest") + + +def _require_commit(value: str, field: str) -> None: + if not isinstance(value, str) or not _COMMIT.fullmatch(value): + raise ValueError(f"{field} must be a 40-character lowercase commit") + + +def _is_absolute(value: str) -> bool: + return Path(value).is_absolute() or bool(_WINDOWS_ABSOLUTE.match(value)) + + +def _require_public_text(value: str, field: str) -> None: + if not isinstance(value, str) or not value or len(value) > 512: + raise ValueError(f"{field} must be non-empty bounded text") + if _SECRET_VALUE.search(value): + raise ValueError(f"{field} contains a secret") + if _is_absolute(value): + raise ValueError(f"{field} contains a private local path") + + +@dataclass(frozen=True) +class ArtifactIdentity: + kind: str + identity: str + revision: str | None + digest: str + + def __post_init__(self) -> None: + if self.kind not in {"local", "hub", "generated"}: + raise ValueError("artifact identity kind is invalid") + _require_public_text(self.identity, "artifact identity") + if self.revision is not None: + _require_public_text(self.revision, "artifact revision") + _require_digest(self.digest, "artifact digest") + + def to_dict(self) -> dict[str, Any]: + return { + "kind": self.kind, + "identity": self.identity, + "revision": self.revision, + "digest": self.digest, + } + + +@dataclass(frozen=True) +class ToolIdentity: + name: str + version: str + commit: str + + def __post_init__(self) -> None: + _require_public_text(self.name, "tool name") + _require_public_text(self.version, "tool version") + _require_commit(self.commit, "tool commit") + + def to_dict(self) -> dict[str, str]: + return {"name": self.name, "version": self.version, "commit": self.commit} + + +@dataclass(frozen=True) +class LineageEvent: + event_id: str + event_type: str + parent_artifact_ids: tuple[str, ...] + tool: str + transformations: tuple[str, ...] + + def __post_init__(self) -> None: + _require_public_text(self.event_id, "lineage event ID") + if self.event_type not in _LINEAGE_TYPES: + raise ValueError("lineage event type is invalid") + _require_public_text(self.tool, "lineage tool") + for parent in self.parent_artifact_ids: + if not _ARTIFACT_ID.fullmatch(parent): + raise ValueError("lineage parent artifact ID is invalid") + for transformation in self.transformations: + _require_public_text(transformation, "lineage transformation") + + def to_dict(self) -> dict[str, Any]: + return { + "event_id": self.event_id, + "event_type": self.event_type, + "parent_artifact_ids": sorted(set(self.parent_artifact_ids)), + "tool": self.tool, + "transformations": sorted(set(self.transformations)), + } + + +@dataclass(frozen=True) +class AdapterIdentity: + adapter_type: str + base_model: ArtifactIdentity + config_digest: str + key_map_digest: str + + def __post_init__(self) -> None: + _require_public_text(self.adapter_type, "adapter type") + _require_digest(self.config_digest, "adapter config digest") + _require_digest(self.key_map_digest, "adapter key-map digest") + + def to_dict(self) -> dict[str, Any]: + return { + "adapter_type": self.adapter_type, + "base_model": self.base_model.to_dict(), + "config_digest": self.config_digest, + "key_map_digest": self.key_map_digest, + } + + +@dataclass(frozen=True) +class DatasetIdentity: + identifier: str + revision: str | None + digest: str + split: str | None + subset: str | None + record_count: int + + def __post_init__(self) -> None: + _require_public_text(self.identifier, "dataset identifier") + for field, value in ( + ("dataset revision", self.revision), + ("dataset split", self.split), + ("dataset subset", self.subset), + ): + if value is not None: + _require_public_text(value, field) + _require_digest(self.digest, "dataset digest") + if type(self.record_count) is not int or not 0 <= self.record_count <= (1 << 63) - 1: + raise ValueError("dataset record count is invalid") + + def to_dict(self) -> dict[str, Any]: + return { + "identifier": self.identifier, + "revision": self.revision, + "digest": self.digest, + "split": self.split, + "subset": self.subset, + "record_count": self.record_count, + } + + +@dataclass(frozen=True) +class TrainingIdentity: + method: str + framework: str | None + framework_version: str | None + hyperparameters_digest: str | None + + def __post_init__(self) -> None: + if self.method not in {"pretrain", "full_finetune", "adapter_train", "unknown"}: + raise ValueError("training method is invalid") + for field, value in ( + ("training framework", self.framework), + ("training framework version", self.framework_version), + ): + if value is not None: + _require_public_text(value, field) + if self.hyperparameters_digest is not None: + _require_digest(self.hyperparameters_digest, "training hyperparameters digest") + + def to_dict(self) -> dict[str, Any]: + return { + "method": self.method, + "framework": self.framework, + "framework_version": self.framework_version, + "hyperparameters_digest": self.hyperparameters_digest, + } + + +@dataclass(frozen=True) +class ProvenanceRecord: + artifact_id: str + record_digest: str + _json: str + + def __post_init__(self) -> None: + try: + value = json.loads(self._json, object_pairs_hook=_reject_duplicate_pairs) + except (TypeError, json.JSONDecodeError, ValueError) as error: + raise ValueError("provenance JSON is invalid") from error + canonical = json.dumps( + value, + indent=2, + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + ) + "\n" + if self._json != canonical: + raise ValueError("provenance JSON is not canonical") + verified = verify_provenance_record(value) + if ( + verified["artifact_id"] != self.artifact_id + or verified["record_digest"] != self.record_digest + ): + raise ValueError("provenance record identity fields disagree") + + def to_dict(self) -> dict[str, Any]: + return json.loads(self._json) + + def to_json(self) -> str: + return self._json + + +@dataclass(frozen=True) +class LegacyProvenanceFacts: + _json: str + + def to_dict(self) -> dict[str, Any]: + return json.loads(self._json) + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _digest(value: object) -> str: + return f"sha256:{sha256(_canonical_bytes(value)).hexdigest()}" + + +def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _exact_mapping(value: Any, fields: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != fields: + raise ValueError(f"{label} fields are invalid") + return dict(value) + + +def _artifact_identity_from_record(value: Any, label: str) -> ArtifactIdentity: + record = _exact_mapping(value, {"kind", "identity", "revision", "digest"}, label) + try: + identity = ArtifactIdentity( + record["kind"], + record["identity"], + record["revision"], + record["digest"], + ) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} is invalid") from error + if identity.revision is not None and len(identity.revision) > 256: + raise ValueError(f"{label} revision is too large") + return identity + + +def _tool_identity_from_record(value: Any) -> ToolIdentity: + record = _exact_mapping(value, {"name", "version", "commit"}, "converter") + try: + return ToolIdentity(record["name"], record["version"], record["commit"]) + except (TypeError, ValueError) as error: + raise ValueError("converter is invalid") from error + + +def _lineage_from_record(value: Any) -> LineageEvent: + record = _exact_mapping( + value, + {"event_id", "event_type", "parent_artifact_ids", "tool", "transformations"}, + "lineage event", + ) + parents = record["parent_artifact_ids"] + transformations = record["transformations"] + if not isinstance(parents, list) or not isinstance(transformations, list): + raise ValueError("lineage event collections are invalid") + try: + event = LineageEvent( + record["event_id"], + record["event_type"], + tuple(parents), + record["tool"], + tuple(transformations), + ) + except (TypeError, ValueError) as error: + raise ValueError("lineage event is invalid") from error + if event.to_dict() != record: + raise ValueError("lineage event is not canonical") + return event + + +def _adapter_from_record(value: Any) -> AdapterIdentity: + record = _exact_mapping( + value, + {"adapter_type", "base_model", "config_digest", "key_map_digest"}, + "adapter identity", + ) + base_model = _artifact_identity_from_record(record["base_model"], "adapter base model") + try: + return AdapterIdentity( + record["adapter_type"], + base_model, + record["config_digest"], + record["key_map_digest"], + ) + except (TypeError, ValueError) as error: + raise ValueError("adapter identity is invalid") from error + + +def _dataset_from_record(value: Any) -> DatasetIdentity: + record = _exact_mapping( + value, + {"identifier", "revision", "digest", "split", "subset", "record_count"}, + "dataset identity", + ) + for field in ("revision", "split", "subset"): + item = record[field] + if item is not None and (not isinstance(item, str) or len(item) > 256): + raise ValueError(f"dataset {field} is invalid") + try: + return DatasetIdentity( + record["identifier"], + record["revision"], + record["digest"], + record["split"], + record["subset"], + record["record_count"], + ) + except (TypeError, ValueError) as error: + raise ValueError("dataset identity is invalid") from error + + +def _training_from_record(value: Any) -> TrainingIdentity: + record = _exact_mapping( + value, + {"method", "framework", "framework_version", "hyperparameters_digest"}, + "training identity", + ) + for field, maximum in (("framework", 256), ("framework_version", 128)): + item = record[field] + if item is not None and (not isinstance(item, str) or len(item) > maximum): + raise ValueError(f"training {field} is invalid") + try: + return TrainingIdentity( + record["method"], + record["framework"], + record["framework_version"], + record["hyperparameters_digest"], + ) + except (TypeError, ValueError) as error: + raise ValueError("training identity is invalid") from error + + +def _verify_canonical_string_set( + value: Any, + field: str, + *, + nonempty: bool = False, + digests: bool = False, +) -> list[str]: + if not isinstance(value, list) or (nonempty and not value): + raise ValueError(f"{field} must be a canonical list") + canonical = _sorted_unique(value, field, digests=digests) + if value != canonical: + raise ValueError(f"{field} must be sorted and unique") + return canonical + + +def _normalize_public(value: Any, field: str, *, depth: int = 0) -> Any: + if depth > _MAX_NESTING_DEPTH: + raise ValueError(f"{field} nesting is too deep") + if value is None or isinstance(value, bool): + return value + if type(value) is int: + if not -(1 << 63) <= value <= (1 << 63) - 1: + raise ValueError(f"{field} integer is outside int64") + return value + if isinstance(value, str): + if _SECRET_VALUE.search(value): + raise ValueError(f"{field} contains a secret") + if _is_absolute(value): + raise ValueError(f"{field} contains a private local path") + if len(value) > 1024: + raise ValueError(f"{field} text is too large") + return value + if isinstance(value, Mapping): + if len(value) > _MAX_COLLECTION_ITEMS: + raise ValueError(f"{field} has too many fields") + if any(not isinstance(key, str) for key in value): + raise ValueError(f"{field} has a non-string key") + if any(not key or len(key) > 512 for key in value): + raise ValueError(f"{field} has an invalid key") + result = {} + for key in sorted(value): + if _SECRET_KEY.search(key) or _PROMPT_KEY.search(key): + raise ValueError(f"{field} contains a sensitive key") + result[key] = _normalize_public( + value[key], + f"{field}.{key}", + depth=depth + 1, + ) + return result + if isinstance(value, (list, tuple)): + if len(value) > _MAX_COLLECTION_ITEMS: + raise ValueError(f"{field} has too many items") + return [ + _normalize_public(item, field, depth=depth + 1) for item in value + ] + raise ValueError(f"{field} contains a non-JSON value") + + +def verify_provenance_record(value: Mapping[str, Any]) -> dict[str, Any]: + """Verify canonical identity, public-data hygiene, and state truth.""" + if not isinstance(value, Mapping): + raise ValueError("provenance record must be an object") + record = _normalize_public(dict(value), "provenance") + required = { + "schema_id", + "schema_version", + "artifact_id", + "record_digest", + "sources", + "converter", + "obliteratus_commit", + "configuration_digest", + "tokenizer", + "base_model", + "command", + "environment", + "source_topology", + "lineage", + "input_digests", + "output_digests", + "transformations", + "state", + "adapter", + "dataset", + "training", + "unknowns", + } + if set(record) != required: + raise ValueError("provenance record fields are invalid") + if ( + record["schema_id"] != "obliteratus.artifact-provenance" + or record["schema_version"] != "1.0.0" + or not isinstance(record["sources"], list) + or not record["sources"] + or not isinstance(record["input_digests"], list) + or not isinstance(record["output_digests"], list) + or not isinstance(record["command"], list) + or not isinstance(record["state"], dict) + ): + raise ValueError("provenance record structure is invalid") + if not isinstance(record["artifact_id"], str) or not _ARTIFACT_ID.fullmatch( + record["artifact_id"] + ): + raise ValueError("provenance artifact ID is invalid") + _require_digest(record["record_digest"], "provenance record digest") + sources = [ + _artifact_identity_from_record(item, "source identity") + for item in record["sources"] + ] + canonical_sources = sorted( + { + json.dumps(item.to_dict(), sort_keys=True): item.to_dict() + for item in sources + }.values(), + key=_canonical_bytes, + ) + if record["sources"] != canonical_sources or len(sources) > _MAX_COLLECTION_ITEMS: + raise ValueError("provenance sources are not canonical") + _tool_identity_from_record(record["converter"]) + _require_commit(record["obliteratus_commit"], "OBLITERATUS commit") + if record["configuration_digest"] is not None: + _require_digest(record["configuration_digest"], "configuration digest") + for field in ("tokenizer", "base_model"): + if record[field] is not None: + _artifact_identity_from_record(record[field], field.replace("_", " ")) + command = record["command"] + if ( + len(command) > _MAX_COMMAND_ARGUMENTS + or any(not isinstance(item, str) or len(item) > 4096 for item in command) + or list(sanitize_command(command)) != command + ): + raise ValueError("provenance command is invalid or not sanitized") + environment = _exact_mapping( + record["environment"], + {"python", "platform", "packages"}, + "environment", + ) + for field, maximum in (("python", 128), ("platform", 256)): + item = environment[field] + if item is not None and (not isinstance(item, str) or len(item) > maximum): + raise ValueError(f"environment {field} is invalid") + packages = environment["packages"] + if ( + not isinstance(packages, Mapping) + or len(packages) > _MAX_COLLECTION_ITEMS + or any( + not isinstance(name, str) + or not isinstance(version, str) + or len(version) > 128 + for name, version in packages.items() + ) + ): + raise ValueError("environment packages are invalid") + if not isinstance(record["source_topology"], dict): + raise ValueError("source topology must be an object") + lineage = record["lineage"] + if not isinstance(lineage, list) or len(lineage) > _MAX_COLLECTION_ITEMS: + raise ValueError("lineage must be a bounded list") + lineage_records = [_lineage_from_record(item).to_dict() for item in lineage] + if lineage_records != sorted( + lineage_records, + key=lambda item: (item["event_id"], item["event_type"]), + ) or len({_canonical_bytes(item) for item in lineage_records}) != len(lineage_records): + raise ValueError("lineage must be sorted and unique") + _verify_canonical_string_set( + record["input_digests"], + "input digests", + nonempty=True, + digests=True, + ) + _verify_canonical_string_set( + record["output_digests"], + "output digests", + nonempty=True, + digests=True, + ) + _verify_canonical_string_set(record["transformations"], "transformations") + state = record["state"] + if ( + set(state) != {"classification", "observed_scopes", "lost_state"} + or not isinstance(state["observed_scopes"], list) + or not isinstance(state["lost_state"], list) + or not isinstance(state["classification"], str) + ): + raise ValueError("provenance state is invalid") + _verify_canonical_string_set(state["observed_scopes"], "observed scopes") + _verify_canonical_string_set(state["lost_state"], "lost state") + if state["classification"] != classify_resume_state(state["observed_scopes"]): + raise ValueError("provenance state classification is not evidence-derived") + if record["adapter"] is not None: + _adapter_from_record(record["adapter"]) + if record["dataset"] is not None: + _dataset_from_record(record["dataset"]) + if record["training"] is not None: + _training_from_record(record["training"]) + _verify_canonical_string_set(record["unknowns"], "unknowns") + without_record_digest = { + key: item for key, item in record.items() if key != "record_digest" + } + if record["record_digest"] != _digest(without_record_digest): + raise ValueError("provenance record digest mismatch") + identity_core = { + key: item for key, item in without_record_digest.items() if key != "artifact_id" + } + expected_artifact_id = _digest(identity_core).replace( + "sha256:", + "artifact-sha256:", + 1, + ) + if record["artifact_id"] != expected_artifact_id: + raise ValueError("provenance artifact ID mismatch") + return record + + +def _local_path_token(value: str) -> str: + return f"[LOCAL_PATH:sha256:{sha256(value.encode('utf-8')).hexdigest()}]" + + +def sanitize_command(arguments: Sequence[str]) -> tuple[str, ...]: + """Remove secret values, raw prompt text, and private local paths from a command.""" + if isinstance(arguments, (str, bytes)) or len(arguments) > _MAX_COMMAND_ARGUMENTS: + raise ValueError("command argument list is invalid or too large") + result: list[str] = [] + redact_next = False + for raw in arguments: + if not isinstance(raw, str) or len(raw) > 4096: + raise ValueError("command argument must be bounded text") + argument = raw + if redact_next: + result.append("[REDACTED]") + redact_next = False + continue + if argument.startswith("--") and "=" in argument: + option, value = argument.split("=", 1) + if _SECRET_KEY.search(option) or _PROMPT_KEY.search(option): + result.append(f"{option}=[REDACTED]") + elif _is_absolute(value): + result.append(f"{option}={_local_path_token(value)}") + elif _SECRET_VALUE.search(value): + result.append(f"{option}=[REDACTED]") + else: + result.append(argument) + continue + if argument.startswith("--") and ( + _SECRET_KEY.search(argument) or _PROMPT_KEY.search(argument) + ): + result.append(argument) + redact_next = True + elif _is_absolute(argument): + result.append(_local_path_token(argument)) + elif _SECRET_VALUE.search(argument): + result.append("[REDACTED]") + else: + result.append(argument) + return tuple(result) + + +def classify_resume_state(observed_scopes: Sequence[str]) -> str: + """Derive the strongest truthful state class; caller claims are never accepted.""" + if ( + isinstance(observed_scopes, (str, bytes)) + or len(observed_scopes) > _MAX_COLLECTION_ITEMS + or any(not isinstance(scope, str) for scope in observed_scopes) + ): + raise ValueError("observed scopes must be a bounded string collection") + scopes = frozenset(observed_scopes) + if _EXACT_RESUME_SCOPES <= scopes: + return "exact_resume" + if {"model_weights", "optimizer_state"} <= scopes: + return "model_and_optimizer" + if "model_weights" in scopes or "adapter_weights" in scopes: + return "weights_only" + return "unknown" + + +def _sorted_unique(values: Sequence[str], field: str, *, digests: bool = False) -> list[str]: + if isinstance(values, (str, bytes)) or len(values) > _MAX_COLLECTION_ITEMS: + raise ValueError(f"{field} collection is invalid or too large") + if any(not isinstance(value, str) for value in values): + raise ValueError(f"{field} must contain strings") + result = sorted(set(values)) + for value in result: + if digests: + _require_digest(value, field) + else: + _require_public_text(value, field) + return result + + +def build_provenance( + *, + sources: Sequence[ArtifactIdentity], + converter: ToolIdentity, + obliteratus_commit: str, + configuration_digest: str | None, + tokenizer: ArtifactIdentity | None, + base_model: ArtifactIdentity | None, + command: Sequence[str], + environment: Mapping[str, Any], + source_topology: Mapping[str, Any], + lineage: Sequence[LineageEvent], + input_digests: Sequence[str], + output_digests: Sequence[str], + transformations: Sequence[str], + observed_scopes: Sequence[str], + lost_state: Sequence[str], + adapter: AdapterIdentity | None = None, + dataset: DatasetIdentity | None = None, + training: TrainingIdentity | None = None, + unknowns: Sequence[str] = (), +) -> ProvenanceRecord: + """Build a strict content-addressed record from explicit evidence only.""" + if not sources: + raise ValueError("at least one source identity is required") + for field, values in (("sources", sources), ("lineage", lineage)): + if isinstance(values, (str, bytes)) or len(values) > _MAX_COLLECTION_ITEMS: + raise ValueError(f"{field} collection is invalid or too large") + if any(not isinstance(source, ArtifactIdentity) for source in sources): + raise ValueError("sources must contain artifact identities") + if any(not isinstance(event, LineageEvent) for event in lineage): + raise ValueError("lineage must contain lineage events") + _require_commit(obliteratus_commit, "OBLITERATUS commit") + if configuration_digest is not None: + _require_digest(configuration_digest, "configuration digest") + environment_record = { + "python": environment.get("python"), + "platform": environment.get("platform"), + "packages": environment.get("packages", {}), + } + extra_environment = set(environment) - set(environment_record) + for key in environment: + if _SECRET_KEY.search(str(key)) or _PROMPT_KEY.search(str(key)): + raise ValueError("environment contains a sensitive key") + if extra_environment: + raise ValueError("environment contains unsupported fields") + environment_record = _normalize_public(environment_record, "environment") + topology_record = _normalize_public(source_topology, "source_topology") + source_records = sorted( + {json.dumps(source.to_dict(), sort_keys=True): source.to_dict() for source in sources}.values(), + key=lambda item: _canonical_bytes(item), + ) + lineage_records = sorted( + (event.to_dict() for event in lineage), + key=lambda item: (item["event_id"], item["event_type"]), + ) + scopes = _sorted_unique(observed_scopes, "observed scope") + core: dict[str, Any] = { + "schema_id": "obliteratus.artifact-provenance", + "schema_version": "1.0.0", + "sources": source_records, + "converter": converter.to_dict(), + "obliteratus_commit": obliteratus_commit, + "configuration_digest": configuration_digest, + "tokenizer": tokenizer.to_dict() if tokenizer is not None else None, + "base_model": base_model.to_dict() if base_model is not None else None, + "command": list(sanitize_command(command)), + "environment": environment_record, + "source_topology": topology_record, + "lineage": lineage_records, + "input_digests": _sorted_unique(input_digests, "input digest", digests=True), + "output_digests": _sorted_unique(output_digests, "output digest", digests=True), + "transformations": _sorted_unique(transformations, "transformation"), + "state": { + "classification": classify_resume_state(scopes), + "observed_scopes": scopes, + "lost_state": _sorted_unique(lost_state, "lost state"), + }, + "adapter": adapter.to_dict() if adapter is not None else None, + "dataset": dataset.to_dict() if dataset is not None else None, + "training": training.to_dict() if training is not None else None, + "unknowns": _sorted_unique(unknowns, "unknown"), + } + artifact_id = f"artifact-sha256:{sha256(_canonical_bytes(core)).hexdigest()}" + with_identity = {**core, "artifact_id": artifact_id} + record_digest = _digest(with_identity) + record = {**with_identity, "record_digest": record_digest} + canonical = json.dumps( + record, + indent=2, + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + ) + "\n" + return ProvenanceRecord(artifact_id, record_digest, canonical) + + +def migrate_legacy_metadata(metadata: Mapping[str, Any]) -> LegacyProvenanceFacts: + """Extract only explicit safe legacy facts and mark absent evidence as unknown.""" + if not isinstance(metadata, Mapping) or len(metadata) > _MAX_COLLECTION_ITEMS: + raise ValueError("legacy metadata must be a bounded mapping") + model = metadata.get("model") + model_identity = ( + model + if isinstance(model, str) + and model + and len(model) <= 512 + and not _is_absolute(model) + and not _SECRET_VALUE.search(model) + else None + ) + revision = metadata.get("model_revision") + if ( + not isinstance(revision, str) + or not revision + or len(revision) > 512 + or _is_absolute(revision) + or _SECRET_VALUE.search(revision) + ): + revision = None + tokenizer_revision = metadata.get("tokenizer_revision") + if ( + not isinstance(tokenizer_revision, str) + or not tokenizer_revision + or len(tokenizer_revision) > 512 + or _is_absolute(tokenizer_revision) + or _SECRET_VALUE.search(tokenizer_revision) + ): + tokenizer_revision = None + datasets = [] + dataset_inputs = metadata.get("dataset_inputs", []) + if not isinstance(dataset_inputs, (list, tuple)): + dataset_inputs = [] + for item in dataset_inputs[:_MAX_COLLECTION_ITEMS]: + if not isinstance(item, Mapping): + continue + identifier = item.get("identifier") + digest = item.get("sha256") + if ( + isinstance(identifier, str) + and identifier + and len(identifier) <= 512 + and not _is_absolute(identifier) + and not _SECRET_VALUE.search(identifier) + and isinstance(digest, str) + and re.fullmatch(r"[0-9a-f]{64}", digest) + ): + datasets.append({"identifier": identifier, "digest": f"sha256:{digest}"}) + unknowns = ["base_model_digest", "tokenizer_digest"] + known = {"model", "model_revision", "tokenizer_revision", "seed", "dataset_inputs"} + if set(metadata) - known: + unknowns.append("unmapped_fields_omitted") + if model_identity is None: + unknowns.append("base_model_identity") + seed = metadata.get("seed") + if type(seed) is int and -(1 << 63) <= seed <= (1 << 63) - 1: + seed_value = str(seed) + elif ( + isinstance(seed, str) + and seed + and len(seed) <= 128 + and not _is_absolute(seed) + and not _SECRET_VALUE.search(seed) + ): + seed_value = seed + else: + seed_value = None + if seed is not None: + unknowns.append("seed") + record = { + "schema_id": "obliteratus.legacy-provenance-facts", + "schema_version": "1.0.0", + "base_model": { + "identity": model_identity, + "revision": revision, + "digest": None, + }, + "tokenizer": {"revision": tokenizer_revision, "digest": None}, + "seed": seed_value, + "datasets": sorted(datasets, key=lambda item: (item["identifier"], item["digest"])), + "unknowns": sorted(unknowns), + } + return LegacyProvenanceFacts( + json.dumps(record, indent=2, sort_keys=True, ensure_ascii=True, allow_nan=False) + "\n" + ) diff --git a/obliteratus/checkpoint_service.py b/obliteratus/checkpoint_service.py new file mode 100644 index 0000000..34f23f4 --- /dev/null +++ b/obliteratus/checkpoint_service.py @@ -0,0 +1,29 @@ +"""Public Wave 2 service for offline checkpoint structural inspection.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from obliteratus.checkpoint_capabilities import AdapterRegistry +from obliteratus.checkpoint_inspection import ( + CheckpointInspection, + InspectionLimits, + inspect_checkpoint, +) + + +@dataclass(frozen=True) +class CheckpointService: + """Expose the common safe plane without trusted readers or producer adapters.""" + + inspection_limits: InspectionLimits = field(default_factory=InspectionLimits) + adapter_registry: AdapterRegistry = field(default_factory=AdapterRegistry) + + def inspect(self, source: Path | str) -> CheckpointInspection: + """Return a bounded structure-only descriptor for one local source.""" + return inspect_checkpoint( + source, + limits=self.inspection_limits, + adapter_registry=self.adapter_registry, + ) diff --git a/obliteratus/checkpoint_writer.py b/obliteratus/checkpoint_writer.py new file mode 100644 index 0000000..1bdfa5c --- /dev/null +++ b/obliteratus/checkpoint_writer.py @@ -0,0 +1,1027 @@ +"""Transactional canonical safetensors writer for validated neutral fragments.""" + +from __future__ import annotations + +import json +import os +import shutil +import stat +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Mapping, Sequence + +import torch +from safetensors.torch import load_file, save_file + +from obliteratus.checkpoint_errors import CheckpointContractError +from obliteratus.checkpoint_fragments import ( + TensorFragment, + ValidationResult, + reconstruct_logical_tensor, + validate_fragments, +) +from obliteratus.checkpoint_provenance import ( + ProvenanceRecord, + verify_provenance_record, +) +from obliteratus.persistence_contracts import atomic_checkpoint_directory + + +_DIGEST_PREFIX = "sha256:" +_INT64_MAX = (1 << 63) - 1 +_WRITER_BUFFER_BYTES = 1 << 20 +_COPY_KINDS = frozenset({"configuration", "tokenizer"}) +_CANONICAL_DTYPES = frozenset( + { + "bool", + "uint8", + "uint16", + "uint32", + "uint64", + "int8", + "int16", + "int32", + "int64", + "float8_e4m3fn", + "float8_e5m2", + "float16", + "bfloat16", + "float32", + "float64", + "complex64", + } +) + + +@dataclass(frozen=True) +class WriterLimits: + """Admission limits applied before a staging directory is created.""" + + max_shard_bytes: int = 5 << 30 + max_output_bytes: int = 1 << 40 + max_temp_bytes: int = 1 << 40 + max_peak_ram_bytes: int = 16 << 30 + min_free_headroom_percent: int = 10 + + def __post_init__(self) -> None: + for name, value in vars(self).items(): + if type(value) is not int or value <= 0: + raise ValueError(f"{name} must be a positive integer") + if self.min_free_headroom_percent > 100: + raise ValueError("min_free_headroom_percent cannot exceed 100") + + +@dataclass(frozen=True) +class VerifiedSourceFile: + """One source artifact whose path and digest are independently rechecked.""" + + path: Path + relative_path: str + expected_sha256: str + + def __post_init__(self) -> None: + object.__setattr__(self, "path", Path(self.path)) + _safe_relative(self.relative_path) + _require_digest(self.expected_sha256, "source digest") + + +@dataclass(frozen=True) +class ImmutableCopy: + """A base configuration/tokenizer file copied only after exact hash match.""" + + relative_path: str + source_path: Path + expected_sha256: str + kind: str + + def __post_init__(self) -> None: + object.__setattr__(self, "source_path", Path(self.source_path)) + _safe_basename(self.relative_path) + _require_digest(self.expected_sha256, "copy digest") + if self.kind not in _COPY_KINDS: + raise ValueError("copy kind must be configuration or tokenizer") + + +@dataclass(frozen=True) +class CanonicalWriteResult: + """Identity and immutable location of a successfully promoted checkpoint.""" + + artifact_id: str + output_path: Path + weight_files: tuple[str, ...] + manifest_digest: str + + +@dataclass(frozen=True) +class _Fingerprint: + device: int + inode: int + mode: int + size: int + modified_ns: int + + @classmethod + def from_stat(cls, value: os.stat_result) -> _Fingerprint: + return cls(value.st_dev, value.st_ino, value.st_mode, value.st_size, value.st_mtime_ns) + + +@dataclass(frozen=True) +class _Snapshot: + path: Path + relative_path: str + fingerprint: _Fingerprint + sha256: str + + def file_record(self) -> dict[str, Any]: + return { + "relative_path": self.relative_path, + "size_bytes": self.fingerprint.size, + "sha256": self.sha256, + } + + +@dataclass(frozen=True) +class _TensorSpec: + name: str + logical_tensor_id: str + size_bytes: int + + +@dataclass(frozen=True) +class _TensorOracle: + shape: tuple[int, ...] + dtype: str + sha256: str + + +@dataclass(frozen=True) +class _Admission: + logical_bytes: int + estimated_output_bytes: int + estimated_temp_bytes: int + estimated_peak_ram_bytes: int + + +def _safe_relative(value: str) -> None: + pure = PurePosixPath(value) + if ( + not isinstance(value, str) + or not value + or pure.is_absolute() + or any(part in {"", ".", ".."} for part in pure.parts) + or len(value) > 4096 + ): + raise ValueError("relative path is unsafe") + + +def _safe_basename(value: str) -> None: + _safe_relative(value) + if PurePosixPath(value).name != value: + raise ValueError("canonical copied artifacts must be root-level files") + + +def _require_digest(value: str, field: str) -> None: + if ( + not isinstance(value, str) + or not value.startswith(_DIGEST_PREFIX) + or len(value) != len(_DIGEST_PREFIX) + 64 + or any(character not in "0123456789abcdef" for character in value[len(_DIGEST_PREFIX) :]) + ): + raise ValueError(f"{field} must be a sha256 digest") + + +def _source_changed(reference: str) -> CheckpointContractError: + return CheckpointContractError( + "DCI_SOURCE_CHANGED", + detail="source_digest_or_identity_changed", + affected_refs=(reference,), + ) + + +def _open_flags() -> int: + return os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + + +def _open_snapshot(path: Path, reference: str) -> int: + try: + return os.open(path, _open_flags()) + except OSError as error: + raise _source_changed(reference) from error + + +def _hash_descriptor(descriptor: int, reference: str) -> str: + digest = sha256() + try: + while True: + chunk = os.read(descriptor, 1 << 20) + if not chunk: + return f"sha256:{digest.hexdigest()}" + digest.update(chunk) + except OSError as error: + raise _source_changed(reference) from error + + +def _read_snapshot(descriptor: int, size: int, reference: str) -> bytes: + try: + return os.read(descriptor, size) + except OSError as error: + raise _source_changed(reference) from error + + +def _descriptor_fingerprint(descriptor: int, reference: str) -> _Fingerprint: + try: + return _Fingerprint.from_stat(os.fstat(descriptor)) + except OSError as error: + raise _source_changed(reference) from error + + +def _snapshot(path: Path, relative_path: str, expected_digest: str) -> _Snapshot: + try: + observed = path.lstat() + except OSError as error: + raise _source_changed(relative_path) from error + if not stat.S_ISREG(observed.st_mode): + raise CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail="source_not_regular_file", + affected_refs=(relative_path,), + ) + try: + if path.absolute() != path.resolve(strict=True): + raise CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail="source_symlink", + affected_refs=(relative_path,), + ) + except OSError as error: + raise _source_changed(relative_path) from error + fingerprint = _Fingerprint.from_stat(observed) + descriptor = _open_snapshot(path, relative_path) + try: + if _descriptor_fingerprint(descriptor, relative_path) != fingerprint: + raise _source_changed(relative_path) + digest = _hash_descriptor(descriptor, relative_path) + if _descriptor_fingerprint(descriptor, relative_path) != fingerprint: + raise _source_changed(relative_path) + finally: + os.close(descriptor) + if digest != expected_digest: + raise _source_changed(relative_path) + try: + if _Fingerprint.from_stat(path.lstat()) != fingerprint: + raise _source_changed(relative_path) + except OSError as error: + raise _source_changed(relative_path) from error + return _Snapshot(path, relative_path, fingerprint, digest) + + +def _revalidate_snapshot(snapshot: _Snapshot) -> None: + try: + observed = _Fingerprint.from_stat(snapshot.path.lstat()) + except OSError as error: + raise _source_changed(snapshot.relative_path) from error + if observed != snapshot.fingerprint: + raise _source_changed(snapshot.relative_path) + descriptor = _open_snapshot(snapshot.path, snapshot.relative_path) + try: + if _descriptor_fingerprint(descriptor, snapshot.relative_path) != snapshot.fingerprint: + raise _source_changed(snapshot.relative_path) + digest = _hash_descriptor(descriptor, snapshot.relative_path) + finally: + os.close(descriptor) + if digest != snapshot.sha256: + raise _source_changed(snapshot.relative_path) + + +def _tensor_bytes(shape: tuple[int, ...], element_size: int, reference: str) -> int: + elements = 1 + for dimension in shape: + if dimension and elements > _INT64_MAX // dimension: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="integer_overflow", + affected_refs=(reference,), + ) + elements *= dimension + if elements and elements > _INT64_MAX // element_size: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="integer_overflow", + affected_refs=(reference,), + ) + return elements * element_size + + +def _tensor_specs(validation: ValidationResult) -> tuple[_TensorSpec, ...]: + names = [logical.fqn for logical in validation.logical_tensors] + if len(set(names)) != len(names): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="canonical_tensor_name_duplicate", + affected_refs=names, + ) + specs = [] + for logical in validation.logical_tensors: + if logical.dtype not in _CANONICAL_DTYPES: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="canonical_dtype_unsupported", + affected_refs=(logical.logical_tensor_id,), + ) + payload = logical.fragments[0].payload + if payload is None: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="payload_unavailable", + affected_refs=(logical.logical_tensor_id,), + ) + specs.append( + _TensorSpec( + name=logical.fqn, + logical_tensor_id=logical.logical_tensor_id, + size_bytes=_tensor_bytes( + logical.global_shape, + payload.element_size(), + logical.logical_tensor_id, + ), + ) + ) + return tuple(sorted(specs, key=lambda item: item.name)) + + +def _shard_plan(specs: tuple[_TensorSpec, ...], max_shard_bytes: int) -> tuple[tuple[_TensorSpec, ...], ...]: + oversized = next((spec for spec in specs if spec.size_bytes > max_shard_bytes), None) + if oversized is not None: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail="tensor_exceeds_max_shard_bytes", + affected_refs=(oversized.logical_tensor_id,), + ) + shards: list[list[_TensorSpec]] = [] + current: list[_TensorSpec] = [] + current_bytes = 0 + for spec in specs: + if current and current_bytes + spec.size_bytes > max_shard_bytes: + shards.append(current) + current = [] + current_bytes = 0 + current.append(spec) + current_bytes += spec.size_bytes + if current: + shards.append(current) + return tuple(tuple(shard) for shard in shards) + + +def _admit( + destination: Path, + plan: tuple[tuple[_TensorSpec, ...], ...], + copy_snapshots: tuple[_Snapshot, ...], + limits: WriterLimits, +) -> _Admission: + logical_bytes = sum(spec.size_bytes for shard in plan for spec in shard) + copied_bytes = sum(snapshot.fingerprint.size for snapshot in copy_snapshots) + header_allowance = ( + sum(len(spec.name.encode("utf-8")) + 256 for shard in plan for spec in shard) + + 4096 * max(1, len(plan)) + + 8192 + ) + estimated_output = logical_bytes + copied_bytes + header_allowance + estimated_temp = estimated_output + largest_shard = max((sum(spec.size_bytes for spec in shard) for shard in plan), default=0) + largest_tensor = max((spec.size_bytes for shard in plan for spec in shard), default=0) + peak_ram = largest_shard + largest_tensor + _WRITER_BUFFER_BYTES + checks = ( + (estimated_output, limits.max_output_bytes, "max_output_bytes"), + (estimated_temp, limits.max_temp_bytes, "max_temp_bytes"), + (peak_ram, limits.max_peak_ram_bytes, "max_peak_ram_bytes"), + ) + for required, maximum, detail in checks: + if required > maximum: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail=detail, + affected_refs=(f"required:{required}", f"available:{maximum}"), + ) + try: + destination.parent.mkdir(parents=True, exist_ok=True) + except OSError as error: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail="destination_parent_unavailable", + affected_refs=("destination",), + ) from error + if destination.parent.is_symlink(): + raise CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail="destination_parent_symlink", + affected_refs=("destination",), + ) + try: + available = shutil.disk_usage(destination.parent).free + except OSError as error: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail="filesystem_capacity_unavailable", + affected_refs=("destination",), + ) from error + required_with_headroom = estimated_temp * (100 + limits.min_free_headroom_percent) + required_with_headroom = (required_with_headroom + 99) // 100 + if available < required_with_headroom: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail="filesystem_free_bytes", + affected_refs=(f"required:{required_with_headroom}", f"available:{available}"), + ) + return _Admission(logical_bytes, estimated_output, estimated_temp, peak_ram) + + +def _copy_snapshot(snapshot: _Snapshot, destination: Path) -> None: + source = _open_snapshot(snapshot.path, snapshot.relative_path) + try: + if _descriptor_fingerprint(source, snapshot.relative_path) != snapshot.fingerprint: + raise _source_changed(snapshot.relative_path) + with destination.open("xb") as output: + while True: + chunk = _read_snapshot(source, 1 << 20, snapshot.relative_path) + if not chunk: + break + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + if _descriptor_fingerprint(source, snapshot.relative_path) != snapshot.fingerprint: + raise _source_changed(snapshot.relative_path) + finally: + os.close(source) + if _file_digest(destination) != snapshot.sha256: + raise _source_changed(snapshot.relative_path) + + +def _save_safetensors_file(tensors: Mapping[str, torch.Tensor], path: Path) -> None: + save_file(dict(sorted(tensors.items())), path) + + +def _file_digest(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return f"sha256:{digest.hexdigest()}" + + +def _tensor_digest(tensor: torch.Tensor) -> str: + raw = tensor.detach().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + return f"sha256:{sha256(raw).hexdigest()}" + + +def _write_json(path: Path, value: object) -> None: + payload = json.dumps( + value, + indent=2, + sort_keys=True, + ensure_ascii=True, + allow_nan=False, + ) + "\n" + with path.open("x", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + + +def _output_record(path: Path) -> dict[str, Any]: + observed = path.lstat() + if not stat.S_ISREG(observed.st_mode): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_not_regular_file", + affected_refs=(path.name,), + ) + return { + "relative_path": path.name, + "size_bytes": observed.st_size, + "sha256": _file_digest(path), + } + + +def _canonical_digest(value: object) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + return f"sha256:{sha256(payload).hexdigest()}" + + +def _manifest( + *, + artifact_id: str, + descriptor_digest: str, + source_records: list[dict[str, Any]], + source_topology: Mapping[str, Any], + validation: ValidationResult, + output_records: list[dict[str, Any]], + index_name: str | None, + admission: _Admission, + provenance: dict[str, Any], +) -> dict[str, Any]: + lost_state = provenance["state"]["lost_state"] + return { + "schema_id": "obliteratus.conversion-manifest", + "schema_version": "1.0.0", + "manifest_id": artifact_id, + "descriptor": {"schema_version": "1.0.0", "digest": descriptor_digest}, + "source_inventory_digest": _canonical_digest(source_records), + "source_files": source_records, + "adapter": { + "id": "producer-neutral-fragments", + "version": "1.0.0", + "capability_digest": validation.manifest_digest, + "producer": "already-normalized-input", + "producer_version": "1.0.0", + }, + "source_topology": dict(source_topology), + "state": { + "source_classification": provenance["state"]["classification"], + "output_classification": "weights_only", + "observed_scopes": provenance["state"]["observed_scopes"], + }, + "dropped_scopes": [ + {"scope": scope, "reason": "Canonical Wave 2 output contains weights only."} + for scope in lost_state + ], + "canonical_output": { + "format": "hf_safetensors", + "dtype_policy": "preserve_exact", + "files": output_records, + "hf_index": index_name, + "logical_tensor_count": len(validation.logical_tensors), + "logical_bytes": admission.logical_bytes, + }, + "resource_usage": { + "estimated_peak_ram_bytes": admission.estimated_peak_ram_bytes, + "actual_peak_ram_bytes": None, + "estimated_temp_bytes": admission.estimated_temp_bytes, + "actual_temp_bytes": None, + }, + "validation": { + "coverage": True, + "replicas": True, + "ties": True, + "hashes": True, + "index": True, + "safe_reload": True, + "source_unchanged": True, + "result": "passed", + }, + "provenance": { + "obliteratus_commit": provenance["obliteratus_commit"], + "configuration_digest": provenance["configuration_digest"], + "tokenizer_digest": ( + provenance["tokenizer"]["digest"] if provenance["tokenizer"] else None + ), + "base_model": { + "identity": provenance["base_model"]["identity"] if provenance["base_model"] else None, + "revision": provenance["base_model"]["revision"] if provenance["base_model"] else None, + "digest": provenance["base_model"]["digest"] if provenance["base_model"] else None, + }, + "transformation_log": provenance["transformations"], + "unknowns": provenance["unknowns"], + }, + "publication": { + "staging_validated": True, + "promoted": True, + "atomic_strategy": "sibling-stage-fsync-replace", + "rollback_result": "not_required", + }, + } + + +def _verify_provenance( + record: dict[str, Any], + *, + output_digests: tuple[str, ...], + source_digests: tuple[str, ...], + configuration_digest: str, + source_topology: Mapping[str, Any], +) -> dict[str, Any]: + try: + verified = verify_provenance_record(record) + except (TypeError, ValueError) as error: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_contract_invalid", + ) from error + if sorted(verified["output_digests"]) != sorted(set(output_digests)): + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_output_digest_mismatch", + ) + if sorted(verified["input_digests"]) != sorted(set(source_digests)): + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_input_digest_mismatch", + ) + if verified["configuration_digest"] != configuration_digest: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_configuration_digest_mismatch", + ) + try: + topology_matches = dict(source_topology) == verified["source_topology"] + except (TypeError, ValueError) as error: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_source_topology_invalid", + ) from error + if not topology_matches: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_source_topology_mismatch", + ) + return verified + + +def _verify_json_object(path: Path) -> dict[str, Any]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 << 20: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_json_invalid", + affected_refs=(path.name,), + ) + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (UnicodeError, json.JSONDecodeError) as error: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_json_invalid", + affected_refs=(path.name,), + ) from error + if not isinstance(value, dict): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_json_invalid", + affected_refs=(path.name,), + ) + return value + + +def _verify_staging( + staging: Path, + *, + weight_map: dict[str, str], + oracles: dict[str, _TensorOracle], + expected_files: set[str], + snapshots: tuple[_Snapshot, ...], + source_snapshots: tuple[_Snapshot, ...], + configuration_digest: str, + descriptor_digest: str, + validation_digest: str, + artifact_id: str, + limits: WriterLimits, +) -> None: + actual_files = {path.name for path in staging.iterdir() if path.is_file() and not path.is_symlink()} + if actual_files != expected_files or any(path.is_symlink() or not path.is_file() for path in staging.iterdir()): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_file_set_mismatch", + ) + actual_bytes = sum((staging / name).lstat().st_size for name in expected_files) + if actual_bytes > limits.max_output_bytes or actual_bytes > limits.max_temp_bytes: + raise CheckpointContractError( + "DCI_ADMISSION_DENIED", + detail="post_write_size_limit", + affected_refs=(f"required:{actual_bytes}",), + ) + loaded_names: set[str] = set() + loaded_bytes = 0 + for shard_name in sorted(set(weight_map.values())): + shard = load_file(staging / shard_name, device="cpu") + for name, tensor in shard.items(): + if weight_map.get(name) != shard_name or name not in oracles: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_index_mismatch", + affected_refs=(name,), + ) + oracle = oracles[name] + if ( + tuple(tensor.shape) != oracle.shape + or str(tensor.dtype).removeprefix("torch.") != oracle.dtype + or _tensor_digest(tensor) != oracle.sha256 + ): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_tensor_mismatch", + affected_refs=(name,), + ) + loaded_names.add(name) + loaded_bytes += tensor.numel() * tensor.element_size() + if loaded_names != set(weight_map) or loaded_names != set(oracles): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_tensor_set_mismatch", + ) + index_name = ( + "model.safetensors.index.json" + if "model.safetensors.index.json" in expected_files + else None + ) + if index_name is not None: + index = _verify_json_object(staging / index_name) + if index != { + "metadata": {"total_size": loaded_bytes}, + "weight_map": dict(sorted(weight_map.items())), + }: + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_index_mismatch", + ) + for name in ("config.json", "tokenizer_config.json"): + _verify_json_object(staging / name) + provenance = _verify_json_object(staging / "checkpoint-provenance.json") + manifest = _verify_json_object(staging / "conversion-manifest.json") + metadata = _verify_json_object(staging / "abliteration_metadata.json") + evidence_files = { + "checkpoint-provenance.json", + "conversion-manifest.json", + "abliteration_metadata.json", + } + output_records = [ + _output_record(staging / name) + for name in sorted(expected_files - evidence_files) + ] + output_digests = tuple(record["sha256"] for record in output_records) + verified_provenance = _verify_provenance( + provenance, + output_digests=output_digests, + source_digests=tuple(snapshot.sha256 for snapshot in source_snapshots), + configuration_digest=configuration_digest, + source_topology=provenance.get("source_topology", {}), + ) + canonical_output = manifest.get("canonical_output") + adapter = manifest.get("adapter") + descriptor = manifest.get("descriptor") + if ( + not isinstance(canonical_output, dict) + or not isinstance(adapter, dict) + or not isinstance(descriptor, dict) + or canonical_output.get("files") != output_records + or canonical_output.get("hf_index") != index_name + or adapter.get("capability_digest") != validation_digest + or descriptor.get("digest") != descriptor_digest + or manifest.get("source_files") + != [snapshot.file_record() for snapshot in source_snapshots] + or manifest.get("source_topology") != verified_provenance["source_topology"] + ): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="output_manifest_mismatch", + ) + if not ( + provenance.get("artifact_id") + == manifest.get("manifest_id") + == metadata.get("artifact_id") + == artifact_id + ): + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="artifact_identity_mismatch", + ) + for snapshot in snapshots: + _revalidate_snapshot(snapshot) + + +def write_canonical_checkpoint( + destination: Path | str, + fragments: Sequence[TensorFragment], + *, + source_files: Sequence[VerifiedSourceFile], + copies: Sequence[ImmutableCopy], + descriptor_digest: str, + source_topology: Mapping[str, Any], + provenance_factory: Callable[[tuple[str, ...]], ProvenanceRecord], + tie_policy: str | None = None, + limits: WriterLimits | None = None, +) -> CanonicalWriteResult: + """Write validated CPU fragments without invoking a producer reader or adapter.""" + _require_digest(descriptor_digest, "descriptor digest") + active_limits = limits or WriterLimits() + validation = validate_fragments(tuple(fragments)) + if any(logical.tie_group_id is not None for logical in validation.logical_tensors): + if tie_policy != "duplicate_validated": + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="tie_policy_required", + affected_refs=( + logical.logical_tensor_id + for logical in validation.logical_tensors + if logical.tie_group_id is not None + ), + ) + if not source_files: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="source_inventory_empty", + ) + source_paths = [item.relative_path for item in source_files] + if len(set(source_paths)) != len(source_paths): + raise ValueError("source relative paths must be unique") + copy_paths = [item.relative_path for item in copies] + if len(set(copy_paths)) != len(copy_paths): + raise ValueError("copy output paths must be unique") + if not {"config.json", "tokenizer_config.json"} <= set(copy_paths): + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="canonical_config_or_tokenizer_missing", + ) + copy_kinds = {item.relative_path: item.kind for item in copies} + if ( + copy_kinds["config.json"] != "configuration" + or copy_kinds["tokenizer_config.json"] != "tokenizer" + ): + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="canonical_config_or_tokenizer_kind_mismatch", + ) + source_snapshots = tuple( + sorted( + ( + _snapshot(item.path, item.relative_path, item.expected_sha256) + for item in source_files + ), + key=lambda snapshot: snapshot.relative_path, + ) + ) + copy_snapshots = tuple( + _snapshot(item.source_path, item.relative_path, item.expected_sha256) for item in copies + ) + specs = _tensor_specs(validation) + plan = _shard_plan(specs, active_limits.max_shard_bytes) + output_path = Path(destination) + try: + destination_unsafe = ( + output_path.is_symlink() + or output_path.absolute() != output_path.resolve(strict=False) + ) + except OSError as error: + raise CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail="destination_unresolvable", + affected_refs=("destination",), + ) from error + if destination_unsafe: + raise CheckpointContractError( + "DCI_SOURCE_BOUNDARY_VIOLATION", + detail="destination_symlink", + affected_refs=("destination",), + ) + admission = _admit(output_path, plan, copy_snapshots, active_limits) + shard_names = ( + ("model.safetensors",) + if len(plan) == 1 + else tuple( + f"model-{index:05d}-of-{len(plan):05d}.safetensors" + for index in range(1, len(plan) + 1) + ) + ) + weight_map = { + spec.name: shard_name + for shard_name, shard in zip(shard_names, plan, strict=True) + for spec in shard + } + oracles: dict[str, _TensorOracle] = {} + index_name = "model.safetensors.index.json" if len(plan) > 1 else None + artifact_id = "" + manifest_digest = "" + all_snapshots = (*source_snapshots, *copy_snapshots) + + def verify(staging: Path) -> None: + expected = { + *copy_paths, + *shard_names, + "checkpoint-provenance.json", + "conversion-manifest.json", + "abliteration_metadata.json", + } + if index_name is not None: + expected.add(index_name) + _verify_staging( + staging, + weight_map=weight_map, + oracles=oracles, + expected_files=expected, + snapshots=all_snapshots, + source_snapshots=source_snapshots, + configuration_digest=next( + snapshot.sha256 + for snapshot in copy_snapshots + if snapshot.relative_path == "config.json" + ), + descriptor_digest=descriptor_digest, + validation_digest=validation.manifest_digest, + artifact_id=artifact_id, + limits=active_limits, + ) + + try: + with atomic_checkpoint_directory(output_path, validate=verify) as staging: + try: + for copy, snapshot in zip(copies, copy_snapshots, strict=True): + _copy_snapshot(snapshot, staging / copy.relative_path) + for shard_name, shard in zip(shard_names, plan, strict=True): + tensors: dict[str, torch.Tensor] = {} + for spec in shard: + tensor = reconstruct_logical_tensor(validation, spec.logical_tensor_id) + tensors[spec.name] = tensor + oracles[spec.name] = _TensorOracle( + shape=tuple(tensor.shape), + dtype=str(tensor.dtype).removeprefix("torch."), + sha256=_tensor_digest(tensor), + ) + _save_safetensors_file(tensors, staging / shard_name) + del tensors + if index_name is not None: + _write_json( + staging / index_name, + { + "metadata": {"total_size": admission.logical_bytes}, + "weight_map": dict(sorted(weight_map.items())), + }, + ) + output_names = [*copy_paths, *shard_names] + if index_name is not None: + output_names.append(index_name) + output_records = [ + _output_record(staging / name) for name in sorted(output_names) + ] + output_digests = tuple(sorted(item["sha256"] for item in output_records)) + try: + provenance_record = provenance_factory(output_digests) + provenance = _verify_provenance( + provenance_record.to_dict(), + output_digests=output_digests, + source_digests=tuple( + snapshot.sha256 for snapshot in source_snapshots + ), + configuration_digest=next( + snapshot.sha256 + for snapshot in copy_snapshots + if snapshot.relative_path == "config.json" + ), + source_topology=source_topology, + ) + except CheckpointContractError: + raise + except Exception as error: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_factory_failed", + ) from error + artifact_id = provenance["artifact_id"] + if getattr(provenance_record, "artifact_id", None) != artifact_id: + raise CheckpointContractError( + "DCI_EVIDENCE_UNAVAILABLE", + detail="provenance_artifact_identity_mismatch", + ) + _write_json(staging / "checkpoint-provenance.json", provenance) + source_records = [snapshot.file_record() for snapshot in source_snapshots] + manifest = _manifest( + artifact_id=artifact_id, + descriptor_digest=descriptor_digest, + source_records=source_records, + source_topology=provenance["source_topology"], + validation=validation, + output_records=output_records, + index_name=index_name, + admission=admission, + provenance=provenance, + ) + _write_json(staging / "conversion-manifest.json", manifest) + manifest_digest = _file_digest(staging / "conversion-manifest.json") + _write_json( + staging / "abliteration_metadata.json", + { + "schema_version": 2, + "artifact_id": artifact_id, + "checkpoint_provenance": "checkpoint-provenance.json", + "conversion_manifest": "conversion-manifest.json", + "state_classification": "weights_only", + "lost_state": provenance["state"]["lost_state"], + }, + ) + except CheckpointContractError: + raise + except Exception as error: + raise CheckpointContractError( + "DCI_MATERIALIZE_FAILED", + detail="canonical_write_failed", + ) from error + except CheckpointContractError: + raise + except Exception as error: + raise CheckpointContractError( + "DCI_PROMOTION_FAILED", + detail="canonical_promotion_failed", + ) from error + return CanonicalWriteResult( + artifact_id=artifact_id, + output_path=output_path.resolve(), + weight_files=shard_names, + manifest_digest=manifest_digest, + ) diff --git a/obliteratus/cli.py b/obliteratus/cli.py index 9c896e0..15feb5a 100644 --- a/obliteratus/cli.py +++ b/obliteratus/cli.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import json import os +import sys import tempfile from pathlib import Path @@ -148,7 +149,35 @@ def _apply_gpu_selection(args): def main(argv: list[str] | None = None): - console.print(_banner_for_console()) + effective_argv = list(sys.argv[1:]) if argv is None else argv + sensitive_options = { + "--api-key", + "--credential", + "--credentials", + "--password", + "--private-key", + "--secret", + "--token", + } + if any(item.split("=", 1)[0].lower() in sensitive_options for item in effective_argv): + console.print("[red]Secret-bearing command-line inputs are forbidden.[/]") + raise SystemExit(2) + if ( + "distributed" in effective_argv + and "preflight" in effective_argv + and effective_argv[:2] != ["distributed", "preflight"] + ): + console.print("[red]Distributed preflight command shape is invalid.[/]") + raise SystemExit(2) + if effective_argv[:2] == ["distributed", "preflight"]: + allowed_options = {"--json"} + positional = [item for item in effective_argv[2:] if item not in allowed_options] + unsafe_option = any( + item.startswith("--") and item not in allowed_options for item in effective_argv[2:] + ) + if unsafe_option or len(positional) != 1: + console.print("[red]Distributed preflight accepts only PROFILE.json and --json.[/]") + raise SystemExit(2) parser = argparse.ArgumentParser( prog="obliteratus", description="Master Ablation Suite for HuggingFace transformers", @@ -160,6 +189,53 @@ def main(argv: list[str] | None = None): ) subparsers = parser.add_subparsers(dest="command", required=True) + # --- checkpoint safe structural inspection --- + checkpoint_parser = subparsers.add_parser( + "checkpoint", + help="Inspect local checkpoint structure without loading tensor payloads", + ) + checkpoint_subparsers = checkpoint_parser.add_subparsers( + dest="checkpoint_command", + required=True, + ) + checkpoint_inspect = checkpoint_subparsers.add_parser( + "inspect", + help="Classify one local checkpoint using bounded inert evidence", + ) + checkpoint_inspect.add_argument("source", type=Path) + checkpoint_inspect.add_argument( + "--json", + action="store_true", + help="Emit only the versioned checkpoint descriptor JSON", + ) + checkpoint_inspect.add_argument("--max-files", type=_positive_int, default=None) + checkpoint_inspect.add_argument("--max-total-bytes", type=_positive_int, default=None) + checkpoint_inspect.add_argument("--max-json-bytes", type=_positive_int, default=None) + checkpoint_inspect.add_argument( + "--max-header-bytes", + type=_positive_int, + default=None, + help="Maximum safetensors header bytes", + ) + + # --- explicit fixed-membership distributed boundary --- + distributed_parser = subparsers.add_parser( + "distributed", + help="Validate an externally launched fixed worker group", + ) + distributed_subparsers = distributed_parser.add_subparsers( + dest="distributed_command", + required=True, + ) + distributed_preflight = distributed_subparsers.add_parser( + "preflight", + help="Validate a strict local profile before any model allocation", + ) + distributed_preflight.add_argument("profile", type=Path) + distributed_preflight.add_argument( + "--json", action="store_true", help="Emit only bounded redacted result JSON" + ) + # --- run --- run_parser = subparsers.add_parser("run", help="Run an ablation from a YAML config") run_parser.add_argument("config", type=str, help="Path to YAML config file") @@ -585,7 +661,19 @@ def main(argv: list[str] | None = None): runs_prune.add_argument("run_id") runs_prune.add_argument("--reason", required=True) - args = parser.parse_args(argv) + args = parser.parse_args(effective_argv) + + machine_json = ( + args.command == "checkpoint" + and args.checkpoint_command == "inspect" + and args.json + ) or ( + args.command == "distributed" + and args.distributed_command == "preflight" + and args.json + ) + if not machine_json: + console.print(_banner_for_console()) if getattr(args, "prompt_pairs_file", None): residue_only_options = [] @@ -604,6 +692,12 @@ def main(argv: list[str] | None = None): # Apply GPU selection early (before any CUDA init) _apply_gpu_selection(args) + if args.command == "checkpoint": + _cmd_checkpoint(args) + return + if args.command == "distributed": + _cmd_distributed(args) + return if args.command == "gpu-calc": _cmd_gpu_calc(args) return @@ -687,6 +781,88 @@ def main(argv: list[str] | None = None): _cmd_abliterate(args) +def _cmd_checkpoint(args): + """Run the Wave 2 structure-only checkpoint inspection surface.""" + + from rich.table import Table + + from obliteratus.checkpoint_errors import CheckpointContractError + from obliteratus.checkpoint_inspection import InspectionLimits + from obliteratus.checkpoint_service import CheckpointService + + defaults = InspectionLimits() + limits = InspectionLimits( + max_files=args.max_files or defaults.max_files, + max_directories=defaults.max_directories, + max_total_bytes=args.max_total_bytes or defaults.max_total_bytes, + max_json_bytes=args.max_json_bytes or defaults.max_json_bytes, + max_safetensors_header_bytes=( + args.max_header_bytes or defaults.max_safetensors_header_bytes + ), + max_tensors=defaults.max_tensors, + hash_chunk_bytes=defaults.hash_chunk_bytes, + ) + try: + report = CheckpointService(inspection_limits=limits).inspect(args.source) + except CheckpointContractError as error: + if args.json: + print(json.dumps(error.to_diagnostic(), indent=2, sort_keys=True)) + else: + console.print(f"[bold red]{error.code}[/]: {error.detail}") + console.print(error.next_action) + raise SystemExit(2) from None + if args.json: + print(report.to_json(), end="") + return + descriptor = report.to_dict() + table = Table(title="Checkpoint structural inspection") + table.add_column("Field", style="cyan") + table.add_column("Result") + table.add_row("format", descriptor["primary_format"]) + table.add_row("confidence", descriptor["classification_confidence"]) + table.add_row("support", descriptor["support_decision"]) + table.add_row("trust required", str(descriptor["safety"]["trust_required"]).lower()) + table.add_row("files", str(descriptor["resource_estimate"]["file_count"])) + table.add_row("tensors", str(descriptor["resource_estimate"]["tensor_count"])) + console.print(table) + for blocker in descriptor["blockers"]: + console.print(f"[yellow]{blocker['code']}[/]: {blocker['next_action']}") + + +def _cmd_distributed(args): + """Run only the explicit scheduler-provided distributed preflight.""" + + from obliteratus.distributed.config import DistributedPreflightConfig + from obliteratus.distributed.contracts import ContractError, RuntimeContractError + from obliteratus.distributed.launcher import TorchrunEnvironment + from obliteratus.distributed.preflight import execute_preflight + + try: + config = DistributedPreflightConfig.from_file(args.profile) + launch = TorchrunEnvironment.from_environ(os.environ, config) + evidence = execute_preflight(config, launch) + except ContractError as error: + code = ( + error.code + if isinstance(error, RuntimeContractError) + else "LMS_LAUNCH_IDENTITY_INVALID" + ) + payload = {"schema_version": 1, "result": "failed", "error_code": code} + if args.json: + print(json.dumps(payload, sort_keys=True)) + else: + console.print(f"[bold red]{code}[/]: distributed preflight refused") + raise SystemExit(2) from None + payload = json.loads(evidence.to_bytes()) + if args.json: + print(json.dumps(payload, sort_keys=True)) + else: + console.print( + f"[green]Distributed preflight accepted {evidence.accepted_ranks}/" + f"{evidence.world_size} ranks.[/]" + ) + + def _cmd_runs(args): """Dispatch the stable durable-run control surface.""" diff --git a/obliteratus/distributed/__init__.py b/obliteratus/distributed/__init__.py new file mode 100644 index 0000000..855a638 --- /dev/null +++ b/obliteratus/distributed/__init__.py @@ -0,0 +1,46 @@ +"""Fail-closed contracts for the opt-in distributed runtime prototype.""" + +from obliteratus.distributed.config import DistributedPreflightConfig +from obliteratus.distributed.consensus import require_record_consensus +from obliteratus.distributed.contracts import ( + ContractError, + LogicalPlacement, + PlacementKind, + RankInventory, + RunIdentity, + RuntimeStage, + RuntimeContractError, + StageMessage, + TopologyPlan, + Vote, + advance_stage, + canonical_record, + contract_digest, + validate_inventory_consensus, +) +from obliteratus.distributed.evidence import PreflightEvidence +from obliteratus.distributed.launcher import TorchrunEnvironment +from obliteratus.distributed.preflight import PreflightResult, RankAttestation + +__all__ = [ + "ContractError", + "DistributedPreflightConfig", + "LogicalPlacement", + "PlacementKind", + "PreflightEvidence", + "PreflightResult", + "RankInventory", + "RunIdentity", + "RankAttestation", + "RuntimeStage", + "RuntimeContractError", + "StageMessage", + "TopologyPlan", + "TorchrunEnvironment", + "Vote", + "advance_stage", + "canonical_record", + "contract_digest", + "require_record_consensus", + "validate_inventory_consensus", +] diff --git a/obliteratus/distributed/config.py b/obliteratus/distributed/config.py new file mode 100644 index 0000000..9452ae1 --- /dev/null +++ b/obliteratus/distributed/config.py @@ -0,0 +1,498 @@ +"""Strict configuration for the explicit distributed preflight surface.""" + +from __future__ import annotations + +import ipaddress +import json +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from obliteratus.distributed.contracts import ContractError, contract_digest + + +MAX_PROFILE_BYTES = 64 * 1024 +MAX_TIMEOUT_SECONDS = 3600 +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_RUN_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_INTERFACE_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$") +_ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,127}$") +_SECRET_KEYS = frozenset( + {"password", "secret", "token", "api_key", "api-key", "credential", "credentials"} +) + +_ROOT_FIELDS = frozenset( + { + "schema_version", + "run", + "identity", + "topology", + "network", + "source", + "staging", + "resources", + "timeouts", + "software", + "execution", + "evidence", + } +) +_SECTION_FIELDS = { + "run": frozenset({"run_id", "rendezvous_id", "world_size", "local_world_size"}), + "identity": frozenset( + { + "source_digest", + "model_digest", + "tokenizer_digest", + "commit_sha", + "code_digest", + } + ), + "topology": frozenset( + { + "tensor_parallel_size", + "coordinator_rank", + "placement_plan_digest", + "dimension_divisors", + } + ), + "network": frozenset( + {"master_addr", "master_port", "interface", "allowed_master_cidrs"} + ), + "source": frozenset({"path"}), + "staging": frozenset({"path", "storage_digest"}), + "resources": frozenset( + { + "min_free_device_memory_bytes", + "min_free_host_memory_bytes", + "min_free_staging_bytes", + "max_source_files", + "max_source_bytes", + "max_source_file_bytes", + } + ), + "timeouts": frozenset( + {"source_seconds", "init_seconds", "collective_seconds", "teardown_seconds"} + ), + "software": frozenset( + { + "python", + "platform", + "machine", + "torch", + "transformers", + "accelerate", + "safetensors", + "cuda", + "nccl", + "driver", + } + ), + "execution": frozenset( + { + "device_kind", + "device_name", + "compute_capability", + "evidence_tier", + "allowed_environment_keys", + "local_files_only", + "trust_remote_code", + "allow_runtime_install", + "allow_plugins", + "allow_compilation", + "allow_adapters", + "allow_quantization", + } + ), + "evidence": frozenset({"path"}), +} + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ContractError("distributed profile contains a duplicate field") + result[key] = value + return result + + +def _mapping(value: object, name: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ContractError(f"{name} must be an object") + return value + + +def _integer(value: object, name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ContractError(f"{name} must be an integer") + if not minimum <= value <= maximum: + raise ContractError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _text(value: object, name: str, pattern: re.Pattern[str] | None = None) -> str: + try: + encoded = value.encode("utf-8") if isinstance(value, str) else b"" + except UnicodeError as exc: + raise ContractError(f"{name} must be valid UTF-8") from exc + if not isinstance(value, str) or not value or len(encoded) > 256: + raise ContractError(f"{name} must be bounded non-empty text") + if pattern is not None and pattern.fullmatch(value) is None: + raise ContractError(f"{name} has an invalid format") + return value + + +def _absolute_path(value: object, name: str) -> Path: + text = _text(value, name) + path = Path(text) + if not path.is_absolute() or "://" in text or "\x00" in text: + raise ContractError(f"{name} must be an absolute local path") + return path + + +def _closed_section(root: dict[str, Any], name: str) -> dict[str, Any]: + section = _mapping(root.get(name), name) + unknown = set(section) - _SECTION_FIELDS[name] + missing = _SECTION_FIELDS[name] - set(section) + if unknown: + raise ContractError(f"unknown profile field in {name}") + if missing: + raise ContractError(f"required profile field is missing from {name}") + return section + + +@dataclass(frozen=True) +class DistributedPreflightConfig: + """Validated, closed-schema inputs for one non-resumable attempt.""" + + run_id: str + rendezvous_id: str + world_size: int + local_world_size: int + source_digest: str + model_digest: str + tokenizer_digest: str + commit_sha: str + code_digest: str + tensor_parallel_size: int + coordinator_rank: int + placement_plan_digest: str + dimension_divisors: tuple[int, ...] + master_addr: str + master_port: int + network_interface: str + allowed_master_cidrs: tuple[str, ...] + source_path: Path + staging_path: Path + storage_digest: str + min_free_device_memory_bytes: int + min_free_host_memory_bytes: int + min_free_staging_bytes: int + max_source_files: int + max_source_bytes: int + max_source_file_bytes: int + source_timeout_seconds: int + init_timeout_seconds: int + collective_timeout_seconds: int + teardown_timeout_seconds: int + software_versions: tuple[tuple[str, str], ...] + device_kind: str + device_name: str + compute_capability: str + evidence_tier: str + allowed_environment_keys: tuple[str, ...] + local_files_only: bool + trust_remote_code: bool + allow_runtime_install: bool + allow_plugins: bool + allow_compilation: bool + allow_adapters: bool + allow_quantization: bool + evidence_path: Path + digest: str + + @classmethod + def from_file(cls, path: str | Path) -> "DistributedPreflightConfig": + profile_path = Path(path) + descriptor: int | None = None + try: + descriptor = os.open(profile_path, os.O_RDONLY | os.O_NOFOLLOW) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= MAX_PROFILE_BYTES: + raise ContractError( + f"distributed profile must be a 1 through {MAX_PROFILE_BYTES} byte regular file" + ) + raw = os.read(descriptor, MAX_PROFILE_BYTES + 1) + final_metadata = os.fstat(descriptor) + file_identity = lambda item: ( + item.st_dev, + item.st_ino, + item.st_size, + item.st_mtime_ns, + item.st_mode, + ) + if len(raw) != metadata.st_size or file_identity(metadata) != file_identity( + final_metadata + ): + raise ContractError("distributed profile changed while it was read") + except ContractError: + raise + except OSError as exc: + raise ContractError("distributed profile must be a regular non-symlink file") from exc + finally: + if descriptor is not None: + os.close(descriptor) + try: + root = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_object) + except ContractError: + raise + except (UnicodeError, json.JSONDecodeError, RecursionError) as exc: + raise ContractError("distributed profile must be strict UTF-8 JSON") from exc + root = _mapping(root, "profile") + if any(key.lower() in _SECRET_KEYS for key in root): + raise ContractError("unknown profile field") + unknown = set(root) - _ROOT_FIELDS + missing = _ROOT_FIELDS - set(root) + if unknown: + raise ContractError("unknown profile field") + if missing: + raise ContractError("required profile section is missing") + if root["schema_version"] != 1: + raise ContractError("schema_version must be 1") + + sections = {name: _closed_section(root, name) for name in _SECTION_FIELDS} + run, identity = sections["run"], sections["identity"] + topology, network = sections["topology"], sections["network"] + source, staging = sections["source"], sections["staging"] + resources, timeouts = sections["resources"], sections["timeouts"] + software, execution, evidence = ( + sections["software"], + sections["execution"], + sections["evidence"], + ) + cidrs = network["allowed_master_cidrs"] + if not isinstance(cidrs, list) or not cidrs or len(cidrs) > 16: + raise ContractError("allowed_master_cidrs must be a non-empty bounded list") + normalized_cidrs: list[str] = [] + private_ranges = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("fc00::/7"), + ) + for value in cidrs: + try: + item = ipaddress.ip_network(_text(value, "allowed master CIDR"), strict=True) + except ValueError as exc: + raise ContractError("allowed_master_cidrs contains an invalid network") from exc + is_private = any( + ( + isinstance(item, ipaddress.IPv4Network) + and isinstance(private, ipaddress.IPv4Network) + and item.subnet_of(private) + ) + or ( + isinstance(item, ipaddress.IPv6Network) + and isinstance(private, ipaddress.IPv6Network) + and item.subnet_of(private) + ) + for private in private_ranges + ) + if not is_private: + raise ContractError("allowed_master_cidrs must contain only private networks") + normalized_cidrs.append(str(item)) + if len(set(normalized_cidrs)) != len(normalized_cidrs): + raise ContractError("allowed_master_cidrs cannot contain duplicates") + divisors = topology["dimension_divisors"] + if not isinstance(divisors, list) or not divisors or len(divisors) > 256: + raise ContractError("dimension_divisors must be a non-empty bounded list") + environment_keys = execution["allowed_environment_keys"] + if ( + not isinstance(environment_keys, list) + or len(environment_keys) > 128 + or len(set(environment_keys)) != len(environment_keys) + ): + raise ContractError("allowed_environment_keys must be a unique bounded list") + + result = cls( + run_id=_text(run["run_id"], "run_id", _RUN_ID_RE), + rendezvous_id=_text(run["rendezvous_id"], "rendezvous_id", _RUN_ID_RE), + world_size=_integer(run["world_size"], "world_size", 2, 4096), + local_world_size=_integer( + run["local_world_size"], "local_world_size", 1, 4096 + ), + source_digest=_text(identity["source_digest"], "source_digest", _DIGEST_RE), + model_digest=_text(identity["model_digest"], "model_digest", _DIGEST_RE), + tokenizer_digest=_text( + identity["tokenizer_digest"], "tokenizer_digest", _DIGEST_RE + ), + commit_sha=_text(identity["commit_sha"], "commit_sha", _COMMIT_RE), + code_digest=_text(identity["code_digest"], "code_digest", _DIGEST_RE), + tensor_parallel_size=_integer( + topology["tensor_parallel_size"], "tensor_parallel_size", 2, 4096 + ), + coordinator_rank=_integer( + topology["coordinator_rank"], "coordinator_rank", 0, 4095 + ), + placement_plan_digest=_text( + topology["placement_plan_digest"], "placement_plan_digest", _DIGEST_RE + ), + dimension_divisors=tuple( + _integer(item, "dimension divisor", 1, 2**31 - 1) for item in divisors + ), + master_addr=_text(network["master_addr"], "master_addr"), + master_port=_integer(network["master_port"], "master_port", 1, 65535), + network_interface=_text( + network["interface"], "network interface", _INTERFACE_RE + ), + allowed_master_cidrs=tuple(normalized_cidrs), + source_path=_absolute_path(source["path"], "source.path"), + staging_path=_absolute_path(staging["path"], "staging.path"), + storage_digest=_text(staging["storage_digest"], "storage_digest", _DIGEST_RE), + min_free_device_memory_bytes=_integer( + resources["min_free_device_memory_bytes"], + "min_free_device_memory_bytes", + 1, + 2**63 - 1, + ), + min_free_host_memory_bytes=_integer( + resources["min_free_host_memory_bytes"], + "min_free_host_memory_bytes", + 1, + 2**63 - 1, + ), + min_free_staging_bytes=_integer( + resources["min_free_staging_bytes"], + "min_free_staging_bytes", + 1, + 2**63 - 1, + ), + max_source_files=_integer( + resources["max_source_files"], "max_source_files", 1, 1_000_000 + ), + max_source_bytes=_integer( + resources["max_source_bytes"], "max_source_bytes", 1, 2**63 - 1 + ), + max_source_file_bytes=_integer( + resources["max_source_file_bytes"], + "max_source_file_bytes", + 1, + 2**63 - 1, + ), + source_timeout_seconds=_integer( + timeouts["source_seconds"], "source_seconds", 1, MAX_TIMEOUT_SECONDS + ), + init_timeout_seconds=_integer( + timeouts["init_seconds"], "init_seconds", 1, MAX_TIMEOUT_SECONDS + ), + collective_timeout_seconds=_integer( + timeouts["collective_seconds"], + "collective_seconds", + 1, + MAX_TIMEOUT_SECONDS, + ), + teardown_timeout_seconds=_integer( + timeouts["teardown_seconds"], + "teardown_seconds", + 1, + MAX_TIMEOUT_SECONDS, + ), + software_versions=tuple( + (key, _text(value, f"software.{key}")) for key, value in sorted(software.items()) + ), + device_kind=_text(execution["device_kind"], "device_kind"), + device_name=_text(execution["device_name"], "device_name"), + compute_capability=_text( + execution["compute_capability"], "compute_capability" + ), + evidence_tier=_text(execution["evidence_tier"], "evidence_tier"), + allowed_environment_keys=tuple( + sorted( + _text(item, "allowed environment key", _ENV_NAME_RE) + for item in environment_keys + ) + ), + local_files_only=execution["local_files_only"], + trust_remote_code=execution["trust_remote_code"], + allow_runtime_install=execution["allow_runtime_install"], + allow_plugins=execution["allow_plugins"], + allow_compilation=execution["allow_compilation"], + allow_adapters=execution["allow_adapters"], + allow_quantization=execution["allow_quantization"], + evidence_path=_absolute_path(evidence["path"], "evidence.path"), + digest=contract_digest(root), + ) + return result.validate() + + def validate(self) -> "DistributedPreflightConfig": + """Recheck invariants after direct construction or dataclass replacement.""" + if self.run_id == self.rendezvous_id: + raise ContractError("run_id and rendezvous_id must be distinct") + if self.world_size % self.local_world_size: + raise ContractError("world_size must be divisible by local_world_size") + if self.tensor_parallel_size != self.world_size: + raise ContractError("version 1 tensor_parallel_size must equal world_size") + if self.max_source_file_bytes > self.max_source_bytes: + raise ContractError("max_source_file_bytes cannot exceed max_source_bytes") + expected_evidence = self.staging_path / self.run_id / "preflight.json" + if self.evidence_path != expected_evidence: + raise ContractError( + "evidence.path must be the run-scoped staging preflight record" + ) + if not 0 <= self.coordinator_rank < self.world_size: + raise ContractError("coordinator_rank must be smaller than world_size") + if self.device_kind not in {"cuda", "cpu"}: + raise ContractError("device_kind must be cuda or cpu") + tiers = {"cpu": "protocol_cpu", "cuda": "candidate_preflight"} + if self.evidence_tier != tiers[self.device_kind]: + raise ContractError("evidence_tier does not match the configured device kind") + policies = { + "local_files_only": self.local_files_only, + "trust_remote_code": self.trust_remote_code, + "allow_runtime_install": self.allow_runtime_install, + "allow_plugins": self.allow_plugins, + "allow_compilation": self.allow_compilation, + "allow_adapters": self.allow_adapters, + "allow_quantization": self.allow_quantization, + } + if policies["local_files_only"] is not True: + raise ContractError("local_files_only must remain true") + for name in ( + "trust_remote_code", + "allow_runtime_install", + "allow_plugins", + "allow_compilation", + "allow_adapters", + "allow_quantization", + ): + if policies[name] is not False: + raise ContractError(f"{name} must remain false") + sensitive_markers = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "API_KEY", + "ACCESS_KEY", + "PRIVATE_KEY", + "CREDENTIAL", + ) + sensitive_prefixes = ("AWS_", "AZURE_", "GOOGLE_", "HF_", "OPENAI_", "ANTHROPIC_") + proxy_keys = {"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY"} + if any( + name in proxy_keys + or name.startswith(sensitive_prefixes) + or any(marker in name for marker in sensitive_markers) + for name in self.allowed_environment_keys + ): + raise ContractError("allowed_environment_keys cannot authorize secret or proxy fields") + return self + + @property + def software_digest(self) -> str: + return contract_digest(dict(self.software_versions)) diff --git a/obliteratus/distributed/consensus.py b/obliteratus/distributed/consensus.py new file mode 100644 index 0000000..354682d --- /dev/null +++ b/obliteratus/distributed/consensus.py @@ -0,0 +1,137 @@ +"""Bounded Gloo consensus primitives for the CPU protocol test lane.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch +import torch.distributed as dist + +from obliteratus.distributed.contracts import ( + MAX_CONSENSUS_BYTES, + MAX_WORLD_SIZE, + ContractError, + canonical_record, + contract_digest, +) + + +_LENGTH_BYTES = 4 + + +def encode_frame(payload: bytes, *, capacity: int = MAX_CONSENSUS_BYTES) -> torch.Tensor: + """Encode bytes into one fixed-size uint8 frame with zero padding.""" + if isinstance(capacity, bool) or not isinstance(capacity, int) or not 1 <= capacity <= MAX_CONSENSUS_BYTES: + raise ContractError(f"capacity must be between 1 and {MAX_CONSENSUS_BYTES}") + if not isinstance(payload, bytes): + raise ContractError("frame payload must be bytes") + if len(payload) > capacity: + raise ContractError(f"frame payload exceeds {capacity} bytes") + frame = bytearray(_LENGTH_BYTES + capacity) + frame[:_LENGTH_BYTES] = len(payload).to_bytes(_LENGTH_BYTES, byteorder="big") + frame[_LENGTH_BYTES : _LENGTH_BYTES + len(payload)] = payload + return torch.frombuffer(frame, dtype=torch.uint8).clone() + + +def decode_frame(frame: torch.Tensor) -> bytes: + """Decode a canonical fixed-size frame and reject non-zero padding.""" + if not isinstance(frame, torch.Tensor) or frame.dtype != torch.uint8 or frame.ndim != 1: + raise ContractError("frame must be a one-dimensional uint8 tensor") + capacity = frame.numel() - _LENGTH_BYTES + if not 1 <= capacity <= MAX_CONSENSUS_BYTES: + raise ContractError("frame has an invalid capacity") + raw = bytes(frame.detach().cpu().tolist()) + size = int.from_bytes(raw[:_LENGTH_BYTES], byteorder="big") + if size > capacity: + raise ContractError("frame length exceeds its capacity") + if any(raw[_LENGTH_BYTES + size :]): + raise ContractError("frame padding must be zero") + return raw[_LENGTH_BYTES : _LENGTH_BYTES + size] + + +def _require_gloo(group: dist.ProcessGroup | None) -> tuple[int, int]: + if not dist.is_available() or not dist.is_initialized(): + raise ContractError("a Gloo process group must be initialized") + backend = str(dist.get_backend(group)).lower() + if backend != "gloo": + raise ContractError("CPU protocol consensus requires the Gloo backend") + rank, world_size = dist.get_rank(group), dist.get_world_size(group) + if not 2 <= world_size <= MAX_WORLD_SIZE: + raise ContractError(f"Gloo world_size must be between 2 and {MAX_WORLD_SIZE}") + return rank, world_size + + +def gloo_all_gather_records( + record: object, + *, + group: dist.ProcessGroup | None = None, + capacity: int = MAX_CONSENSUS_BYTES, +) -> tuple[bytes, ...]: + """Gather bounded canonical records without Python object collectives.""" + payload = canonical_record(record, max_bytes=capacity) + _rank, world_size = _require_gloo(group) + frame = encode_frame(payload, capacity=capacity) + gathered = [torch.zeros_like(frame) for _ in range(world_size)] + dist.all_gather(gathered, frame, group=group) + return tuple(decode_frame(item) for item in gathered) + + +def require_consensus_digest( + digest: str, + *, + group: dist.ProcessGroup | None = None, +) -> str: + """Require every rank to supply the same lowercase SHA-256 digest.""" + if not isinstance(digest, str) or len(digest) != 64: + raise ContractError("digest must be 64 lowercase hexadecimal characters") + try: + payload = bytes.fromhex(digest) + except ValueError as exc: + raise ContractError("digest must be 64 lowercase hexadecimal characters") from exc + if digest != digest.lower() or payload.hex() != digest: + raise ContractError("digest must be 64 lowercase hexadecimal characters") + _rank, world_size = _require_gloo(group) + local = torch.tensor(tuple(payload), dtype=torch.uint8) + gathered = [torch.zeros_like(local) for _ in range(world_size)] + dist.all_gather(gathered, local, group=group) + if any(not torch.equal(local, item) for item in gathered): + raise ContractError("rank digests disagree") + return digest + + +def require_record_consensus( + record: object, + *, + group: dist.ProcessGroup | None = None, +) -> str: + """Require every rank to present the same bounded canonical record.""" + return require_consensus_digest(contract_digest(record), group=group) + + +def unanimous_vote( + sequence: int, + accepted: bool, + *, + group: dist.ProcessGroup | None = None, +) -> bool: + """Return true only when every rank votes yes for the same sequence.""" + if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 0: + raise ContractError("vote sequence must be a non-negative integer") + if not isinstance(accepted, bool): + raise ContractError("accepted must be a boolean") + _rank, world_size = _require_gloo(group) + local = torch.tensor((sequence, int(accepted)), dtype=torch.int64) + gathered = [torch.zeros_like(local) for _ in range(world_size)] + dist.all_gather(gathered, local, group=group) + if any(int(item[0]) != sequence for item in gathered): + raise ContractError("rank vote sequences disagree") + return all(bool(int(item[1])) for item in gathered) + + +def assert_rank_order(records: Sequence[object], *, world_size: int) -> None: + """Reject missing, duplicate, or reordered rank-bearing records.""" + if len(records) != world_size: + raise ContractError("record count does not match world_size") + ranks = [getattr(record, "rank", None) for record in records] + if ranks != list(range(world_size)): + raise ContractError("records must appear once in global-rank order") diff --git a/obliteratus/distributed/contracts.py b/obliteratus/distributed/contracts.py new file mode 100644 index 0000000..6baee86 --- /dev/null +++ b/obliteratus/distributed/contracts.py @@ -0,0 +1,481 @@ +"""Pure, bounded records for distributed runtime identity and lifecycle.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass, fields, is_dataclass +from enum import Enum +from typing import Any + + +MAX_CONSENSUS_BYTES = 64 * 1024 +MAX_TEXT_BYTES = 256 +MAX_WORLD_SIZE = 4096 +MAX_RECORD_DEPTH = 16 +MAX_RECORD_ITEMS = 4096 +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_RUN_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_ERROR_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,63}$") + + +class ContractError(ValueError): + """A distributed record violates a bounded, fail-closed contract.""" + + +class RuntimeContractError(ContractError): + """A sanitized runtime refusal with one stable degraded-mode code.""" + + def __init__(self, code: str, message: str) -> None: + _require_text(code, "error_code", pattern=_ERROR_RE) + super().__init__(message) + self.code = code + + +class RuntimeStage(str, Enum): + CREATED = "created" + PREFLIGHTED = "preflighted" + LOADED = "loaded" + PROBED = "probed" + DISTILLED = "distilled" + PREPARED = "prepared" + MUTATING = "mutating" + VERIFIED = "verified" + STAGED = "staged" + PUBLISHED = "published" + ABORTING = "aborting" + ABORTED = "aborted" + QUARANTINED = "quarantined" + + +class Vote(str, Enum): + PREPARED = "prepared" + COMMITTED = "committed" + ABORT = "abort" + + +class PlacementKind(str, Enum): + COLUMN_WISE = "column_wise" + ROW_WISE = "row_wise" + REPLICATED = "replicated" + + +_SUPPORTED_DTYPES = frozenset({"float16", "bfloat16", "float32", "float64"}) + + +_ALLOWED_TRANSITIONS: dict[RuntimeStage, frozenset[RuntimeStage]] = { + RuntimeStage.CREATED: frozenset({RuntimeStage.PREFLIGHTED, RuntimeStage.ABORTING}), + RuntimeStage.PREFLIGHTED: frozenset({RuntimeStage.LOADED, RuntimeStage.ABORTING}), + RuntimeStage.LOADED: frozenset({RuntimeStage.PROBED, RuntimeStage.ABORTING}), + RuntimeStage.PROBED: frozenset({RuntimeStage.DISTILLED, RuntimeStage.ABORTING}), + RuntimeStage.DISTILLED: frozenset({RuntimeStage.PREPARED, RuntimeStage.ABORTING}), + RuntimeStage.PREPARED: frozenset({RuntimeStage.MUTATING, RuntimeStage.ABORTING}), + RuntimeStage.MUTATING: frozenset({RuntimeStage.VERIFIED, RuntimeStage.ABORTING}), + RuntimeStage.VERIFIED: frozenset({RuntimeStage.STAGED, RuntimeStage.ABORTING}), + RuntimeStage.STAGED: frozenset({RuntimeStage.PUBLISHED, RuntimeStage.ABORTING}), + RuntimeStage.ABORTING: frozenset({RuntimeStage.ABORTED, RuntimeStage.QUARANTINED}), + RuntimeStage.PUBLISHED: frozenset(), + RuntimeStage.ABORTED: frozenset(), + RuntimeStage.QUARANTINED: frozenset(), +} + + +def _require_int(value: object, name: str, *, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ContractError(f"{name} must be an integer") + if not minimum <= value <= maximum: + raise ContractError(f"{name} must be between {minimum} and {maximum}") + return value + + +def _require_text(value: object, name: str, *, pattern: re.Pattern[str] | None = None) -> str: + if not isinstance(value, str) or not value: + raise ContractError(f"{name} must be non-empty text") + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeError as exc: + raise ContractError(f"{name} must be valid UTF-8") from exc + if len(encoded) > MAX_TEXT_BYTES: + raise ContractError(f"{name} exceeds {MAX_TEXT_BYTES} UTF-8 bytes") + if pattern is not None and pattern.fullmatch(value) is None: + raise ContractError(f"{name} has an invalid format") + return value + + +def _require_digest(value: object, name: str) -> str: + return _require_text(value, name, pattern=_DIGEST_RE) + + +@dataclass(frozen=True) +class RunIdentity: + """Immutable identity shared by every rank in one non-resumable attempt.""" + + run_id: str + config_digest: str + source_digest: str + model_digest: str + tokenizer_digest: str + commit_sha: str + world_size: int + + def __post_init__(self) -> None: + _require_text(self.run_id, "run_id", pattern=_RUN_ID_RE) + _require_digest(self.config_digest, "config_digest") + _require_digest(self.source_digest, "source_digest") + _require_digest(self.model_digest, "model_digest") + _require_digest(self.tokenizer_digest, "tokenizer_digest") + _require_text(self.commit_sha, "commit_sha", pattern=_COMMIT_RE) + _require_int(self.world_size, "world_size", minimum=2, maximum=MAX_WORLD_SIZE) + + +@dataclass(frozen=True) +class RankInventory: + """Redaction-safe rank, host, device, software, and storage identity.""" + + rank: int + local_rank: int + world_size: int + host_digest: str + device_digest: str + device_kind: str + total_memory_bytes: int + free_memory_bytes: int + software_digest: str + storage_digest: str + + def __post_init__(self) -> None: + _require_int(self.world_size, "world_size", minimum=2, maximum=MAX_WORLD_SIZE) + _require_int(self.rank, "rank", minimum=0, maximum=self.world_size - 1) + _require_int(self.local_rank, "local_rank", minimum=0, maximum=self.world_size - 1) + _require_digest(self.host_digest, "host_digest") + _require_digest(self.device_digest, "device_digest") + if self.device_kind not in {"cpu", "cuda"}: + raise ContractError("device_kind must be 'cpu' or 'cuda'") + total = _require_int( + self.total_memory_bytes, + "total_memory_bytes", + minimum=1, + maximum=2**63 - 1, + ) + free = _require_int( + self.free_memory_bytes, + "free_memory_bytes", + minimum=0, + maximum=2**63 - 1, + ) + if free > total: + raise ContractError("free_memory_bytes cannot exceed total_memory_bytes") + _require_digest(self.software_digest, "software_digest") + _require_digest(self.storage_digest, "storage_digest") + + +@dataclass(frozen=True) +class TopologyPlan: + """Fixed process-group topology accepted before model allocation.""" + + world_size: int + coordinator_rank: int + backend: str + placement_plan_digest: str + + def __post_init__(self) -> None: + _require_int(self.world_size, "world_size", minimum=2, maximum=MAX_WORLD_SIZE) + _require_int( + self.coordinator_rank, + "coordinator_rank", + minimum=0, + maximum=self.world_size - 1, + ) + if self.backend not in {"gloo", "nccl"}: + raise ContractError("backend must be 'gloo' or 'nccl'") + _require_digest(self.placement_plan_digest, "placement_plan_digest") + + +@dataclass(frozen=True) +class LogicalPlacement: + """One exact, even 2-D parameter shard placement for the prototype.""" + + logical_name: str + global_shape: tuple[int, int] + dtype: str + kind: PlacementKind + rank: int + world_size: int + direction_axis: int + shard_dim: int | None = None + shard_start: int = 0 + shard_end: int = 0 + + def __post_init__(self) -> None: + _require_text(self.logical_name, "logical_name", pattern=_NAME_RE) + if not isinstance(self.global_shape, tuple) or len(self.global_shape) != 2: + raise ContractError("global_shape must be a positive two-dimensional integer tuple") + for size in self.global_shape: + if isinstance(size, bool) or not isinstance(size, int) or not 1 <= size <= 2**63 - 1: + raise ContractError("global_shape must be a positive two-dimensional integer tuple") + _require_text(self.dtype, "dtype", pattern=_NAME_RE) + if self.dtype not in _SUPPORTED_DTYPES: + raise ContractError("dtype is not supported by the distributed semantic oracle") + if not isinstance(self.kind, PlacementKind): + raise ContractError("kind must be a PlacementKind") + _require_int(self.world_size, "world_size", minimum=2, maximum=MAX_WORLD_SIZE) + _require_int(self.rank, "rank", minimum=0, maximum=self.world_size - 1) + _require_int(self.direction_axis, "direction_axis", minimum=0, maximum=1) + + if self.kind is PlacementKind.REPLICATED: + if self.shard_dim is not None or self.shard_start != 0 or self.shard_end != 0: + raise ContractError("replicated placement cannot declare a shard interval") + return + + expected_dim = 0 if self.kind is PlacementKind.COLUMN_WISE else 1 + if self.shard_dim != expected_dim: + raise ContractError(f"{self.kind.value} placement requires shard_dim={expected_dim}") + extent = self.global_shape[expected_dim] + if extent % self.world_size != 0: + raise ContractError("prototype placements require equal shard extents") + shard_size = extent // self.world_size + expected_start = self.rank * shard_size + expected_end = expected_start + shard_size + if (self.shard_start, self.shard_end) != (expected_start, expected_end): + raise ContractError("shard interval does not match the fixed rank partition") + + @property + def local_shape(self) -> tuple[int, int]: + if self.kind is PlacementKind.REPLICATED: + return self.global_shape + result = list(self.global_shape) + assert self.shard_dim is not None + result[self.shard_dim] = self.shard_end - self.shard_start + return result[0], result[1] + + +@dataclass(frozen=True) +class StageMessage: + """Sequenced lifecycle evidence emitted by one rank.""" + + run_id: str + identity_digest: str + rank: int + sequence: int + stage: RuntimeStage + vote: Vote | None = None + error_code: str | None = None + + def __post_init__(self) -> None: + _require_text(self.run_id, "run_id", pattern=_RUN_ID_RE) + _require_digest(self.identity_digest, "identity_digest") + _require_int(self.rank, "rank", minimum=0, maximum=MAX_WORLD_SIZE - 1) + _require_int(self.sequence, "sequence", minimum=0, maximum=2**63 - 1) + if not isinstance(self.stage, RuntimeStage): + raise ContractError("stage must be a RuntimeStage") + if self.vote is not None and not isinstance(self.vote, Vote): + raise ContractError("vote must be a Vote") + if self.error_code is not None: + _require_text(self.error_code, "error_code", pattern=_ERROR_RE) + if self.vote is Vote.ABORT and self.error_code is None: + raise ContractError("an abort vote requires an error_code") + if self.error_code is not None and self.vote is not Vote.ABORT: + raise ContractError("error_code is valid only with an abort vote") + + def to_bytes(self) -> bytes: + """Encode one lifecycle record in its sole canonical representation.""" + + return canonical_record(self) + + @classmethod + def from_bytes(cls, payload: bytes) -> "StageMessage": + """Decode a closed lifecycle record and reject alternate JSON.""" + + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ContractError("lifecycle message contains a duplicate field") + result[key] = value + return result + + try: + value = json.loads(payload.decode("utf-8"), object_pairs_hook=unique_object) + if not isinstance(value, dict) or set(value) != set(cls.__dataclass_fields__): + raise ContractError("lifecycle message fields do not match the closed schema") + converted = dict(value) + converted["stage"] = RuntimeStage(converted["stage"]) + if converted["vote"] is not None: + converted["vote"] = Vote(converted["vote"]) + result = cls(**converted) + except ContractError: + raise + except (UnicodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise ContractError("lifecycle message must be canonical UTF-8 JSON") from exc + if payload != result.to_bytes(): + raise ContractError("lifecycle message is not canonical") + return result + + +def advance_stage(current: RuntimeStage, requested: RuntimeStage) -> RuntimeStage: + """Validate one lifecycle transition without performing side effects.""" + if not isinstance(current, RuntimeStage) or not isinstance(requested, RuntimeStage): + raise ContractError("current and requested stages must be RuntimeStage values") + if requested not in _ALLOWED_TRANSITIONS[current]: + raise ContractError( + f"invalid distributed stage transition: {current.value} -> {requested.value}" + ) + return requested + + +def _json_value( + value: Any, + *, + depth: int, + active: set[int], + item_count: list[int], + max_bytes: int, +) -> Any: + if depth > MAX_RECORD_DEPTH: + raise ContractError(f"canonical record exceeds nesting depth {MAX_RECORD_DEPTH}") + item_count[0] += 1 + if item_count[0] > MAX_RECORD_ITEMS: + raise ContractError(f"canonical record exceeds {MAX_RECORD_ITEMS} items") + if isinstance(value, Enum): + return _json_value( + value.value, + depth=depth + 1, + active=active, + item_count=item_count, + max_bytes=max_bytes, + ) + if is_dataclass(value) and not isinstance(value, type): + identity = id(value) + if identity in active: + raise ContractError("canonical record contains a reference cycle") + active.add(identity) + try: + return { + field.name: _json_value( + getattr(value, field.name), + depth=depth + 1, + active=active, + item_count=item_count, + max_bytes=max_bytes, + ) + for field in fields(value) + } + finally: + active.remove(identity) + if isinstance(value, dict): + identity = id(value) + if identity in active: + raise ContractError("canonical record contains a reference cycle") + active.add(identity) + converted: dict[str, Any] = {} + try: + for key, item in value.items(): + if not isinstance(key, str): + raise ContractError("canonical record keys must be strings") + if len(key.encode("utf-8", errors="strict")) > max_bytes: + raise ContractError(f"canonical record exceeds {max_bytes} bytes") + converted[key] = _json_value( + item, + depth=depth + 1, + active=active, + item_count=item_count, + max_bytes=max_bytes, + ) + return converted + except UnicodeError as exc: + raise ContractError("canonical record key must be valid UTF-8") from exc + finally: + active.remove(identity) + if isinstance(value, (list, tuple)): + identity = id(value) + if identity in active: + raise ContractError("canonical record contains a reference cycle") + active.add(identity) + try: + return [ + _json_value( + item, + depth=depth + 1, + active=active, + item_count=item_count, + max_bytes=max_bytes, + ) + for item in value + ] + finally: + active.remove(identity) + if isinstance(value, str): + try: + if len(value.encode("utf-8", errors="strict")) > max_bytes: + raise ContractError(f"canonical record exceeds {max_bytes} bytes") + except UnicodeError as exc: + raise ContractError("canonical record string must be valid UTF-8") from exc + return value + if value is None or isinstance(value, bool): + return value + if isinstance(value, int): + if not -(2**63) <= value <= 2**63 - 1: + raise ContractError("canonical record integer exceeds the signed 64-bit bound") + return value + raise ContractError(f"canonical record contains unsupported type {type(value).__name__}") + + +def canonical_record(value: Any, *, max_bytes: int = MAX_CONSENSUS_BYTES) -> bytes: + """Serialize a record deterministically, rejecting floats and oversized data.""" + _require_int(max_bytes, "max_bytes", minimum=1, maximum=MAX_CONSENSUS_BYTES) + try: + encoded = json.dumps( + _json_value( + value, + depth=0, + active=set(), + item_count=[0], + max_bytes=max_bytes, + ), + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8", errors="strict") + except ContractError: + raise + except (TypeError, ValueError, UnicodeError) as exc: + raise ContractError("record is not canonical JSON") from exc + if len(encoded) > max_bytes: + raise ContractError(f"canonical record exceeds {max_bytes} bytes") + return encoded + + +def contract_digest(value: Any) -> str: + """Return the SHA-256 identity of one canonical bounded record.""" + return hashlib.sha256(canonical_record(value)).hexdigest() + + +def validate_inventory_consensus( + identity: RunIdentity, + inventories: tuple[RankInventory, ...], +) -> None: + """Validate a complete homogeneous fixed-world inventory.""" + if not isinstance(identity, RunIdentity): + raise ContractError("identity must be a RunIdentity") + if not isinstance(inventories, tuple) or len(inventories) != identity.world_size: + raise ContractError("inventory must contain exactly one record per rank") + if any(not isinstance(item, RankInventory) for item in inventories): + raise ContractError("inventory contains an invalid rank record") + if {item.rank for item in inventories} != set(range(identity.world_size)): + raise ContractError("inventory ranks do not exactly cover the fixed world") + if any(item.world_size != identity.world_size for item in inventories): + raise ContractError("inventory world_size disagrees with the run identity") + devices = {(item.host_digest, item.device_digest) for item in inventories} + if len(devices) != identity.world_size: + raise ContractError("each rank must own a unique host/device pair") + host_local_ranks = {(item.host_digest, item.local_rank) for item in inventories} + if len(host_local_ranks) != identity.world_size: + raise ContractError("local ranks must be unique within each host") + if len({item.software_digest for item in inventories}) != 1: + raise ContractError("rank software identities disagree") + if len({item.storage_digest for item in inventories}) != 1: + raise ContractError("rank storage identities disagree") diff --git a/obliteratus/distributed/evidence.py b/obliteratus/distributed/evidence.py new file mode 100644 index 0000000..fcaba22 --- /dev/null +++ b/obliteratus/distributed/evidence.py @@ -0,0 +1,299 @@ +"""Allowlist-built evidence for distributed preflight attempts.""" + +from __future__ import annotations + +import json +import os +import re +import secrets +import stat +from dataclasses import asdict, dataclass +from pathlib import Path + +from obliteratus.distributed.contracts import ( + MAX_CONSENSUS_BYTES, + ContractError, + StageMessage, +) + + +MAX_EVIDENCE_BYTES = 64 * 1024 +_RUN_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_ERROR_CODES = frozenset( + { + "LMS_ATOMIC_PROMOTION_UNAVAILABLE", + "LMS_ATTEMPT_CANCELLED", + "LMS_ATTEMPT_IDENTITY_CONFLICT", + "LMS_CLEANUP_INCOMPLETE", + "LMS_CLOCK_PROFILE_MISMATCH", + "LMS_COLLECTIVE_FAILED", + "LMS_CONSENSUS_RECORD_INVALID", + "LMS_DIAGNOSTIC_REDACTION_FAILED", + "LMS_DISTRIBUTED_INTENT_REQUIRED", + "LMS_ELASTICITY_FORBIDDEN", + "LMS_EVIDENCE_SCOPE_MISMATCH", + "LMS_EVIDENCE_UNAVAILABLE", + "LMS_FORBIDDEN_RUNTIME_CAPABILITY", + "LMS_IDENTITY_MISMATCH", + "LMS_LAUNCH_IDENTITY_INVALID", + "LMS_LIFECYCLE_INVALID", + "LMS_MEMBERSHIP_INVALID", + "LMS_MEMBERSHIP_TIMEOUT", + "LMS_NETWORK_PROFILE_DENIED", + "LMS_RANK_DEVICE_CONFLICT", + "LMS_RESOURCE_ADMISSION_DENIED", + "LMS_RUNTIME_PROFILE_MISMATCH", + "LMS_SECRET_INPUT_REJECTED", + "LMS_SECURITY_BASELINE_REVOKED", + "LMS_SOURCE_BOUNDARY_VIOLATION", + "LMS_SOURCE_CHANGED", + "LMS_STAGE_TIMEOUT", + "LMS_STORAGE_PROFILE_MISMATCH", + "LMS_TOPOLOGY_UNSUPPORTED", + "LMS_TRANSPORT_BOUNDARY_UNSATISFIED", + } +) + + +@dataclass(frozen=True) +class PreflightEvidence: + """A bounded public result containing no raw operational values.""" + + schema_version: int + result: str + run_id: str + config_digest: str + world_size: int + accepted_ranks: int + evidence_tier: str + identity_digest: str | None + error_code: str | None + + def __post_init__(self) -> None: + if self.schema_version != 1: + raise ContractError("evidence schema_version must be 1") + if self.result not in {"preflighted", "failed", "quarantined"}: + raise ContractError("evidence result is invalid") + if _RUN_ID_RE.fullmatch(self.run_id) is None: + raise ContractError("evidence run_id is invalid") + if _DIGEST_RE.fullmatch(self.config_digest) is None: + raise ContractError("evidence config_digest is invalid") + if isinstance(self.world_size, bool) or not 2 <= self.world_size <= 4096: + raise ContractError("evidence world_size is invalid") + if isinstance(self.accepted_ranks, bool) or not 0 <= self.accepted_ranks <= self.world_size: + raise ContractError("evidence accepted_ranks is invalid") + if self.evidence_tier not in {"protocol_cpu", "candidate_preflight"}: + raise ContractError("evidence tier is invalid") + if self.identity_digest is not None and _DIGEST_RE.fullmatch(self.identity_digest) is None: + raise ContractError("evidence identity_digest is invalid") + if self.error_code is not None and self.error_code not in _ERROR_CODES: + raise ContractError("evidence error_code is invalid") + if self.result == "preflighted": + if self.accepted_ranks != self.world_size or self.identity_digest is None: + raise ContractError("successful evidence requires the complete accepted world") + if self.error_code is not None: + raise ContractError("successful evidence cannot contain an error") + elif self.error_code is None: + raise ContractError("unsuccessful evidence requires a stable error code") + + @classmethod + def success( + cls, + *, + run_id: str, + config_digest: str, + world_size: int, + evidence_tier: str, + identity_digest: str, + ) -> "PreflightEvidence": + return cls( + 1, + "preflighted", + run_id, + config_digest, + world_size, + world_size, + evidence_tier, + identity_digest, + None, + ) + + @classmethod + def failure( + cls, + *, + run_id: str, + config_digest: str, + code: str, + world_size: int, + evidence_tier: str, + detail: str | None = None, + ) -> "PreflightEvidence": + del detail # raw exception and operator context never enter retained evidence + result = "quarantined" if code == "LMS_CLEANUP_INCOMPLETE" else "failed" + return cls(1, result, run_id, config_digest, world_size, 0, evidence_tier, None, code) + + def to_bytes(self) -> bytes: + encoded = (json.dumps(asdict(self), sort_keys=True, separators=(",", ":")) + "\n").encode() + if len(encoded) > MAX_EVIDENCE_BYTES: + raise ContractError("preflight evidence exceeds its byte bound") + return encoded + + @classmethod + def from_bytes(cls, payload: bytes) -> "PreflightEvidence": + """Decode only the one canonical, closed evidence representation.""" + + if not 1 <= len(payload) <= MAX_EVIDENCE_BYTES or not payload.endswith(b"\n"): + raise ContractError("preflight evidence has an invalid byte envelope") + + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ContractError("preflight evidence contains a duplicate field") + result[key] = value + return result + + try: + value = json.loads(payload[:-1].decode("utf-8"), object_pairs_hook=unique_object) + except ContractError: + raise + except (UnicodeError, json.JSONDecodeError) as exc: + raise ContractError("preflight evidence is not strict UTF-8 JSON") from exc + if not isinstance(value, dict) or set(value) != set(cls.__dataclass_fields__): + raise ContractError("preflight evidence fields do not match the closed schema") + result = cls(**value) + if payload != result.to_bytes(): + raise ContractError("preflight evidence is not canonical") + return result + + +def _open_private_parent(target: Path) -> int: + """Open an absolute parent through descriptor-relative, no-symlink traversal.""" + + if not target.is_absolute() or target.name in {"", ".", ".."}: + raise ContractError("evidence target must be an absolute file path") + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + descriptor = os.open("/", flags) + try: + for component in target.parent.parts[1:]: + next_descriptor = os.open(component, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = next_descriptor + metadata = os.fstat(descriptor) + effective_uid = getattr(os, "geteuid", lambda: metadata.st_uid)() + if metadata.st_uid != effective_uid or metadata.st_mode & 0o077: + raise ContractError("evidence parent must be private and owned by the worker") + return descriptor + except Exception: + os.close(descriptor) + raise + + +def _write_private_payload(path: str | Path, payload: bytes) -> None: + """Atomically create one bounded private record without following links.""" + + target = Path(path) + temporary = f".{target.name}.tmp.{os.getpid()}.{secrets.token_hex(8)}" + parent_descriptor: int | None = None + descriptor: int | None = None + try: + parent_descriptor = _open_private_parent(target) + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=parent_descriptor, + ) + written = 0 + while written < len(payload): + count = os.write(descriptor, payload[written:]) + if count <= 0: + raise ContractError("evidence write did not make progress") + written += count + os.fsync(descriptor) + os.close(descriptor) + descriptor = None + os.link( + temporary, + target.name, + src_dir_fd=parent_descriptor, + dst_dir_fd=parent_descriptor, + follow_symlinks=False, + ) + os.unlink(temporary, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except FileExistsError as exc: + raise ContractError("evidence target already exists") from exc + except OSError as exc: + raise ContractError("evidence sink is unavailable") from exc + finally: + if descriptor is not None: + os.close(descriptor) + if parent_descriptor is not None: + try: + os.unlink(temporary, dir_fd=parent_descriptor) + except FileNotFoundError: + pass + os.close(parent_descriptor) + + +def _read_private_payload(path: str | Path, *, maximum: int) -> bytes: + """Read one stable bounded private record without pathname races.""" + + target = Path(path) + parent_descriptor: int | None = None + descriptor: int | None = None + try: + parent_descriptor = _open_private_parent(target) + descriptor = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=parent_descriptor) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or not 1 <= before.st_size <= maximum + or before.st_mode & 0o077 + ): + raise ContractError("private record file is not private and regular") + payload = os.read(descriptor, maximum + 1) + after = os.fstat(descriptor) + if len(payload) != before.st_size or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns): + raise ContractError("private record changed while it was read") + return payload + except Exception: + raise + finally: + if descriptor is not None: + os.close(descriptor) + if parent_descriptor is not None: + os.close(parent_descriptor) + + +def write_evidence(path: str | Path, evidence: PreflightEvidence) -> None: + """Atomically create private evidence without following links or overwriting.""" + + _write_private_payload(path, evidence.to_bytes()) + + +def read_evidence(path: str | Path) -> PreflightEvidence: + """Read and validate one private evidence record without pathname races.""" + + return PreflightEvidence.from_bytes(_read_private_payload(path, maximum=MAX_EVIDENCE_BYTES)) + + +def write_stage_message(path: str | Path, message: StageMessage) -> None: + """Persist one private lifecycle receipt using a non-evidence schema.""" + + _write_private_payload(path, message.to_bytes()) + + +def read_stage_message(path: str | Path) -> StageMessage: + """Read one canonical private lifecycle receipt.""" + + return StageMessage.from_bytes(_read_private_payload(path, maximum=MAX_CONSENSUS_BYTES)) diff --git a/obliteratus/distributed/launcher.py b/obliteratus/distributed/launcher.py new file mode 100644 index 0000000..7676d2d --- /dev/null +++ b/obliteratus/distributed/launcher.py @@ -0,0 +1,375 @@ +"""Consume fixed torchrun membership without launching or provisioning workers.""" + +from __future__ import annotations + +import ipaddress +import os +import socket +import threading +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import timedelta + +import psutil # type: ignore[import-untyped] +import torch.distributed as dist + +from obliteratus.distributed.config import DistributedPreflightConfig +from obliteratus.distributed.contracts import ( + ContractError, + RuntimeContractError, + contract_digest, +) + + +_REQUIRED_ENV = frozenset( + { + "RANK", + "LOCAL_RANK", + "WORLD_SIZE", + "LOCAL_WORLD_SIZE", + "GROUP_RANK", + "ROLE_RANK", + "ROLE_WORLD_SIZE", + "MASTER_ADDR", + "MASTER_PORT", + "TORCHELASTIC_RUN_ID", + "TORCHELASTIC_RESTART_COUNT", + "TORCHELASTIC_MAX_RESTARTS", + "OBLITERATUS_RUN_ID", + "GLOO_SOCKET_IFNAME", + "NCCL_SOCKET_IFNAME", + } +) +_SENSITIVE_ENV_PARTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "API_KEY", + "ACCESS_KEY", + "PRIVATE_KEY", + "CREDENTIAL", +) +_SENSITIVE_ENV_PREFIXES = ("AWS_", "AZURE_", "GOOGLE_", "HF_", "OPENAI_", "ANTHROPIC_") +_PROXY_ENV = frozenset({"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY"}) +_BACKEND_DIAGNOSTIC_LOCK = threading.RLock() +_CLEANUP_TIMEOUT_EXIT_CODE = 70 + + +def _sensitive_environment_key(name: str) -> bool: + upper = name.upper() + return ( + upper in _PROXY_ENV + or upper.startswith(_SENSITIVE_ENV_PREFIXES) + or any(part in upper for part in _SENSITIVE_ENV_PARTS) + ) + + +def _env_integer(environ: Mapping[str, str], name: str, minimum: int, maximum: int) -> int: + value = environ[name] + if not value or len(value) > 10 or not value.isascii() or not value.isdecimal(): + raise ContractError(f"{name.lower()} must be an unsigned decimal integer") + parsed = int(value) + if not minimum <= parsed <= maximum: + raise ContractError(f"{name.lower()} is outside its allowed range") + return parsed + + +def _private_allowlisted_address(address: str, cidrs: tuple[str, ...]) -> bool: + try: + candidate = ipaddress.ip_address(address) + except ValueError: + return False + private_ranges = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("fc00::/7"), + ) + if not any(candidate in item for item in private_ranges): + return False + return any(candidate in ipaddress.ip_network(item, strict=True) for item in cidrs) + + +def validate_network_interface( + interface: str, + allowed_cidrs: tuple[str, ...], + master_addr: str, + *, + coordinator: bool, +) -> None: + """Require an approved private address before opening a rendezvous socket.""" + + interfaces = psutil.net_if_addrs() + if interface not in interfaces: + raise RuntimeContractError( + "LMS_NETWORK_PROFILE_DENIED", "configured network interface is unavailable" + ) + networks = tuple(ipaddress.ip_network(item, strict=True) for item in allowed_cidrs) + addresses: set[ipaddress.IPv4Address | ipaddress.IPv6Address] = set() + for observed in interfaces[interface]: + if observed.family not in {socket.AF_INET, socket.AF_INET6}: + continue + try: + address = ipaddress.ip_address(observed.address.split("%", 1)[0]) + except ValueError: + continue + if any(address in network for network in networks if address.version == network.version): + addresses.add(address) + if not addresses: + raise RuntimeContractError( + "LMS_NETWORK_PROFILE_DENIED", + "configured network interface has no private allowlisted address", + ) + if coordinator and ipaddress.ip_address(master_addr) not in addresses: + raise RuntimeContractError( + "LMS_NETWORK_PROFILE_DENIED", + "coordinator endpoint is not bound to the configured interface", + ) + + +@contextmanager +def _backend_diagnostic_guard() -> Iterator[None]: + """Discard native fd-2 diagnostics and fail if the backend emits any bytes.""" + + with _BACKEND_DIAGNOSTIC_LOCK: + read_descriptor: int | None = None + write_descriptor: int | None = None + saved_stderr: int | None = None + reader: threading.Thread | None = None + observed = threading.Event() + reader_failed = threading.Event() + body_error: BaseException | None = None + try: + read_descriptor, write_descriptor = os.pipe() + saved_stderr = os.dup(2) + + def discard() -> None: + assert read_descriptor is not None + try: + while chunk := os.read(read_descriptor, 8192): + observed.set() + del chunk + except OSError: + reader_failed.set() + finally: + os.close(read_descriptor) + + reader = threading.Thread( + target=discard, + daemon=True, + name="obliteratus-backend-diagnostic-sink", + ) + reader.start() + os.dup2(write_descriptor, 2) + os.close(write_descriptor) + write_descriptor = None + try: + yield + except BaseException as error: + body_error = error + except OSError as error: + body_error = RuntimeContractError( + "LMS_DIAGNOSTIC_REDACTION_FAILED", + "backend diagnostic containment is unavailable", + ) + del error + finally: + if saved_stderr is not None: + try: + os.dup2(saved_stderr, 2) + except OSError: + reader_failed.set() + os.close(saved_stderr) + if write_descriptor is not None: + os.close(write_descriptor) + if reader is not None: + reader.join(1) + if reader.is_alive(): + reader_failed.set() + elif read_descriptor is not None: + os.close(read_descriptor) + if ( + isinstance(body_error, RuntimeContractError) + and body_error.code == "LMS_CLEANUP_INCOMPLETE" + ): + raise body_error.with_traceback(body_error.__traceback__) + if observed.is_set() or reader_failed.is_set(): + raise RuntimeContractError( + "LMS_DIAGNOSTIC_REDACTION_FAILED", + "backend diagnostic output was suppressed", + ) from None + if body_error is not None: + raise body_error.with_traceback(body_error.__traceback__) + + +@dataclass(frozen=True) +class TorchrunEnvironment: + """One validated worker identity supplied by an external fixed scheduler.""" + + rank: int + local_rank: int + world_size: int + local_world_size: int + group_rank: int + role_rank: int + role_world_size: int + master_endpoint_digest: str + run_id: str + rendezvous_id: str + network_interface: str + + @classmethod + def from_environ( + cls, + environ: Mapping[str, str], + config: DistributedPreflightConfig, + ) -> "TorchrunEnvironment": + if _REQUIRED_ENV - set(environ): + raise RuntimeContractError( + "LMS_LAUNCH_IDENTITY_INVALID", + "required torchrun environment is incomplete", + ) + if any(_sensitive_environment_key(name) for name in environ): + raise RuntimeContractError( + "LMS_SECRET_INPUT_REJECTED", + "secret-bearing or proxy environment fields are forbidden", + ) + allowed = _REQUIRED_ENV | set(config.allowed_environment_keys) + if set(environ) - allowed: + raise RuntimeContractError( + "LMS_FORBIDDEN_RUNTIME_CAPABILITY", + "environment contains fields outside the explicit allowlist", + ) + rank = _env_integer(environ, "RANK", 0, 4095) + local_rank = _env_integer(environ, "LOCAL_RANK", 0, 4095) + world_size = _env_integer(environ, "WORLD_SIZE", 2, 4096) + local_world_size = _env_integer(environ, "LOCAL_WORLD_SIZE", 1, 4096) + group_rank = _env_integer(environ, "GROUP_RANK", 0, 4095) + role_rank = _env_integer(environ, "ROLE_RANK", 0, 4095) + role_world_size = _env_integer(environ, "ROLE_WORLD_SIZE", 1, 4096) + master_port = _env_integer(environ, "MASTER_PORT", 1, 65535) + restart_count = _env_integer(environ, "TORCHELASTIC_RESTART_COUNT", 0, 4096) + max_restarts = _env_integer(environ, "TORCHELASTIC_MAX_RESTARTS", 0, 4096) + + if rank >= world_size: + raise ContractError("rank must be smaller than world_size") + if local_rank >= local_world_size: + raise ContractError("local_rank must be smaller than local_world_size") + if world_size != config.world_size: + raise ContractError("world_size disagrees with the distributed profile") + if local_world_size != config.local_world_size: + raise ContractError("local_world_size disagrees with the distributed profile") + expected_groups = world_size // local_world_size + if group_rank >= expected_groups: + raise ContractError("group_rank must be smaller than the fixed group count") + if rank != group_rank * local_world_size + local_rank: + raise ContractError("global, group, and local ranks disagree") + if role_rank != rank or role_world_size != world_size: + raise ContractError("role_world_size disagrees with fixed membership") + if restart_count != 0 or max_restarts != 0: + raise RuntimeContractError( + "LMS_ELASTICITY_FORBIDDEN", + "torchrun elasticity and restarts are forbidden", + ) + if environ["OBLITERATUS_RUN_ID"] != config.run_id: + raise ContractError("run_id disagrees with the distributed profile") + if environ["TORCHELASTIC_RUN_ID"] != config.rendezvous_id: + raise ContractError("rendezvous_id disagrees with the distributed profile") + if environ["MASTER_ADDR"] != config.master_addr or master_port != config.master_port: + raise ContractError("master endpoint disagrees with the distributed profile") + if not _private_allowlisted_address(config.master_addr, config.allowed_master_cidrs): + raise RuntimeContractError( + "LMS_NETWORK_PROFILE_DENIED", + "master endpoint must be a private allowlisted numeric address", + ) + if ( + environ["GLOO_SOCKET_IFNAME"] != config.network_interface + or environ["NCCL_SOCKET_IFNAME"] != config.network_interface + ): + raise ContractError("network interface disagrees with the distributed profile") + return cls( + rank=rank, + local_rank=local_rank, + world_size=world_size, + local_world_size=local_world_size, + group_rank=group_rank, + role_rank=role_rank, + role_world_size=role_world_size, + master_endpoint_digest=contract_digest( + {"address": config.master_addr, "port": config.master_port} + ), + run_id=config.run_id, + rendezvous_id=config.rendezvous_id, + network_interface=config.network_interface, + ) + + +@contextmanager +def control_group( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, +) -> Iterator[None]: + """Own one bounded Gloo control group and always tear it down.""" + if not dist.is_available() or dist.is_initialized(): + raise ContractError("distributed control group is unavailable or already initialized") + validate_network_interface( + config.network_interface, + config.allowed_master_cidrs, + config.master_addr, + coordinator=launch.rank == config.coordinator_rank, + ) + + def bounded_destroy() -> None: + if not dist.is_initialized(): + return + errors: list[BaseException] = [] + + def destroy() -> None: + try: + dist.destroy_process_group() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=destroy, daemon=True, name="obliteratus-gloo-teardown") + thread.start() + thread.join(config.teardown_timeout_seconds) + if thread.is_alive(): + # A Python thread cannot be killed safely. Exiting this externally + # launched worker while fd 2 is still guarded is the only bounded + # way to prevent late native diagnostics or continued group use. + os._exit(_CLEANUP_TIMEOUT_EXIT_CODE) + if errors: + raise RuntimeContractError( + "LMS_CLEANUP_INCOMPLETE", "control group teardown failed" + ) from None + + with _backend_diagnostic_guard(): + try: + dist.init_process_group( + backend="gloo", + init_method="env://", + rank=launch.rank, + world_size=launch.world_size, + timeout=timedelta( + seconds=min( + config.init_timeout_seconds, + config.collective_timeout_seconds, + ) + ), + ) + except BaseException as error: + bounded_destroy() + if isinstance(error, TimeoutError) or type(error).__name__ == "DistStoreError": + raise RuntimeContractError( + "LMS_MEMBERSHIP_TIMEOUT", + "fixed membership did not rendezvous within its bound", + ) from None + raise RuntimeContractError( + "LMS_COLLECTIVE_FAILED", "control group initialization failed" + ) from None + try: + yield + finally: + bounded_destroy() diff --git a/obliteratus/distributed/numerical.py b/obliteratus/distributed/numerical.py new file mode 100644 index 0000000..0ce27b7 --- /dev/null +++ b/obliteratus/distributed/numerical.py @@ -0,0 +1,252 @@ +"""Placement-aware projection oracle for the two-rank Gloo prototype.""" + +from __future__ import annotations + +import hashlib +import math + +import torch +import torch.distributed as dist + +from obliteratus.analysis.numerical_contracts import ( + ProjectionResult, + project_weight_against_direction, + select_projection_coefficients, +) +from obliteratus.distributed.consensus import require_consensus_digest +from obliteratus.distributed.contracts import ( + ContractError, + LogicalPlacement, + PlacementKind, + contract_digest, +) + + +def _tensor_digest(tensor: torch.Tensor) -> str: + value = tensor.detach().cpu().contiguous() + digest = hashlib.sha256() + digest.update(str(value.dtype).encode("ascii")) + digest.update(repr(tuple(value.shape)).encode("ascii")) + digest.update(value.view(torch.uint8).numpy().tobytes()) + return digest.hexdigest() + + +def _group_info(group: dist.ProcessGroup | None) -> tuple[int, int]: + if not dist.is_available() or not dist.is_initialized(): + raise ContractError("distributed projection requires an initialized process group") + if str(dist.get_backend(group)).lower() != "gloo": + raise ContractError("the prototype distributed projection requires Gloo") + return dist.get_rank(group), dist.get_world_size(group) + + +def _all_reduce_sum(value: torch.Tensor, group: dist.ProcessGroup | None) -> torch.Tensor: + result = value.clone() + dist.all_reduce(result, op=dist.ReduceOp.SUM, group=group) + return result + + +def _validate_placement_world( + placement: LogicalPlacement, + group: dist.ProcessGroup | None, +) -> None: + rank, world_size = _group_info(group) + if placement.rank != rank or placement.world_size != world_size: + raise ContractError("placement rank/world does not match the process group") + kind_code = { + PlacementKind.REPLICATED: 0, + PlacementKind.COLUMN_WISE: 1, + PlacementKind.ROW_WISE: 2, + }[placement.kind] + local = torch.tensor( + ( + kind_code, + placement.shard_dim if placement.shard_dim is not None else -1, + placement.shard_start, + placement.shard_end, + placement.direction_axis, + placement.global_shape[0], + placement.global_shape[1], + ), + dtype=torch.int64, + ) + gathered = [torch.zeros_like(local) for _ in range(world_size)] + dist.all_gather(gathered, local, group=group) + expected_shard_dim = int(local[1]) + expected_invariants = tuple(int(value) for value in local[4:]) + for other_rank, item in enumerate(gathered): + values = tuple(int(value) for value in item) + if values[0] != kind_code or values[1] != expected_shard_dim or values[4:] != expected_invariants: + raise ContractError("rank placement metadata disagrees") + if placement.kind is PlacementKind.REPLICATED: + if values[2:4] != (0, 0): + raise ContractError("replicated rank declared a shard interval") + else: + assert placement.shard_dim is not None + shard_size = placement.global_shape[placement.shard_dim] // world_size + if values[2:4] != (other_rank * shard_size, (other_rank + 1) * shard_size): + raise ContractError("rank shard intervals do not exactly tile the logical tensor") + invariant_digest = contract_digest( + { + "logical_name": placement.logical_name, + "global_shape": placement.global_shape, + "dtype": placement.dtype, + "kind": placement.kind, + "world_size": placement.world_size, + "direction_axis": placement.direction_axis, + "shard_dim": placement.shard_dim, + } + ) + require_consensus_digest(invariant_digest, group=group) + + +def _global_finite(local: torch.Tensor, group: dist.ProcessGroup | None) -> bool: + flag = torch.tensor(int(torch.isfinite(local).all()), dtype=torch.int64) + dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group) + return bool(int(flag)) + + +def _global_norm_sq(local: torch.Tensor, group: dist.ProcessGroup | None) -> float: + value = local.pow(2).sum().reshape(1) + return float(_all_reduce_sum(value, group).item()) + + +def _gather_coefficients(local: torch.Tensor, group: dist.ProcessGroup | None) -> torch.Tensor: + world_size = dist.get_world_size(group) + gathered = [torch.zeros_like(local) for _ in range(world_size)] + dist.all_gather(gathered, local, group=group) + return torch.cat(gathered, dim=0) + + +def _validate_ratio(value: object, name: str, *, lower_inclusive: bool) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ContractError(f"{name} must be a finite number") + normalized = float(value) + lower_ok = normalized >= 0.0 if lower_inclusive else normalized > 0.0 + if not lower_ok or normalized > 1.0: + interval = "[0, 1]" if lower_inclusive else "(0, 1]" + raise ContractError(f"{name} must be in {interval}") + return normalized + + +def distributed_project_weight( + weight: torch.Tensor, + direction: torch.Tensor, + placement: LogicalPlacement, + *, + group: dist.ProcessGroup | None = None, + norm_preserve: bool = False, + regularization: float = 0.0, + projection_row_fraction: float = 1.0, + max_norm_ratio: float = 1.10, +) -> ProjectionResult: + """Project one local shard with complete-logical-tensor semantics.""" + if not isinstance(placement, LogicalPlacement): + raise ContractError("placement must be a LogicalPlacement") + _validate_placement_world(placement, group) + if not isinstance(weight, torch.Tensor) or tuple(weight.shape) != placement.local_shape: + raise ContractError("local weight shape does not match the placement") + if not weight.is_floating_point(): + raise ContractError("distributed projection requires floating-point weights") + if weight.device.type != "cpu": + raise ContractError("the Gloo semantic oracle requires CPU tensors") + if str(weight.dtype).removeprefix("torch.") != placement.dtype: + raise ContractError("local weight dtype does not match the placement") + if ( + not isinstance(direction, torch.Tensor) + or direction.ndim != 1 + or direction.numel() != placement.global_shape[placement.direction_axis] + or not direction.is_floating_point() + ): + raise ContractError("direction does not match the declared logical axis") + if direction.device.type != "cpu": + raise ContractError("the Gloo semantic oracle requires CPU tensors") + regularization = _validate_ratio(regularization, "regularization", lower_inclusive=True) + projection_row_fraction = _validate_ratio( + projection_row_fraction, + "projection_row_fraction", + lower_inclusive=False, + ) + if ( + isinstance(max_norm_ratio, bool) + or not isinstance(max_norm_ratio, (int, float)) + or not math.isfinite(max_norm_ratio) + or max_norm_ratio <= 0 + ): + raise ContractError("max_norm_ratio must be positive and finite") + max_norm_ratio = float(max_norm_ratio) + + require_consensus_digest(_tensor_digest(direction), group=group) + if placement.kind is PlacementKind.REPLICATED: + require_consensus_digest(_tensor_digest(weight), group=group) + result = project_weight_against_direction( + weight, + direction, + norm_preserve=norm_preserve, + regularization=regularization, + projection_row_fraction=projection_row_fraction, + max_norm_ratio=max_norm_ratio, + ) + require_consensus_digest(_tensor_digest(result.weight), group=group) + return result + + compute_dtype = torch.float64 if torch.float64 in {weight.dtype, direction.dtype} else torch.float32 + work = weight.to(dtype=compute_dtype) + full_direction = direction.to(dtype=compute_dtype) + if not _global_finite(work, group) or not torch.isfinite(full_direction).all(): + return ProjectionResult(weight=weight.clone(), projected=False, coefficient_norm_sq=0.0, layout=None) + direction_norm = full_direction.norm() + if direction_norm < 1e-8: + return ProjectionResult(weight=weight.clone(), projected=False, coefficient_norm_sq=0.0, layout=None) + full_direction = full_direction / direction_norm + scale = 1.0 - regularization + direction_axis = placement.direction_axis + shard_dim = placement.shard_dim + layout = "standard" if direction_axis == 1 else "transposed" + + if shard_dim == direction_axis: + local_direction = full_direction[placement.shard_start : placement.shard_end] + if direction_axis == 1: + local_coefficient = work @ local_direction.reshape(-1, 1) + coefficient = _all_reduce_sum(local_coefficient, group) + selected = select_projection_coefficients(coefficient, projection_row_fraction) + projected = work - selected * (scale * local_direction.reshape(1, -1)) + else: + local_coefficient = local_direction.reshape(1, -1) @ work + coefficient = _all_reduce_sum(local_coefficient, group) + selected = select_projection_coefficients(coefficient, projection_row_fraction) + projected = work - (scale * local_direction.reshape(-1, 1)) * selected + coefficient_norm_sq = float(selected.pow(2).sum().item()) + else: + if direction_axis == 1: + local_coefficient = work @ full_direction.reshape(-1, 1) + combined = _gather_coefficients(local_coefficient, group) + selected_all = select_projection_coefficients(combined, projection_row_fraction) + local_count = local_coefficient.shape[0] + start = placement.rank * local_count + selected = selected_all[start : start + local_count] + projected = work - selected * (scale * full_direction.reshape(1, -1)) + else: + local_coefficient = (full_direction.reshape(1, -1) @ work).T + combined = _gather_coefficients(local_coefficient, group) + selected_all = select_projection_coefficients(combined, projection_row_fraction) + local_count = local_coefficient.shape[0] + start = placement.rank * local_count + selected = selected_all[start : start + local_count].T + projected = work - (scale * full_direction.reshape(-1, 1)) * selected + coefficient_norm_sq = float(selected_all.pow(2).sum().item()) + + if norm_preserve: + original_norm_sq = _global_norm_sq(work, group) + new_norm_sq = max( + 0.0, + original_norm_sq - scale * (2.0 - scale) * coefficient_norm_sq, + ) + if original_norm_sq > 0 and new_norm_sq > 0: + projected = projected * min(math.sqrt(original_norm_sq / new_norm_sq), max_norm_ratio) + + return ProjectionResult( + weight=projected.to(dtype=weight.dtype), + projected=True, + coefficient_norm_sq=coefficient_norm_sq if norm_preserve else 0.0, + layout=layout, + ) diff --git a/obliteratus/distributed/preflight.py b/obliteratus/distributed/preflight.py new file mode 100644 index 0000000..087ee86 --- /dev/null +++ b/obliteratus/distributed/preflight.py @@ -0,0 +1,1829 @@ +"""Admission and local probes for fixed-membership distributed preflight.""" + +from __future__ import annotations + +import ctypes +import hashlib +import hmac +import json +import os +import platform +import re +import signal +import shutil +import stat +import threading +import time +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Protocol + +import accelerate # type: ignore[import-untyped] +import safetensors +import torch +import torch.distributed as dist +import transformers + +from obliteratus.distributed.config import DistributedPreflightConfig +from obliteratus.distributed.consensus import ( + assert_rank_order, + gloo_all_gather_records, + require_record_consensus, + unanimous_vote, +) +from obliteratus.distributed.contracts import ( + ContractError, + RankInventory, + RuntimeContractError, + RuntimeStage, + RunIdentity, + StageMessage, + TopologyPlan, + Vote, + advance_stage, + canonical_record, + contract_digest, +) +from obliteratus.distributed.evidence import ( + PreflightEvidence, + read_evidence, + read_stage_message, + write_evidence, + write_stage_message, +) +from obliteratus.distributed.launcher import ( + TorchrunEnvironment, + control_group, + validate_network_interface, +) +from obliteratus.checkpoint_errors import CheckpointContractError +from obliteratus.checkpoint_inspection import InspectionLimits, inspect_checkpoint + + +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_ALLOWED_SOURCE_SUFFIXES = frozenset({".json", ".model", ".safetensors", ".tiktoken", ".txt"}) +_FORBIDDEN_SOURCE_SUFFIXES = frozenset({".bin", ".ckpt", ".pkl", ".pickle", ".pt", ".pth", ".py"}) +HASH_CHUNK_BYTES = 8 * 1024 * 1024 +_ACTIVE_DEADLINE_DEPTH = 0 + + +def _digest(value: object, name: str, pattern: re.Pattern[str] = _DIGEST_RE) -> None: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise ContractError(f"{name} has an invalid format") + + +def _memory(value: object, name: str, *, allow_zero: bool = False) -> None: + minimum = 0 if allow_zero else 1 + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= 2**63 - 1: + raise ContractError(f"{name} is outside its allowed range") + + +@dataclass(frozen=True) +class RankAttestation: + """Redaction-safe, exact local facts submitted by one fixed worker.""" + + rank: int + local_rank: int + local_world_size: int + group_rank: int + world_size: int + host_digest: str + device_digest: str + device_profile_digest: str + device_config_digest: str + device_kind: str + total_device_memory_bytes: int + free_device_memory_bytes: int + total_host_memory_bytes: int + free_host_memory_bytes: int + free_staging_bytes: int + software_digest: str + storage_digest: str + source_digest: str + model_digest: str + tokenizer_digest: str + config_digest: str + commit_sha: str + code_digest: str + placement_plan_digest: str + network_interface_digest: str + + def __post_init__(self) -> None: + RankInventory( + rank=self.rank, + local_rank=self.local_rank, + world_size=self.world_size, + host_digest=self.host_digest, + device_digest=self.device_digest, + device_kind=self.device_kind, + total_memory_bytes=self.total_device_memory_bytes, + free_memory_bytes=self.free_device_memory_bytes, + software_digest=self.software_digest, + storage_digest=self.storage_digest, + ) + _memory(self.total_host_memory_bytes, "total_host_memory_bytes") + _memory(self.free_host_memory_bytes, "free_host_memory_bytes", allow_zero=True) + if self.free_host_memory_bytes > self.total_host_memory_bytes: + raise ContractError("free_host_memory_bytes cannot exceed total_host_memory_bytes") + _memory(self.free_staging_bytes, "free_staging_bytes", allow_zero=True) + for name in ( + "device_profile_digest", + "device_config_digest", + "source_digest", + "model_digest", + "tokenizer_digest", + "config_digest", + "code_digest", + "placement_plan_digest", + "network_interface_digest", + ): + _digest(getattr(self, name), name) + _digest(self.commit_sha, "commit_sha", _COMMIT_RE) + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + @classmethod + def from_bytes(cls, payload: bytes) -> "RankAttestation": + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ContractError("rank attestation contains a duplicate field") + result[key] = value + return result + + try: + value = json.loads(payload.decode("utf-8"), object_pairs_hook=unique_object) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ContractError("rank attestation must be strict UTF-8 JSON") from exc + if not isinstance(value, dict) or set(value) != set(cls.__dataclass_fields__): + raise ContractError("rank attestation fields do not match the closed schema") + result = cls(**value) + if payload != canonical_record(result): + raise ContractError("rank attestation is not canonical") + return result + + +def _stage_message_from_bytes(payload: bytes) -> StageMessage: + try: + return StageMessage.from_bytes(payload) + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError( + "LMS_CONSENSUS_RECORD_INVALID", "lifecycle message is invalid" + ) from exc + + +def _validate_stage_messages( + messages: tuple[StageMessage, ...], + *, + run_id: str, + world_size: int, + stage: RuntimeStage, + sequence: int, + identity_digest: str, + vote: Vote | None, +) -> None: + if len(messages) != world_size or {item.rank for item in messages} != set(range(world_size)): + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", "lifecycle records do not cover the fixed world" + ) + if any( + item.run_id != run_id + or item.stage is not stage + or item.sequence != sequence + or item.identity_digest != identity_digest + or item.vote is not vote + or item.error_code is not None + for item in messages + ): + raise RuntimeContractError("LMS_LIFECYCLE_INVALID", "rank lifecycle records disagree") + + +@dataclass(frozen=True) +class SourceIdentity: + source_digest: str + model_digest: str + tokenizer_digest: str + file_count: int + total_bytes: int + + +@dataclass(frozen=True) +class LocalSnapshot: + host_identity: str + device_identity: str + device_name: str + compute_capability: str + device_kind: str + total_device_memory_bytes: int + free_device_memory_bytes: int + total_host_memory_bytes: int + free_host_memory_bytes: int + free_staging_bytes: int + storage_identity: str + source: SourceIdentity + software_versions: tuple[tuple[str, str], ...] + commit_sha: str + code_digest: str + + +@dataclass(frozen=True) +class PreflightResult: + """Capability that later distributed code must require before allocation.""" + + identity: RunIdentity + topology: TopologyPlan + attestations: tuple[RankAttestation, ...] + identity_digest: str + attempt_path: Path + source_identity: SourceIdentity + lifecycle_identity_digest: str + created_messages: tuple[StageMessage, ...] + + +class PreflightProbes(Protocol): + def collect( + self, config: DistributedPreflightConfig, launch: TorchrunEnvironment + ) -> LocalSnapshot: ... + + +def _hash_file( + path: Path, + *, + deadline: float, + max_bytes: int, + require_read_only: bool = True, +) -> tuple[int, str]: + descriptor: int | None = None + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or (require_read_only and before.st_mode & 0o222): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source files must be immutable regular files", + ) + if before.st_nlink != 1: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source files cannot have hard-link aliases", + ) + if before.st_size > max_bytes: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "source file exceeds the configured byte bound", + ) + digest = hashlib.sha256() + while chunk := os.read(descriptor, HASH_CHUNK_BYTES): + if time.monotonic() > deadline: + raise RuntimeContractError( + "LMS_STAGE_TIMEOUT", + "source inspection exceeded its explicit timeout", + ) + digest.update(chunk) + after = os.fstat(descriptor) + final_path = path.stat(follow_symlinks=False) + except RuntimeContractError: + raise + except OSError as exc: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source files must be immutable regular files", + ) from exc + finally: + if descriptor is not None: + os.close(descriptor) + identity = lambda item: ( + item.st_dev, + item.st_ino, + item.st_size, + item.st_mtime_ns, + item.st_mode, + ) + if identity(before) != identity(after) or identity(after) != identity(final_path): + raise RuntimeContractError( + "LMS_SOURCE_CHANGED", "source changed during preflight inspection" + ) + return before.st_size, digest.hexdigest() + + +@contextmanager +def _stage_deadline(timeout_seconds: int, message: str): + """Enforce a nestable stage bound with a process-level Linux timer.""" + + global _ACTIVE_DEADLINE_DEPTH + + if ( + not hasattr(signal, "SIGALRM") + or not hasattr(signal, "setitimer") + or threading.current_thread() is not threading.main_thread() + ): + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "the runtime cannot enforce the source inspection deadline", + ) + previous_timer = signal.getitimer(signal.ITIMER_REAL) + if previous_timer[0] > 0 and _ACTIVE_DEADLINE_DEPTH == 0: + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "the stage deadline conflicts with an active runtime timer", + ) + previous_handler = signal.getsignal(signal.SIGALRM) + started = time.monotonic() + + def expire(_signum: int, _frame: object) -> None: + raise RuntimeContractError("LMS_STAGE_TIMEOUT", message) + + signal.signal(signal.SIGALRM, expire) + duration = ( + min(float(timeout_seconds), previous_timer[0]) + if previous_timer[0] > 0 + else float(timeout_seconds) + ) + signal.setitimer(signal.ITIMER_REAL, duration) + _ACTIVE_DEADLINE_DEPTH += 1 + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + _ACTIVE_DEADLINE_DEPTH -= 1 + signal.signal(signal.SIGALRM, previous_handler) + if previous_timer[0] > 0: + remaining = previous_timer[0] - (time.monotonic() - started) + if remaining > 0: + signal.setitimer(signal.ITIMER_REAL, remaining, previous_timer[1]) + + +def inspect_source( + path: Path, + *, + max_files: int = 100_000, + max_total_bytes: int = 2**44, + max_file_bytes: int = 2**41, + timeout_seconds: int = 3600, +) -> SourceIdentity: + """Hash an immutable local HF safetensors tree without loading tensor payloads.""" + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, int) + or not 1 <= timeout_seconds <= 3600 + ): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "timeout_seconds is outside the bounded source-inspection contract", + ) + with _stage_deadline(timeout_seconds, "source inspection exceeded its explicit timeout"): + try: + return _inspect_source( + path, + max_files=max_files, + max_total_bytes=max_total_bytes, + max_file_bytes=max_file_bytes, + timeout_seconds=timeout_seconds, + ) + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source violates the bounded immutable input contract", + ) from exc + + +def _inspect_source( + path: Path, + *, + max_files: int, + max_total_bytes: int, + max_file_bytes: int, + timeout_seconds: int, +) -> SourceIdentity: + if path.is_symlink() or not path.is_dir(): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source must be a local non-symlink directory", + ) + if path.stat(follow_symlinks=False).st_mode & 0o222: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", "source directory must be immutable" + ) + for value, name, maximum in ( + (max_files, "max_files", 1_000_000), + (max_total_bytes, "max_total_bytes", 2**63 - 1), + (max_file_bytes, "max_file_bytes", 2**63 - 1), + (timeout_seconds, "timeout_seconds", 3600), + ): + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= maximum: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + f"{name} is outside the bounded source-inspection contract", + ) + if max_file_bytes > max_total_bytes: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "max_file_bytes cannot exceed max_total_bytes", + ) + deadline = time.monotonic() + timeout_seconds + files: list[Path] = [] + observed_bytes = 0 + observed_inodes: set[tuple[int, int]] = set() + for directory, directory_names, file_names in os.walk(path, topdown=True, followlinks=False): + if time.monotonic() > deadline: + raise RuntimeContractError( + "LMS_STAGE_TIMEOUT", "source inspection exceeded its explicit timeout" + ) + directory_names.sort() + file_names.sort() + directory_path = Path(directory) + directory_metadata = directory_path.stat(follow_symlinks=False) + if not stat.S_ISDIR(directory_metadata.st_mode) or directory_metadata.st_mode & 0o222: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "every source directory must be immutable", + ) + for name in (*directory_names, *file_names): + item = directory_path / name + metadata = item.stat(follow_symlinks=False) + if stat.S_ISLNK(metadata.st_mode): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source cannot contain symbolic links", + ) + for name in file_names: + item = directory_path / name + metadata = item.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source can contain only regular files and directories", + ) + if metadata.st_nlink != 1: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source files cannot have hard-link aliases", + ) + files.append(item) + if len(files) > max_files: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "source file count exceeds the configured bound", + ) + if metadata.st_size > max_file_bytes: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "source file exceeds the configured byte bound", + ) + observed_bytes += metadata.st_size + if observed_bytes > max_total_bytes: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "source exceeds the configured total-byte bound", + ) + inode = (metadata.st_dev, metadata.st_ino) + if inode in observed_inodes: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source files cannot alias the same inode", + ) + observed_inodes.add(inode) + if not files: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source file count is outside the bounded contract", + ) + for item in files: + suffix = item.suffix.lower() + if suffix in _FORBIDDEN_SOURCE_SUFFIXES or suffix not in _ALLOWED_SOURCE_SUFFIXES: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source contains a serialization type outside the safetensors envelope", + ) + try: + report = inspect_checkpoint( + path, + limits=InspectionLimits( + max_files=max_files, + max_directories=min(1_000_000, max_files + 1), + max_total_bytes=max_total_bytes, + max_json_bytes=min(max_file_bytes, 8 << 20), + max_safetensors_header_bytes=min(max_file_bytes, 64 << 20), + max_tensors=2_000_000, + hash_chunk_bytes=HASH_CHUNK_BYTES, + ), + ) + except CheckpointContractError as exc: + if exc.code == "DCI_SOURCE_CHANGED": + raise RuntimeContractError( + "LMS_SOURCE_CHANGED", "source changed during safe-structure inspection" + ) from exc + if exc.code == "DCI_RESOURCE_LIMIT" and exc.detail == "deadline": + raise RuntimeContractError( + "LMS_STAGE_TIMEOUT", "source inspection exceeded its explicit timeout" + ) from exc + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source failed bounded safe-structure inspection", + ) from exc + if time.monotonic() > deadline: + raise RuntimeContractError( + "LMS_STAGE_TIMEOUT", "source inspection exceeded its explicit timeout" + ) + descriptor = report.to_dict() + component_formats = {component["format"] for component in descriptor["components"]} + if ( + report.primary_format != "hf_safetensors" + or report.support_decision != "canonical_hf_ready" + or component_formats != {"hf_safetensors"} + ): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source must be a canonical Hugging Face safetensors checkpoint without adapters", + ) + inventory = descriptor["source_inventory"] + inventory_files = inventory["files"] + if len(inventory_files) != len(files) or inventory["total_bytes"] != observed_bytes: + raise RuntimeContractError( + "LMS_SOURCE_CHANGED", "source changed during safe-structure inspection" + ) + records: list[dict[str, object]] = [] + model_records: list[dict[str, object]] = [] + tokenizer_records: list[dict[str, object]] = [] + for observed in inventory_files: + relative_path = str(observed["relative_path"]) + record = { + "path": relative_path, + "size_bytes": observed["size_bytes"], + "sha256": observed["sha256"], + } + records.append(record) + if relative_path.lower().endswith(".safetensors"): + model_records.append(record) + name = Path(relative_path).name.lower() + if any(marker in name for marker in ("token", "vocab", "merges")): + tokenizer_records.append(record) + if not model_records: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source must contain at least one safetensors file", + ) + if not tokenizer_records: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "source must contain immutable tokenizer metadata", + ) + return SourceIdentity( + source_digest=contract_digest(records), + model_digest=contract_digest(model_records), + tokenizer_digest=contract_digest(tokenizer_records), + file_count=len(records), + total_bytes=observed_bytes, + ) + + +def revalidate_source( + config: DistributedPreflightConfig, expected: SourceIdentity +) -> SourceIdentity: + """Re-hash immediately before #60 allocation; replacement fails capability use.""" + observed = inspect_source( + config.source_path, + max_files=config.max_source_files, + max_total_bytes=config.max_source_bytes, + max_file_bytes=config.max_source_file_bytes, + timeout_seconds=config.source_timeout_seconds, + ) + if observed != expected: + raise RuntimeContractError( + "LMS_SOURCE_CHANGED", + "source identity changed after distributed preflight", + ) + return observed + + +def _host_memory() -> tuple[int, int]: + page_size = os.sysconf("SC_PAGE_SIZE") + total = page_size * os.sysconf("SC_PHYS_PAGES") + available = page_size * os.sysconf("SC_AVPHYS_PAGES") + return total, available + + +def _cuda_driver_version() -> str: + try: + library = ctypes.CDLL("libcuda.so.1") + if library.cuInit(0) != 0: + return "unavailable" + value = ctypes.c_int() + if library.cuDriverGetVersion(ctypes.byref(value)) != 0: + return "unavailable" + return str(value.value) + except (AttributeError, OSError): + return "unavailable" + + +def _nccl_version() -> str: + try: + value = torch.cuda.nccl.version() + except Exception: + return "unavailable" + if isinstance(value, tuple): + return ".".join(str(item) for item in value) + return str(value) + + +def _bounded_text_file(path: Path, *, limit: int = 4096) -> str: + descriptor: int | None = None + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or not 1 <= metadata.st_size <= limit: + raise ContractError("checkout identity file is invalid") + raw = os.read(descriptor, limit + 1) + if len(raw) != metadata.st_size: + raise ContractError("checkout identity changed while it was read") + return raw.decode("utf-8").strip() + except ContractError: + raise + except (OSError, UnicodeError) as exc: + raise ContractError("checkout identity is unavailable") from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def checkout_commit(root: Path | None = None) -> str: + """Resolve the checkout HEAD without invoking Git or another process.""" + checkout = root or Path(__file__).resolve().parents[2] + dot_git = checkout / ".git" + if dot_git.is_dir() and not dot_git.is_symlink(): + git_dir = dot_git + else: + pointer = _bounded_text_file(dot_git) + prefix = "gitdir: " + if not pointer.startswith(prefix): + raise ContractError("checkout identity is unavailable") + git_dir = Path(pointer[len(prefix) :]) + if not git_dir.is_absolute(): + git_dir = (checkout / git_dir).resolve() + head = _bounded_text_file(git_dir / "HEAD") + if _COMMIT_RE.fullmatch(head): + return head + prefix = "ref: " + if not head.startswith(prefix): + raise ContractError("checkout HEAD has an invalid format") + reference = head[len(prefix) :] + if not reference.startswith("refs/") or ".." in reference or "\\" in reference: + raise ContractError("checkout HEAD reference has an invalid format") + common_dir = git_dir + common_pointer = git_dir / "commondir" + if common_pointer.exists(): + common_dir = (git_dir / _bounded_text_file(common_pointer)).resolve() + reference_path = common_dir / reference + if reference_path.exists(): + commit = _bounded_text_file(reference_path) + else: + packed = _bounded_text_file(common_dir / "packed-refs", limit=1 << 20) + matches = [ + line.split(" ", 1)[0] + for line in packed.splitlines() + if not line.startswith(("#", "^")) and line.endswith(f" {reference}") + ] + if len(matches) != 1: + raise ContractError("checkout HEAD reference is unavailable") + commit = matches[0] + if _COMMIT_RE.fullmatch(commit) is None: + raise ContractError("checkout commit has an invalid format") + return commit + + +def checkout_code_digest(root: Path | None = None) -> str: + """Hash the executable Python package independently of Git metadata.""" + + checkout = root or Path(__file__).resolve().parents[2] + package = checkout / "obliteratus" + records: list[dict[str, object]] = [] + total_bytes = 0 + for path in sorted(package.rglob("*.py")): + if len(records) >= 10_000: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", "executable source exceeds its file bound" + ) + if path.is_symlink() or not path.is_file(): + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", + "executable source must contain only regular Python files", + ) + size, digest = _hash_file( + path, + deadline=time.monotonic() + 60, + max_bytes=8 << 20, + require_read_only=False, + ) + total_bytes += size + if total_bytes > 64 << 20: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", "executable source exceeds its byte bound" + ) + records.append( + { + "path": path.relative_to(checkout).as_posix(), + "size_bytes": size, + "sha256": digest, + } + ) + if not records: + raise RuntimeContractError( + "LMS_SOURCE_BOUNDARY_VIOLATION", "executable source inventory is empty" + ) + return contract_digest(records) + + +def storage_mount_digest(path: Path) -> str: + """Measure the exact Linux mount backing a local staging directory.""" + + try: + resolved = path.resolve(strict=True) + if resolved != path.absolute() or path.is_symlink() or not path.is_dir(): + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "staging must be a local non-symlink directory", + ) + descriptor = os.open("/proc/self/mountinfo", os.O_RDONLY | os.O_NOFOLLOW) + try: + raw = os.read(descriptor, (4 << 20) + 1) + finally: + os.close(descriptor) + if len(raw) > 4 << 20: + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", "mount inventory exceeds its byte bound" + ) + lines = raw.decode("utf-8").splitlines() + except RuntimeContractError: + raise + except (OSError, UnicodeError) as exc: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", "staging mount identity is unavailable" + ) from exc + + def unescape(value: str) -> str: + return ( + value.replace("\\040", " ") + .replace("\\011", "\t") + .replace("\\012", "\n") + .replace("\\134", "\\") + ) + + candidates: list[tuple[int, dict[str, object]]] = [] + for line in lines: + if " - " not in line: + continue + left, right = line.split(" - ", 1) + left_fields = left.split() + right_fields = right.split() + if len(left_fields) < 6 or len(right_fields) < 3: + continue + mountpoint = Path(unescape(left_fields[4])) + try: + resolved.relative_to(mountpoint) + except ValueError: + continue + candidates.append( + ( + len(mountpoint.parts), + { + "root": unescape(left_fields[3]), + "mountpoint": mountpoint.as_posix(), + "mount_options": sorted(left_fields[5].split(",")), + "filesystem": right_fields[0], + "source": unescape(right_fields[1]), + "super_options": sorted(right_fields[2].split(",")), + }, + ) + ) + if not candidates: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", "staging mount identity is unavailable" + ) + return contract_digest(max(candidates, key=lambda item: item[0])[1]) + + +class SystemProbes: + """Production local probes; tests inject deterministic snapshots instead.""" + + def collect( + self, config: DistributedPreflightConfig, launch: TorchrunEnvironment + ) -> LocalSnapshot: + source = inspect_source( + config.source_path, + max_files=config.max_source_files, + max_total_bytes=config.max_source_bytes, + max_file_bytes=config.max_source_file_bytes, + timeout_seconds=config.source_timeout_seconds, + ) + if config.staging_path.is_symlink() or not config.staging_path.is_dir(): + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "staging must be a local non-symlink directory", + ) + disk = shutil.disk_usage(config.staging_path) + validate_network_interface( + config.network_interface, + config.allowed_master_cidrs, + config.master_addr, + coordinator=launch.rank == config.coordinator_rank, + ) + host_total, host_free = _host_memory() + software = tuple( + sorted( + { + "python": platform.python_version(), + "platform": platform.platform(), + "machine": platform.machine(), + "torch": torch.__version__.split("+")[0], + "transformers": transformers.__version__, + "accelerate": accelerate.__version__, + "safetensors": safetensors.__version__, + "cuda": torch.version.cuda or "unavailable", + "nccl": _nccl_version(), + "driver": _cuda_driver_version(), + }.items() + ) + ) + if config.device_kind == "cuda": + if not torch.cuda.is_available(): + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "the configured CUDA device is unavailable", + ) + torch.cuda.set_device(launch.local_rank) + properties = torch.cuda.get_device_properties(launch.local_rank) + device_uuid = getattr(properties, "uuid", None) + if not device_uuid: + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "the CUDA device UUID is unavailable", + ) + free_device, total_device = torch.cuda.mem_get_info(launch.local_rank) + device_identity = str(device_uuid) + device_name = properties.name + compute_capability = f"{properties.major}.{properties.minor}" + else: + total_device, free_device = host_total, host_free + device_identity = f"cpu:{platform.node()}:{platform.machine()}:{launch.local_rank}" + device_name = "cpu" + compute_capability = "none" + try: + commit_sha = checkout_commit() + code_digest = checkout_code_digest() + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "checkout identity cannot be verified", + ) from exc + return LocalSnapshot( + host_identity=platform.node(), + device_identity=device_identity, + device_name=device_name, + compute_capability=compute_capability, + device_kind=config.device_kind, + total_device_memory_bytes=total_device, + free_device_memory_bytes=free_device, + total_host_memory_bytes=host_total, + free_host_memory_bytes=host_free, + free_staging_bytes=disk.free, + storage_identity=storage_mount_digest(config.staging_path), + source=source, + software_versions=software, + commit_sha=commit_sha, + code_digest=code_digest, + ) + + +def _opaque_identity(value: str, rendezvous_id: str) -> str: + return hmac.new(bytes.fromhex(rendezvous_id), value.encode(), hashlib.sha256).hexdigest() + + +def _attestation( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, + snapshot: LocalSnapshot, +) -> RankAttestation: + source = snapshot.source + return RankAttestation( + rank=launch.rank, + local_rank=launch.local_rank, + local_world_size=launch.local_world_size, + group_rank=launch.group_rank, + world_size=launch.world_size, + host_digest=_opaque_identity(snapshot.host_identity, launch.rendezvous_id), + device_digest=_opaque_identity(snapshot.device_identity, launch.rendezvous_id), + device_profile_digest=contract_digest( + { + "name": snapshot.device_name, + "compute_capability": snapshot.compute_capability, + "total_memory_bytes": snapshot.total_device_memory_bytes, + } + ), + device_config_digest=contract_digest( + { + "kind": snapshot.device_kind, + "name": snapshot.device_name, + "compute_capability": snapshot.compute_capability, + } + ), + device_kind=snapshot.device_kind, + total_device_memory_bytes=snapshot.total_device_memory_bytes, + free_device_memory_bytes=snapshot.free_device_memory_bytes, + total_host_memory_bytes=snapshot.total_host_memory_bytes, + free_host_memory_bytes=snapshot.free_host_memory_bytes, + free_staging_bytes=snapshot.free_staging_bytes, + software_digest=contract_digest(dict(snapshot.software_versions)), + storage_digest=snapshot.storage_identity, + source_digest=source.source_digest, + model_digest=source.model_digest, + tokenizer_digest=source.tokenizer_digest, + config_digest=config.digest, + commit_sha=snapshot.commit_sha, + code_digest=snapshot.code_digest, + placement_plan_digest=config.placement_plan_digest, + network_interface_digest=_opaque_identity(config.network_interface, launch.rendezvous_id), + ) + + +def _prepare_attempt_directory( + config: DistributedPreflightConfig, launch: TorchrunEnvironment +) -> Path: + attempt = config.staging_path / config.run_id + staging_descriptor: int | None = None + attempt_descriptor: int | None = None + try: + staging_descriptor = os.open( + config.staging_path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + ) + if launch.rank == config.coordinator_rank: + try: + os.mkdir(config.run_id, mode=0o700, dir_fd=staging_descriptor) + except FileExistsError as exc: + raise RuntimeContractError( + "LMS_ATTEMPT_IDENTITY_CONFLICT", + "distributed staging attempt cannot be reused", + ) from exc + dist.barrier() + attempt_descriptor = os.open( + config.run_id, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=staging_descriptor, + ) + metadata = os.fstat(attempt_descriptor) + effective_uid = getattr(os, "geteuid", lambda: metadata.st_uid)() + if metadata.st_uid != effective_uid or metadata.st_mode & 0o077: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "distributed staging attempt is not private and worker-owned", + ) + dist.barrier() + return attempt + except RuntimeContractError: + raise + except OSError as exc: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "distributed staging attempt is unavailable", + ) from exc + finally: + if attempt_descriptor is not None: + os.close(attempt_descriptor) + if staging_descriptor is not None: + os.close(staging_descriptor) + + +def _probe_shared_staging(config: DistributedPreflightConfig, launch: TorchrunEnvironment) -> Path: + attempt = config.staging_path / config.run_id + pending = ".shared-probe.pending" + ready = ".shared-probe.ready" + nonce = contract_digest( + {"run_id": config.run_id, "rendezvous_id": config.rendezvous_id} + ).encode() + staging_descriptor: int | None = None + attempt_descriptor: int | None = None + try: + staging_descriptor = os.open( + config.staging_path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + ) + try: + attempt_descriptor = os.open( + config.run_id, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=staging_descriptor, + ) + except OSError as exc: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "shared staging attempt is not visible to every rank", + ) from exc + if launch.rank == config.coordinator_rank: + descriptor = os.open( + pending, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=attempt_descriptor, + ) + try: + if os.write(descriptor, nonce) != len(nonce): + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "shared staging probe write was incomplete", + ) + os.fsync(descriptor) + finally: + os.close(descriptor) + os.rename( + pending, + ready, + src_dir_fd=attempt_descriptor, + dst_dir_fd=attempt_descriptor, + ) + os.fsync(attempt_descriptor) + dist.barrier() + descriptor = os.open(ready, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=attempt_descriptor) + try: + observed = os.read(descriptor, len(nonce) + 1) + finally: + os.close(descriptor) + if observed != nonce: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", + "shared staging probe identity disagrees", + ) + dist.barrier() + if launch.rank == config.coordinator_rank: + os.unlink(ready, dir_fd=attempt_descriptor) + os.fsync(attempt_descriptor) + dist.barrier() + return attempt + except ContractError: + raise + except OSError as exc: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", "shared staging probe failed" + ) from exc + finally: + cleanup_failed = False + if attempt_descriptor is not None and launch.rank == config.coordinator_rank: + for marker in (pending, ready): + try: + os.unlink(marker, dir_fd=attempt_descriptor) + except FileNotFoundError: + pass + except OSError: + cleanup_failed = True + try: + os.fsync(attempt_descriptor) + except OSError: + cleanup_failed = True + if attempt_descriptor is not None: + os.close(attempt_descriptor) + if staging_descriptor is not None: + os.close(staging_descriptor) + if cleanup_failed: + raise RuntimeContractError("LMS_CLEANUP_INCOMPLETE", "shared staging cleanup failed") + + +def _run_identity(config: DistributedPreflightConfig) -> RunIdentity: + return RunIdentity( + run_id=config.run_id, + config_digest=config.digest, + source_digest=config.source_digest, + model_digest=config.model_digest, + tokenizer_digest=config.tokenizer_digest, + commit_sha=config.commit_sha, + world_size=config.world_size, + ) + + +def _begin_lifecycle( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, +) -> tuple[RunIdentity, str, tuple[StageMessage, ...]]: + identity = _run_identity(config) + try: + require_record_consensus(identity) + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError("LMS_IDENTITY_MISMATCH", "rank run identities disagree") from exc + lifecycle_identity_digest = contract_digest(identity) + created = StageMessage( + run_id=config.run_id, + identity_digest=lifecycle_identity_digest, + rank=launch.rank, + sequence=0, + stage=RuntimeStage.CREATED, + ) + created_messages = tuple( + _stage_message_from_bytes(item) for item in gloo_all_gather_records(created) + ) + _validate_stage_messages( + created_messages, + run_id=config.run_id, + world_size=config.world_size, + stage=RuntimeStage.CREATED, + sequence=0, + identity_digest=lifecycle_identity_digest, + vote=None, + ) + return identity, lifecycle_identity_digest, created_messages + + +def run_preflight( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, + *, + probes: PreflightProbes | None = None, + lifecycle: tuple[RunIdentity, str, tuple[StageMessage, ...]] | None = None, +) -> PreflightResult: + """Admit the complete worker group; this function has no model allocation path.""" + + try: + config.validate() + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", + "distributed runtime profile validation failed", + ) from exc + identity, lifecycle_identity_digest, created_messages = lifecycle or _begin_lifecycle( + config, launch + ) + attempt = _prepare_attempt_directory(config, launch) + with _stage_deadline( + min(config.source_timeout_seconds, config.collective_timeout_seconds), + "local preflight probes exceeded their explicit timeout", + ): + snapshot = (probes or SystemProbes()).collect(config, launch) + local = _attestation(config, launch, snapshot) + try: + gathered = gloo_all_gather_records(local) + attestations = tuple(RankAttestation.from_bytes(item) for item in gathered) + except RuntimeContractError: + raise + except ContractError as exc: + raise RuntimeContractError( + "LMS_CONSENSUS_RECORD_INVALID", "rank attestation record is invalid" + ) from exc + validate_attestations( + attestations, + world_size=config.world_size, + tensor_parallel_size=config.tensor_parallel_size, + dimension_divisors=config.dimension_divisors, + expected_device_kind=config.device_kind, + expected_device_config_digest=contract_digest( + { + "kind": config.device_kind, + "name": config.device_name, + "compute_capability": config.compute_capability, + } + ), + expected_software_digest=config.software_digest, + expected_source_digest=config.source_digest, + expected_model_digest=config.model_digest, + expected_tokenizer_digest=config.tokenizer_digest, + expected_config_digest=config.digest, + expected_commit_sha=config.commit_sha, + expected_code_digest=config.code_digest, + expected_placement_plan_digest=config.placement_plan_digest, + expected_storage_digest=config.storage_digest, + expected_network_interface_digest=_opaque_identity( + config.network_interface, launch.rendezvous_id + ), + local_world_size=config.local_world_size, + min_free_device_memory_bytes=config.min_free_device_memory_bytes, + min_free_host_memory_bytes=config.min_free_host_memory_bytes, + min_free_staging_bytes=config.min_free_staging_bytes, + ) + if _probe_shared_staging(config, launch) != attempt: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", "staging attempt identity changed" + ) + if not unanimous_vote(0, True): + raise ContractError("preflight admission was not unanimous") + topology = TopologyPlan( + world_size=config.world_size, + coordinator_rank=config.coordinator_rank, + backend="gloo", + placement_plan_digest=config.placement_plan_digest, + ) + identity_digest = contract_digest( + { + "identity": identity, + "topology": topology, + "attestations": attestations, + "created_messages": created_messages, + } + ) + return PreflightResult( + identity, + topology, + attestations, + identity_digest, + attempt, + snapshot.source, + lifecycle_identity_digest, + created_messages, + ) + + +def _failure_code(error: BaseException) -> str: + if isinstance(error, RuntimeContractError): + return error.code + if isinstance(error, KeyboardInterrupt): + return "LMS_ATTEMPT_CANCELLED" + if isinstance(error, TimeoutError): + return "LMS_STAGE_TIMEOUT" + if isinstance(error, ContractError): + return "LMS_LIFECYCLE_INVALID" + return "LMS_COLLECTIVE_FAILED" + + +_FAILURE_PRIORITY = ( + "LMS_CLEANUP_INCOMPLETE", + "LMS_DIAGNOSTIC_REDACTION_FAILED", + "LMS_SECURITY_BASELINE_REVOKED", + "LMS_SECRET_INPUT_REJECTED", + "LMS_SOURCE_CHANGED", + "LMS_SOURCE_BOUNDARY_VIOLATION", + "LMS_IDENTITY_MISMATCH", + "LMS_STORAGE_PROFILE_MISMATCH", + "LMS_NETWORK_PROFILE_DENIED", + "LMS_RUNTIME_PROFILE_MISMATCH", + "LMS_RESOURCE_ADMISSION_DENIED", + "LMS_ATTEMPT_CANCELLED", + "LMS_ATTEMPT_IDENTITY_CONFLICT", + "LMS_MEMBERSHIP_INVALID", + "LMS_MEMBERSHIP_TIMEOUT", + "LMS_ELASTICITY_FORBIDDEN", + "LMS_LAUNCH_IDENTITY_INVALID", + "LMS_RANK_DEVICE_CONFLICT", + "LMS_TOPOLOGY_UNSUPPORTED", + "LMS_TRANSPORT_BOUNDARY_UNSATISFIED", + "LMS_FORBIDDEN_RUNTIME_CAPABILITY", + "LMS_ATOMIC_PROMOTION_UNAVAILABLE", + "LMS_STAGE_TIMEOUT", + "LMS_CONSENSUS_RECORD_INVALID", + "LMS_LIFECYCLE_INVALID", + "LMS_EVIDENCE_UNAVAILABLE", + "LMS_EVIDENCE_SCOPE_MISMATCH", + "LMS_CLOCK_PROFILE_MISMATCH", + "LMS_DISTRIBUTED_INTENT_REQUIRED", + "LMS_COLLECTIVE_FAILED", +) + + +def _select_failure_code(codes: set[str]) -> str: + for code in _FAILURE_PRIORITY: + if code in codes: + return code + return "LMS_COLLECTIVE_FAILED" + + +def _collective_aborting( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, + identity_digest: str, + local_code: str, +) -> tuple[str, tuple[StageMessage, ...]]: + advance_stage(RuntimeStage.CREATED, RuntimeStage.ABORTING) + local = StageMessage( + run_id=config.run_id, + identity_digest=identity_digest, + rank=launch.rank, + sequence=1, + stage=RuntimeStage.ABORTING, + vote=Vote.ABORT, + error_code=local_code, + ) + try: + messages = tuple(_stage_message_from_bytes(item) for item in gloo_all_gather_records(local)) + except RuntimeContractError: + raise + except (ContractError, RuntimeError) as exc: + raise RuntimeContractError( + "LMS_LIFECYCLE_INVALID", + "collective abort lifecycle could not be established", + ) from exc + if len(messages) != config.world_size or {item.rank for item in messages} != set( + range(config.world_size) + ): + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", + "abort lifecycle records do not cover the fixed world", + ) + if any( + item.run_id != config.run_id + or item.identity_digest != identity_digest + or item.sequence != 1 + or item.stage is not RuntimeStage.ABORTING + or item.vote is not Vote.ABORT + or item.error_code is None + for item in messages + ): + raise RuntimeContractError("LMS_LIFECYCLE_INVALID", "rank abort lifecycle records disagree") + return _select_failure_code({item.error_code for item in messages if item.error_code}), messages + + +def _write_local_terminal_receipts( + attempt: Path, + *, + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, + identity_digest: str, + code: str, + quarantined: bool, +) -> None: + aborting = StageMessage( + run_id=config.run_id, + identity_digest=identity_digest, + rank=launch.rank, + sequence=1, + stage=RuntimeStage.ABORTING, + vote=Vote.ABORT, + error_code=code, + ) + terminal_stage = RuntimeStage.QUARANTINED if quarantined else RuntimeStage.ABORTED + advance_stage(RuntimeStage.ABORTING, terminal_stage) + terminal = StageMessage( + run_id=config.run_id, + identity_digest=identity_digest, + rank=launch.rank, + sequence=2, + stage=terminal_stage, + vote=Vote.ABORT, + error_code=code, + ) + write_stage_message(attempt / f".rank-{launch.rank}.aborting.stage.json", aborting) + write_stage_message(attempt / f".rank-{launch.rank}.terminal.stage.json", terminal) + + +def _read_terminal_receipts( + attempt: Path, + *, + config: DistributedPreflightConfig, + identity_digest: str, +) -> tuple[tuple[StageMessage, ...], tuple[StageMessage, ...]]: + aborting = tuple( + read_stage_message(attempt / f".rank-{rank}.aborting.stage.json") + for rank in range(config.world_size) + ) + terminal = tuple( + read_stage_message(attempt / f".rank-{rank}.terminal.stage.json") + for rank in range(config.world_size) + ) + expected_ranks = list(range(config.world_size)) + if [item.rank for item in aborting] != expected_ranks or [ + item.rank for item in terminal + ] != expected_ranks: + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", "terminal lifecycle receipts omit a fixed rank" + ) + for before, after in zip(aborting, terminal, strict=True): + if ( + before.run_id != config.run_id + or after.run_id != config.run_id + or before.identity_digest != identity_digest + or after.identity_digest != identity_digest + or before.sequence != 1 + or after.sequence != 2 + or before.stage is not RuntimeStage.ABORTING + or after.stage not in {RuntimeStage.ABORTED, RuntimeStage.QUARANTINED} + or before.vote is not Vote.ABORT + or after.vote is not Vote.ABORT + or before.error_code is None + or after.error_code != before.error_code + ): + raise RuntimeContractError( + "LMS_LIFECYCLE_INVALID", "terminal lifecycle receipts disagree" + ) + advance_stage(before.stage, after.stage) + return aborting, terminal + + +def _ensure_failure_attempt(config: DistributedPreflightConfig) -> Path: + attempt = config.staging_path / config.run_id + if attempt.exists(): + return attempt + descriptor: int | None = None + try: + descriptor = os.open(config.staging_path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + os.mkdir(config.run_id, mode=0o700, dir_fd=descriptor) + os.fsync(descriptor) + return attempt + except FileExistsError: + return attempt + except OSError as exc: + raise RuntimeContractError( + "LMS_EVIDENCE_UNAVAILABLE", + "failure evidence directory could not be established", + ) from exc + finally: + if descriptor is not None: + os.close(descriptor) + + +def _wait_for_evidence(path: Path, *, timeout_seconds: int) -> PreflightEvidence: + deadline = time.monotonic() + timeout_seconds + while True: + try: + return read_evidence(path) + except FileNotFoundError: + if time.monotonic() >= deadline: + raise RuntimeContractError( + "LMS_EVIDENCE_UNAVAILABLE", + "terminal evidence was not published within its bound", + ) from None + time.sleep(0.01) + + +def _write_terminal_evidence(path: Path, evidence: PreflightEvidence) -> None: + try: + write_evidence(path, evidence) + except Exception as exc: + raise RuntimeContractError( + "LMS_EVIDENCE_UNAVAILABLE", "terminal evidence could not be persisted" + ) from exc + + +def execute_preflight( + config: DistributedPreflightConfig, + launch: TorchrunEnvironment, + *, + probes: PreflightProbes | None = None, +) -> PreflightEvidence: + """Own the control group and emit a redacted terminal preflight record.""" + + result: PreflightResult | None = None + prepared_evidence: PreflightEvidence | None = None + lifecycle_identity_digest = contract_digest(_run_identity(config)) + terminal_identity: str | None = None + local_code: str | None = None + try: + with control_group(config, launch): + try: + lifecycle = _begin_lifecycle(config, launch) + result = run_preflight( + config, + launch, + probes=probes, + lifecycle=lifecycle, + ) + advance_stage(RuntimeStage.CREATED, RuntimeStage.PREFLIGHTED) + preflighted = StageMessage( + run_id=config.run_id, + identity_digest=result.lifecycle_identity_digest, + rank=launch.rank, + sequence=1, + stage=RuntimeStage.PREFLIGHTED, + vote=Vote.PREPARED, + ) + preflighted_messages = tuple( + _stage_message_from_bytes(item) for item in gloo_all_gather_records(preflighted) + ) + _validate_stage_messages( + preflighted_messages, + run_id=config.run_id, + world_size=config.world_size, + stage=RuntimeStage.PREFLIGHTED, + sequence=1, + identity_digest=result.lifecycle_identity_digest, + vote=Vote.PREPARED, + ) + terminal_identity = contract_digest( + { + "admission_identity": result.identity_digest, + "preflighted_messages": preflighted_messages, + } + ) + prepared_evidence = PreflightEvidence.success( + run_id=config.run_id, + config_digest=config.digest, + world_size=config.world_size, + evidence_tier=config.evidence_tier, + identity_digest=terminal_identity, + ) + prepared_marker = StageMessage( + run_id=config.run_id, + identity_digest=terminal_identity, + rank=config.coordinator_rank, + sequence=1, + stage=RuntimeStage.PREFLIGHTED, + vote=Vote.PREPARED, + ) + prepared_path = result.attempt_path / ".preflight.prepared.stage.json" + local_sink_ready = True + if launch.rank == config.coordinator_rank: + try: + write_stage_message(prepared_path, prepared_marker) + except Exception: + local_sink_ready = False + dist.barrier() + try: + observed = read_stage_message(prepared_path) + local_sink_ready = local_sink_ready and observed == prepared_marker + except Exception: + local_sink_ready = False + if not unanimous_vote(2, local_sink_ready): + raise RuntimeContractError( + "LMS_EVIDENCE_UNAVAILABLE", + "prepared lifecycle record was not verified by every rank", + ) + except BaseException as error: + local_code = _failure_code(error) + try: + local_code, _messages = _collective_aborting( + config, + launch, + lifecycle_identity_digest, + local_code, + ) + except BaseException as lifecycle_error: + local_code = _select_failure_code({local_code, _failure_code(lifecycle_error)}) + raise RuntimeContractError(local_code, "distributed preflight aborted") from None + except BaseException as error: + local_code = _select_failure_code( + {code for code in (local_code, _failure_code(error)) if code is not None} + ) + + attempt = result.attempt_path if result is not None else config.staging_path / config.run_id + if launch.rank == config.coordinator_rank and not attempt.exists(): + try: + attempt = _ensure_failure_attempt(config) + except RuntimeContractError as error: + local_code = _select_failure_code( + {code for code in (local_code, error.code) if code is not None} + ) + + if local_code is None and prepared_evidence is not None and terminal_identity is not None: + acknowledgement = StageMessage( + run_id=config.run_id, + identity_digest=terminal_identity, + rank=launch.rank, + sequence=2, + stage=RuntimeStage.PREFLIGHTED, + vote=Vote.COMMITTED, + ) + try: + write_stage_message( + attempt / f".rank-{launch.rank}.teardown.stage.json", + acknowledgement, + ) + except Exception: + local_code = "LMS_CLEANUP_INCOMPLETE" + else: + deadline = time.monotonic() + config.teardown_timeout_seconds + while not attempt.is_dir() and time.monotonic() < deadline: + time.sleep(0.01) + try: + _write_local_terminal_receipts( + attempt, + config=config, + launch=launch, + identity_digest=lifecycle_identity_digest, + code=local_code or "LMS_COLLECTIVE_FAILED", + quarantined=local_code == "LMS_CLEANUP_INCOMPLETE", + ) + except Exception: + local_code = "LMS_CLEANUP_INCOMPLETE" + + if launch.rank == config.coordinator_rank: + terminal: PreflightEvidence + if ( + local_code is not None + or result is None + or prepared_evidence is None + or terminal_identity is None + ): + deadline = time.monotonic() + config.teardown_timeout_seconds + receipts: tuple[tuple[StageMessage, ...], tuple[StageMessage, ...]] | None = None + while time.monotonic() < deadline: + try: + receipts = _read_terminal_receipts( + attempt, + config=config, + identity_digest=lifecycle_identity_digest, + ) + break + except FileNotFoundError: + time.sleep(0.01) + except Exception: + break + terminal_code = local_code or "LMS_COLLECTIVE_FAILED" + if receipts is None: + terminal_code = "LMS_CLEANUP_INCOMPLETE" + else: + _aborting, completed = receipts + terminal_code = _select_failure_code( + {item.error_code for item in completed if item.error_code} + ) + if any(item.stage is RuntimeStage.QUARANTINED for item in completed): + terminal_code = "LMS_CLEANUP_INCOMPLETE" + terminal = PreflightEvidence.failure( + run_id=config.run_id, + config_digest=config.digest, + code=terminal_code, + world_size=config.world_size, + evidence_tier=config.evidence_tier, + ) + else: + deadline = time.monotonic() + config.teardown_timeout_seconds + all_acknowledged = False + while time.monotonic() < deadline: + try: + acknowledgements = tuple( + read_stage_message( + result.attempt_path / f".rank-{rank}.teardown.stage.json" + ) + for rank in range(config.world_size) + ) + except FileNotFoundError: + time.sleep(0.01) + continue + except Exception: + break + try: + _validate_stage_messages( + acknowledgements, + run_id=config.run_id, + world_size=config.world_size, + stage=RuntimeStage.PREFLIGHTED, + sequence=2, + identity_digest=terminal_identity, + vote=Vote.COMMITTED, + ) + all_acknowledged = True + except RuntimeContractError: + all_acknowledged = False + break + terminal = ( + prepared_evidence + if all_acknowledged + else PreflightEvidence.failure( + run_id=config.run_id, + config_digest=config.digest, + code="LMS_CLEANUP_INCOMPLETE", + world_size=config.world_size, + evidence_tier=config.evidence_tier, + ) + ) + _write_terminal_evidence(config.evidence_path, terminal) + + terminal = _wait_for_evidence( + config.evidence_path, + timeout_seconds=min(3600, config.teardown_timeout_seconds * 2 + 1), + ) + if terminal.result != "preflighted" or local_code is not None: + code = terminal.error_code or local_code or "LMS_LIFECYCLE_INVALID" + raise RuntimeContractError(code, "distributed preflight failed") from None + return terminal + + +def _require_expected(records: tuple[RankAttestation, ...], field: str, expected: object) -> None: + if any(getattr(record, field) != expected for record in records): + codes = { + "storage_digest": "LMS_STORAGE_PROFILE_MISMATCH", + "network_interface_digest": "LMS_NETWORK_PROFILE_DENIED", + "device_kind": "LMS_RUNTIME_PROFILE_MISMATCH", + "device_config_digest": "LMS_RUNTIME_PROFILE_MISMATCH", + "software_digest": "LMS_RUNTIME_PROFILE_MISMATCH", + } + raise RuntimeContractError( + codes.get(field, "LMS_IDENTITY_MISMATCH"), + f"rank {field} values disagree with the expected identity", + ) + + +def validate_attestations( + records: tuple[RankAttestation, ...], + *, + world_size: int, + tensor_parallel_size: int, + dimension_divisors: tuple[int, ...], + expected_device_kind: str, + expected_device_config_digest: str, + expected_software_digest: str, + expected_source_digest: str, + expected_model_digest: str, + expected_tokenizer_digest: str, + expected_config_digest: str, + expected_commit_sha: str, + expected_code_digest: str, + expected_placement_plan_digest: str, + expected_storage_digest: str, + expected_network_interface_digest: str, + local_world_size: int, + min_free_device_memory_bytes: int, + min_free_host_memory_bytes: int, + min_free_staging_bytes: int, +) -> None: + """Require one complete, homogeneous, capacity-admitted fixed world.""" + if len(records) != world_size: + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", + "preflight requires exactly one attestation per rank", + ) + assert_rank_order(records, world_size=world_size) + if tensor_parallel_size != world_size: + raise RuntimeContractError( + "LMS_TOPOLOGY_UNSUPPORTED", + "tensor_parallel_size must equal the fixed world_size", + ) + if any(divisor % tensor_parallel_size for divisor in dimension_divisors): + raise RuntimeContractError( + "LMS_TOPOLOGY_UNSUPPORTED", + "each configured dimension divisor must divide by the TP world", + ) + if len({record.device_digest for record in records}) != world_size: + raise RuntimeContractError( + "LMS_RANK_DEVICE_CONFLICT", "global device identities must be unique" + ) + if len({record.device_profile_digest for record in records}) != 1: + raise RuntimeContractError("LMS_RUNTIME_PROFILE_MISMATCH", "rank device profiles disagree") + if len({(record.host_digest, record.local_rank) for record in records}) != world_size: + raise RuntimeContractError( + "LMS_RANK_DEVICE_CONFLICT", + "local ranks must be unique within each host", + ) + group_count = world_size // local_world_size + if {record.group_rank for record in records} != set(range(group_count)): + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", "group ranks do not cover the fixed host groups" + ) + host_groups: set[str] = set() + for group_rank in range(group_count): + members = tuple(record for record in records if record.group_rank == group_rank) + if ( + len(members) != local_world_size + or {record.local_rank for record in members} != set(range(local_world_size)) + or {record.rank for record in members} + != set( + range( + group_rank * local_world_size, + (group_rank + 1) * local_world_size, + ) + ) + or len({record.host_digest for record in members}) != 1 + ): + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", + "local ranks do not match the fixed host-group mapping", + ) + host_groups.add(members[0].host_digest) + if len(host_groups) != group_count: + raise RuntimeContractError( + "LMS_MEMBERSHIP_INVALID", "one host identity spans multiple fixed groups" + ) + if len({record.software_digest for record in records}) != 1: + raise RuntimeContractError( + "LMS_RUNTIME_PROFILE_MISMATCH", "rank software identities disagree" + ) + if len({record.storage_digest for record in records}) != 1: + raise RuntimeContractError( + "LMS_STORAGE_PROFILE_MISMATCH", "rank storage identities disagree" + ) + expectations = { + "world_size": world_size, + "device_kind": expected_device_kind, + "device_config_digest": expected_device_config_digest, + "software_digest": expected_software_digest, + "source_digest": expected_source_digest, + "model_digest": expected_model_digest, + "tokenizer_digest": expected_tokenizer_digest, + "config_digest": expected_config_digest, + "commit_sha": expected_commit_sha, + "code_digest": expected_code_digest, + "placement_plan_digest": expected_placement_plan_digest, + "storage_digest": expected_storage_digest, + "network_interface_digest": expected_network_interface_digest, + "local_world_size": local_world_size, + } + for field, expected in expectations.items(): + _require_expected(records, field, expected) + if any(record.free_device_memory_bytes < min_free_device_memory_bytes for record in records): + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "device memory headroom is below the configured floor", + ) + if any(record.free_host_memory_bytes < min_free_host_memory_bytes for record in records): + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "host memory headroom is below the configured floor", + ) + if any(record.free_staging_bytes < min_free_staging_bytes for record in records): + raise RuntimeContractError( + "LMS_RESOURCE_ADMISSION_DENIED", + "staging headroom is below the configured floor", + ) diff --git a/obliteratus/lora_ablation.py b/obliteratus/lora_ablation.py index b84491d..952725a 100644 --- a/obliteratus/lora_ablation.py +++ b/obliteratus/lora_ablation.py @@ -5,7 +5,7 @@ LoRA adapters. This provides: 1. **Reversibility**: LoRA adapters can be removed to restore original model 2. **Composability**: Multiple ablation adapters can be stacked/blended -3. **PEFT compatibility**: Output adapters work with standard HuggingFace PEFT +3. **PEFT compatibility**: Exact-identity exports use standard HuggingFace PEFT Inspired by Heretic (p-e-w, 2025) which pioneered LoRA-mediated ablation. OBLITERATUS extends this with: @@ -33,12 +33,27 @@ References: from __future__ import annotations +import json import logging +import math +import os +import re +from dataclasses import dataclass +from hashlib import sha256 from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable import torch import torch.nn as nn +from safetensors.torch import load_file, save_file + +from obliteratus.checkpoint_provenance import ( + AdapterIdentity, + ArtifactIdentity, + ProvenanceRecord, + verify_provenance_record, +) +from obliteratus.persistence_contracts import atomic_checkpoint_directory if TYPE_CHECKING: from obliteratus.abliterate import AbliterationPipeline @@ -52,6 +67,89 @@ _LORA_TARGETS = [ "gate", "router", ] +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_IMMUTABLE_REVISION = re.compile(r"^[0-9a-f]{40,64}$") +_MODULE_PATH = re.compile(r"^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+)*$") +_PUBLIC_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,511}$") +_PUBLIC_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,511}$") +_SECRET_VALUE = re.compile(r"(?i)(?:hf_[a-z0-9]{12,}|bearer\s+[a-z0-9._~+/-]{12,})") + + +@dataclass(frozen=True) +class BaseModelIdentity: + """Exact base and tokenizer identity required for a canonical PEFT claim.""" + + repo_id: str + revision: str + weights_digest: str + tokenizer_digest: str + vocab_size: int + architecture: str + tied_embeddings: bool + + def __post_init__(self) -> None: + if ( + not isinstance(self.repo_id, str) + or not self.repo_id + or len(self.repo_id) > 512 + or not _PUBLIC_REPO_ID.fullmatch(self.repo_id) + or Path(self.repo_id).is_absolute() + or ".." in self.repo_id.split("/") + or _SECRET_VALUE.search(self.repo_id) + ): + raise ValueError("base repository identity is invalid") + if not _IMMUTABLE_REVISION.fullmatch(self.revision): + raise ValueError("base revision must be an immutable commit") + for value, field in ( + (self.weights_digest, "base weights digest"), + (self.tokenizer_digest, "tokenizer digest"), + ): + if not _DIGEST.fullmatch(value): + raise ValueError(f"{field} is invalid") + if type(self.vocab_size) is not int or self.vocab_size <= 0: + raise ValueError("base vocabulary size is invalid") + if ( + not isinstance(self.architecture, str) + or not self.architecture + or len(self.architecture) > 512 + or not _PUBLIC_NAME.fullmatch(self.architecture) + or Path(self.architecture).is_absolute() + or _SECRET_VALUE.search(self.architecture) + ): + raise ValueError("base architecture is invalid") + if type(self.tied_embeddings) is not bool: + raise ValueError("base tied-embedding declaration is invalid") + + def to_dict(self) -> dict[str, Any]: + return { + "repo_id": self.repo_id, + "revision": self.revision, + "weights_digest": self.weights_digest, + "tokenizer_digest": self.tokenizer_digest, + "vocab_size": self.vocab_size, + "architecture": self.architecture, + "tied_embeddings": self.tied_embeddings, + } + + def to_artifact_identity(self) -> ArtifactIdentity: + return ArtifactIdentity("hub", self.repo_id, self.revision, self.weights_digest) + + def tokenizer_artifact_identity(self) -> ArtifactIdentity: + return ArtifactIdentity( + "hub", + f"{self.repo_id}#tokenizer", + self.revision, + self.tokenizer_digest, + ) + + +@dataclass(frozen=True) +class AdapterArtifact: + artifact_id: str + weights_path: Path + config_path: Path + provenance_path: Path + def compute_lora_adapters( pipeline: AbliterationPipeline, @@ -90,6 +188,11 @@ def compute_lora_adapters( layers = get_layer_modules(pipeline.handle) arch = pipeline.handle.architecture + module_names = { + id(module): name + for name, module in pipeline.handle.model.named_modules() + if name + } adapters: dict[str, tuple[torch.Tensor, torch.Tensor]] = {} for idx in pipeline._strong_layers: @@ -189,7 +292,7 @@ def compute_lora_adapters( else: continue - key = f"layer.{idx}.{module_label}.{name}" + key = module_names.get(id(proj), f"layer.{idx}.{module_label}.{name}") adapters[key] = (lora_B.half(), lora_A.half()) pipeline.log(f"Computed {len(adapters)} LoRA adapter pairs (rank={rank})") @@ -217,24 +320,25 @@ def apply_lora_adapters( layers = get_layer_modules(pipeline.handle) arch = pipeline.handle.architecture + named_modules = dict(pipeline.handle.model.named_modules()) applied = 0 for key, (lora_B, lora_A) in adapters.items(): - parts = key.split(".") - if len(parts) != 4: - continue - _, idx_str, module_label, weight_name = parts - idx = int(idx_str) - - try: - if module_label == "attn": - module = get_attention_module(layers[idx], arch) - else: - module = get_ffn_module(layers[idx], arch) - except (AttributeError, RuntimeError): - continue - - proj = getattr(module, weight_name, None) + proj = named_modules.get(key) + if proj is None: + parts = key.split(".") + if len(parts) != 4 or parts[0] != "layer" or not parts[1].isdigit(): + continue + _, idx_str, module_label, weight_name = parts + idx = int(idx_str) + try: + if module_label == "attn": + module = get_attention_module(layers[idx], arch) + else: + module = get_ffn_module(layers[idx], arch) + except (AttributeError, RuntimeError): + continue + proj = getattr(module, weight_name, None) if proj is None or not hasattr(proj, "weight"): continue @@ -249,41 +353,576 @@ def apply_lora_adapters( pipeline.log(f"Applied {applied} LoRA adapters (merged into weights)") +def _json_bytes(value: object) -> bytes: + return ( + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=True, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _write_new(path: Path, payload: bytes) -> None: + with path.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + + +def _digest_bytes(payload: bytes) -> str: + return f"sha256:{sha256(payload).hexdigest()}" + + +def _digest_file(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return f"sha256:{digest.hexdigest()}" + + +def _prepare_adapter_state( + adapters: dict[str, tuple[torch.Tensor, torch.Tensor]], + *, + lora_alpha: int | None, +) -> tuple[dict[str, torch.Tensor], list[dict[str, Any]], int, int, float]: + if not adapters: + raise ValueError("canonical PEFT export requires at least one adapter") + ranks = set() + for module_name, pair in adapters.items(): + if not _MODULE_PATH.fullmatch(module_name): + raise ValueError(f"adapter module path is invalid: {module_name!r}") + if not isinstance(pair, tuple) or len(pair) != 2: + raise ValueError(f"adapter pair is invalid: {module_name}") + lora_b, lora_a = pair + if ( + not isinstance(lora_a, torch.Tensor) + or not isinstance(lora_b, torch.Tensor) + or lora_a.ndim != 2 + or lora_b.ndim != 2 + or lora_b.shape[1] != lora_a.shape[0] + or lora_a.dtype != lora_b.dtype + ): + raise ValueError(f"adapter tensor geometry is invalid: {module_name}") + ranks.add(int(lora_a.shape[0])) + if len(ranks) != 1 or next(iter(ranks)) <= 0: + raise ValueError("canonical PEFT export requires one positive rank") + rank = next(iter(ranks)) + alpha = rank if lora_alpha is None else lora_alpha + if type(alpha) is not int or alpha <= 0: + raise ValueError("LoRA alpha must be a positive integer") + scaling = alpha / rank + state: dict[str, torch.Tensor] = {} + key_map: list[dict[str, Any]] = [] + for module_name in sorted(adapters): + lora_b, lora_a = adapters[module_name] + a_key = f"base_model.model.{module_name}.lora_A.weight" + b_key = f"base_model.model.{module_name}.lora_B.weight" + state[a_key] = lora_a.detach().cpu().contiguous() + state[b_key] = lora_b.detach().cpu().contiguous() / scaling + key_map.append( + { + "module_name": module_name, + "target_module": module_name.rsplit(".", 1)[-1], + "lora_A_key": a_key, + "lora_B_key": b_key, + "lora_A_shape": list(lora_a.shape), + "lora_B_shape": list(lora_b.shape), + "rank": rank, + } + ) + return state, key_map, rank, alpha, scaling + + def save_lora_adapters( adapters: dict[str, tuple[torch.Tensor, torch.Tensor]], output_dir: str | Path, -): - """Save LoRA adapters to disk for later use. - - Saves as a simple dict of {key: (B, A)} tensors using torch.save. - Can be loaded and applied to the original model for reversible ablation. - """ + *, + base_model: BaseModelIdentity, + provenance_factory: Callable[[tuple[str, ...], AdapterIdentity], ProvenanceRecord], + lora_alpha: int | None = None, + lora_dropout: float = 0.0, + modules_to_save: tuple[str, ...] = (), + adapter_name: str = "default", +) -> AdapterArtifact: + """Atomically write a standard PEFT LoRA artifact with exact provenance.""" + if not isinstance(base_model, BaseModelIdentity): + raise ValueError("canonical PEFT export requires an exact base model identity") + if ( + isinstance(lora_dropout, bool) + or not isinstance(lora_dropout, (int, float)) + or not math.isfinite(lora_dropout) + or not 0 <= lora_dropout < 1 + ): + raise ValueError("LoRA dropout must be in [0, 1)") + if modules_to_save: + raise ValueError("modules_to_save is unsupported unless its tensors are supplied") + if ( + not isinstance(adapter_name, str) + or not adapter_name + or len(adapter_name) > 512 + or not _PUBLIC_NAME.fullmatch(adapter_name) + or Path(adapter_name).is_absolute() + or _SECRET_VALUE.search(adapter_name) + ): + raise ValueError("adapter name is invalid") + state, key_map, rank, alpha, scaling = _prepare_adapter_state( + adapters, + lora_alpha=lora_alpha, + ) output_path = Path(output_dir) - output_path.mkdir(parents=True, exist_ok=True) - - save_dict = {} - for key, (B, A) in adapters.items(): - save_dict[f"{key}.lora_B"] = B - save_dict[f"{key}.lora_A"] = A - - adapter_path = output_path / "abliteration_lora_adapters.pt" - torch.save(save_dict, adapter_path) - - # Also save adapter config for PEFT compatibility - import json + if ( + output_path.is_symlink() + or output_path.absolute() != output_path.resolve(strict=False) + ): + raise ValueError("adapter output directory must not be a symlink") + if output_path.exists() and (not output_path.is_dir() or any(output_path.iterdir())): + raise FileExistsError("adapter output directory must be absent or empty") config = { - "adapter_type": "obliteratus_abliteration_lora", - "n_adapters": len(adapters), - "target_modules": list(set( - k.split(".")[-1] for k in adapters - )), - "description": ( - "Reversible abliteration LoRA adapters generated by OBLITERATUS. " - "These adapters remove refusal directions from the model when merged." - ), + "base_model_name_or_path": base_model.repo_id, + "bias": "none", + "fan_in_fan_out": False, + "inference_mode": True, + "init_lora_weights": True, + "lora_alpha": alpha, + "lora_dropout": float(lora_dropout), + "modules_to_save": None, + "peft_type": "LORA", + "r": rank, + "revision": base_model.revision, + # Full paths prevent PEFT from creating unsupplied adapters on every + # module that happens to share a leaf name such as ``q_proj``. + "target_modules": sorted(item["module_name"] for item in key_map), + "task_type": "CAUSAL_LM", } - (output_path / "abliteration_lora_config.json").write_text( - json.dumps(config, indent=2) + config_payload = _json_bytes(config) + key_map_digest = _digest_bytes( + json.dumps(key_map, sort_keys=True, separators=(",", ":")).encode("utf-8") + ) + adapter_identity = AdapterIdentity( + adapter_type="lora", + base_model=base_model.to_artifact_identity(), + config_digest=_digest_bytes(config_payload), + key_map_digest=key_map_digest, + ) + card = ( + "---\n" + f"base_model: {base_model.repo_id}\n" + f"base_model_revision: {base_model.revision}\n" + "library_name: peft\n" + "tags:\n - peft\n - lora\n - obliteratus\n" + "---\n\n" + "# OBLITERATUS LoRA adapter\n\n" + "This is an unmerged PEFT LoRA artifact representing an OBLITERATUS surgery event.\n\n" + f"- Exact base: `{base_model.repo_id}@{base_model.revision}`\n" + f"- Base weights: `{base_model.weights_digest}`\n" + f"- Tokenizer: `{base_model.tokenizer_digest}`\n" + "- Scaling: standard PEFT `alpha / rank`; saved B values preserve the exact delta.\n" + ).encode("utf-8") + manifest = { + "schema_id": "obliteratus.peft-adapter-manifest", + "schema_version": "1.0.0", + "adapter_name": adapter_name, + "adapter_type": "lora", + "adapter_format_version": "peft-lora-v1", + "base_model": base_model.to_dict(), + "rank": rank, + "alpha": alpha, + "scaling": scaling, + "dropout": float(lora_dropout), + "bias": "none", + "modules_to_save": [], + "target_modules": config["target_modules"], + "tie_policy": "base_model_declared", + "merged": False, + "key_map_digest": key_map_digest, + "model_card_digest": _digest_bytes(card), + "key_map": key_map, + } + manifest_payload = _json_bytes(manifest) + artifact_id = "" + + def validate_staging(staging: Path) -> None: + expected = { + "README.md", + "adapter_config.json", + "adapter_manifest.json", + "adapter_model.safetensors", + "checkpoint-provenance.json", + } + paths = tuple(staging.iterdir()) + if {path.name for path in paths} != expected or any( + path.is_symlink() or not path.is_file() for path in paths + ): + raise ValueError("adapter artifact set is incomplete") + loaded = load_file(staging / "adapter_model.safetensors", device="cpu") + if set(loaded) != set(state): + raise ValueError("adapter tensor set does not match the manifest") + validate_adapter_base(staging, base_model) + + with atomic_checkpoint_directory(output_path, validate=validate_staging) as staging: + weights_path = staging / "adapter_model.safetensors" + config_path = staging / "adapter_config.json" + manifest_path = staging / "adapter_manifest.json" + provenance_path = staging / "checkpoint-provenance.json" + card_path = staging / "README.md" + save_file(dict(sorted(state.items())), weights_path) + _write_new(config_path, config_payload) + _write_new(manifest_path, manifest_payload) + _write_new(card_path, card) + output_digests = tuple( + sorted( + ( + _digest_file(weights_path), + _digest_file(config_path), + _digest_file(manifest_path), + _digest_file(card_path), + ) + ) + ) + provenance = provenance_factory(output_digests, adapter_identity) + provenance_record = provenance.to_dict() + if ( + provenance_record.get("base_model") != base_model.to_artifact_identity().to_dict() + or provenance_record.get("adapter") != adapter_identity.to_dict() + or sorted(provenance_record.get("output_digests", [])) != list(output_digests) + ): + raise ValueError("adapter provenance does not match the exact exported artifact") + artifact_id = provenance.artifact_id + _write_new(provenance_path, provenance.to_json().encode("utf-8")) + return AdapterArtifact( + artifact_id=artifact_id, + weights_path=output_path / "adapter_model.safetensors", + config_path=output_path / "adapter_config.json", + provenance_path=output_path / "checkpoint-provenance.json", ) - return adapter_path + +def _read_json_object(path: Path, detail: str) -> dict[str, Any]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 << 20: + raise ValueError(detail) + try: + value = json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_reject_duplicate_json_pairs, + ) + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + raise ValueError(detail) from error + if not isinstance(value, dict): + raise ValueError(detail) + return value + + +def _reject_duplicate_json_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _canonical_object_digest(value: object) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + return _digest_bytes(payload) + + +def _validate_adapter_integrity( + root: Path, + config: dict[str, Any], + manifest: dict[str, Any], + provenance: dict[str, Any], +) -> None: + weights = root / "adapter_model.safetensors" + artifact_paths = ( + root / "README.md", + root / "adapter_config.json", + root / "adapter_manifest.json", + weights, + root / "checkpoint-provenance.json", + ) + if any(path.is_symlink() or not path.is_file() for path in artifact_paths): + raise ValueError("adapter_artifact_file_invalid") + if weights.stat().st_size == 0: + raise ValueError("adapter_weights_invalid") + try: + verified_provenance = verify_provenance_record(provenance) + except (TypeError, ValueError) as error: + if "record digest mismatch" in str(error): + raise ValueError("adapter_provenance_digest_mismatch") from error + if "artifact ID mismatch" in str(error): + raise ValueError("adapter_artifact_identity_mismatch") from error + raise ValueError("adapter_provenance_invalid") from error + if verified_provenance != provenance: + raise ValueError("adapter_provenance_invalid") + expected_outputs = provenance.get("output_digests") + if ( + not isinstance(expected_outputs, list) + or any(not isinstance(value, str) or not _DIGEST.fullmatch(value) for value in expected_outputs) + or sorted(expected_outputs) + != sorted( + _digest_file(path) + for path in ( + root / "README.md", + root / "adapter_config.json", + root / "adapter_manifest.json", + weights, + ) + ) + ): + raise ValueError("adapter_artifact_digest_mismatch") + record_digest = provenance.get("record_digest") + record_core = {key: value for key, value in provenance.items() if key != "record_digest"} + if record_digest != _canonical_object_digest(record_core): + raise ValueError("adapter_provenance_digest_mismatch") + artifact_id = provenance.get("artifact_id") + identity_core = {key: value for key, value in record_core.items() if key != "artifact_id"} + expected_artifact_id = _canonical_object_digest(identity_core).replace( + "sha256:", + "artifact-sha256:", + 1, + ) + if artifact_id != expected_artifact_id: + raise ValueError("adapter_artifact_identity_mismatch") + key_map = manifest.get("key_map") + adapter = provenance.get("adapter") + manifest_base = manifest.get("base_model") + if ( + not isinstance(key_map, list) + or not isinstance(adapter, dict) + or not isinstance(manifest_base, dict) + ): + raise ValueError("adapter_manifest_integrity_invalid") + if any( + not isinstance(item, dict) + or not isinstance(item.get("module_name"), str) + or not _MODULE_PATH.fullmatch(item["module_name"]) + for item in key_map + ): + raise ValueError("adapter_manifest_integrity_invalid") + key_map_digest = _canonical_object_digest(key_map) + target_modules = [ + item.get("module_name") for item in key_map if isinstance(item, dict) + ] + rank = manifest.get("rank") + alpha = manifest.get("alpha") + scaling = manifest.get("scaling") + if ( + manifest.get("key_map_digest") != key_map_digest + or manifest.get("model_card_digest") != _digest_file(root / "README.md") + or adapter.get("key_map_digest") != key_map_digest + or adapter.get("config_digest") != _digest_file(root / "adapter_config.json") + or config.get("base_model_name_or_path") != manifest_base.get("repo_id") + or config.get("revision") != manifest_base.get("revision") + or manifest.get("schema_id") != "obliteratus.peft-adapter-manifest" + or manifest.get("schema_version") != "1.0.0" + or config.get("peft_type") != "LORA" + or config.get("target_modules") != sorted(target_modules) + or manifest.get("target_modules") != sorted(target_modules) + or type(rank) is not int + or rank <= 0 + or config.get("r") != rank + or type(alpha) is not int + or alpha <= 0 + or config.get("lora_alpha") != alpha + or not isinstance(scaling, (int, float)) + or isinstance(scaling, bool) + or not math.isfinite(scaling) + or scaling != alpha / rank + or config.get("lora_dropout") != manifest.get("dropout") + or config.get("bias") != manifest.get("bias") + ): + raise ValueError("adapter_manifest_integrity_invalid") + + +def validate_adapter_base( + adapter_dir: str | Path, + base_model: BaseModelIdentity, +) -> dict[str, Any]: + """Fail before loading weights when exact base/tokenizer facts do not match.""" + root = Path(adapter_dir) + if ( + root.is_symlink() + or not root.is_dir() + or root.absolute() != root.resolve(strict=True) + ): + raise ValueError("adapter_directory_invalid") + config = _read_json_object(root / "adapter_config.json", "adapter_config_invalid") + manifest = _read_json_object(root / "adapter_manifest.json", "adapter_manifest_invalid") + provenance = _read_json_object( + root / "checkpoint-provenance.json", + "adapter_provenance_invalid", + ) + _validate_adapter_integrity(root, config, manifest, provenance) + checks = ( + (config.get("base_model_name_or_path"), base_model.repo_id, "base_model_identity_mismatch"), + (config.get("revision"), base_model.revision, "base_model_revision_mismatch"), + ( + manifest.get("base_model", {}).get("weights_digest"), + base_model.weights_digest, + "base_model_digest_mismatch", + ), + ( + manifest.get("base_model", {}).get("tokenizer_digest"), + base_model.tokenizer_digest, + "tokenizer_digest_mismatch", + ), + ( + manifest.get("base_model", {}).get("vocab_size"), + base_model.vocab_size, + "vocab_size_mismatch", + ), + ( + manifest.get("base_model", {}).get("architecture"), + base_model.architecture, + "architecture_mismatch", + ), + ( + manifest.get("base_model", {}).get("tied_embeddings"), + base_model.tied_embeddings, + "tied_embeddings_mismatch", + ), + ) + for actual, expected, detail in checks: + if actual != expected: + raise ValueError(detail) + if provenance.get("base_model") != base_model.to_artifact_identity().to_dict(): + raise ValueError("adapter_provenance_base_mismatch") + return manifest + + +def load_lora_adapters( + adapter_dir: str | Path, + *, + base_model: BaseModelIdentity, +) -> dict[str, tuple[torch.Tensor, torch.Tensor]]: + """Load canonical safetensors only; this path never accepts pickle artifacts.""" + root = Path(adapter_dir) + manifest = validate_adapter_base(root, base_model) + state = load_file(root / "adapter_model.safetensors", device="cpu") + validate_adapter_base(root, base_model) + scaling = manifest.get("scaling") + if ( + isinstance(scaling, bool) + or not isinstance(scaling, (int, float)) + or not math.isfinite(scaling) + or scaling <= 0 + ): + raise ValueError("adapter_scaling_invalid") + result: dict[str, tuple[torch.Tensor, torch.Tensor]] = {} + expected_keys: set[str] = set() + key_map = manifest.get("key_map") + if not isinstance(key_map, list): + raise ValueError("adapter_key_map_invalid") + for item in key_map: + if not isinstance(item, dict): + raise ValueError("adapter_key_map_invalid") + module_name = item.get("module_name") + a_key = item.get("lora_A_key") + b_key = item.get("lora_B_key") + if ( + not isinstance(module_name, str) + or not isinstance(a_key, str) + or not isinstance(b_key, str) + or not _MODULE_PATH.fullmatch(module_name) + or module_name in result + or a_key == b_key + or a_key in expected_keys + or b_key in expected_keys + or a_key not in state + or b_key not in state + ): + raise ValueError("adapter_key_map_invalid") + lora_a = state[a_key] + lora_b = state[b_key] + if ( + lora_a.ndim != 2 + or lora_b.ndim != 2 + or lora_b.shape[1] != lora_a.shape[0] + or lora_a.dtype != lora_b.dtype + or item.get("lora_A_shape") != list(lora_a.shape) + or item.get("lora_B_shape") != list(lora_b.shape) + or item.get("rank") != lora_a.shape[0] + ): + raise ValueError("adapter_tensor_geometry_mismatch") + expected_keys.update((a_key, b_key)) + result[module_name] = (lora_b * scaling, lora_a) + if expected_keys != set(state): + raise ValueError("adapter_tensor_set_mismatch") + return result + + +def save_unsupported_obliteratus_adapters( + adapters: dict[str, tuple[torch.Tensor, torch.Tensor]], + output_dir: str | Path, + *, + reason: str, +) -> Path: + """Persist safe legacy tensors without making a PEFT compatibility claim.""" + if not adapters: + raise ValueError("legacy adapter export requires at least one adapter") + if ( + not isinstance(reason, str) + or not reason + or len(reason) > 1024 + or _SECRET_VALUE.search(reason) + or Path(reason).is_absolute() + ): + raise ValueError("legacy adapter reason is invalid") + output = Path(output_dir) + if output.is_symlink() or output.absolute() != output.resolve(strict=False): + raise ValueError("adapter output directory must not be a symlink") + output.mkdir(parents=True, exist_ok=True) + state = {} + key_map = [] + for module_name in sorted(adapters): + lora_b, lora_a = adapters[module_name] + b_key = f"{module_name}.B" + a_key = f"{module_name}.A" + state[b_key] = lora_b.detach().cpu().contiguous() + state[a_key] = lora_a.detach().cpu().contiguous() + key_map.append({"module_name": module_name, "A": a_key, "B": b_key}) + path = output / "obliteratus_unsupported_adapter.safetensors" + config_path = output / "obliteratus_unsupported_adapter.json" + if path.exists() or config_path.exists(): + raise FileExistsError("unsupported adapter artifact already exists") + save_file(dict(sorted(state.items())), path) + _write_new( + config_path, + _json_bytes( + { + "schema_id": "obliteratus.unsupported-adapter", + "schema_version": "1.0.0", + "support_status": "unsupported_legacy", + "safe_serialization": True, + "peft_compatible": False, + "reason": reason, + "key_map": key_map, + } + ), + ) + return path + + +def save_legacy_pickle_adapters_trusted( + adapters: dict[str, tuple[torch.Tensor, torch.Tensor]], + output_dir: str | Path, + *, + allow_pickle: bool, +) -> Path: + """Write the historical pickle shape only behind an explicit trust gate.""" + if allow_pickle is not True: + raise PermissionError("allow_pickle=True is required for unsafe legacy export") + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + path = output / "obliteratus_legacy_adapter_unsafe.pt" + state = { + f"{module_name}.{suffix}": tensor + for module_name, (lora_b, lora_a) in sorted(adapters.items()) + for suffix, tensor in (("lora_B", lora_b), ("lora_A", lora_a)) + } + torch.save(state, path) + return path diff --git a/obliteratus/run_archive.py b/obliteratus/run_archive.py index 2bcdae7..50886bc 100644 --- a/obliteratus/run_archive.py +++ b/obliteratus/run_archive.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Sequence +from obliteratus.checkpoint_provenance import verify_provenance_record from obliteratus.gpu_lifecycle import AdmissionError, MemoryUsage, from_environment @@ -285,6 +286,36 @@ class RunArchive: stream.write(str(message).rstrip("\n") + "\n") stream.flush() + def attach_checkpoint_provenance(self, run_id: str, provenance: Any) -> dict[str, Any]: + """Attach one canonical artifact identity to the durable run archive.""" + + try: + record = verify_provenance_record(json.loads(provenance.to_json())) + except (AttributeError, TypeError, json.JSONDecodeError, ValueError) as error: + raise ValueError("checkpoint provenance must be a canonical provenance record") from error + artifact_id = record.get("artifact_id") + if ( + not isinstance(artifact_id, str) + or getattr(provenance, "artifact_id", None) != artifact_id + ): + raise ValueError("checkpoint provenance contract is invalid") + manifest = self._load(run_id) + existing = manifest.get("artifact_id") + if existing is not None and existing != artifact_id: + raise ValueError("run archive already names a different artifact identity") + relative_path = "checkpoint-provenance.json" + path = self._run_dir(run_id) / relative_path + _atomic_json(path, record) + manifest["artifact_id"] = artifact_id + manifest["checkpoint_provenance"] = { + "artifact_id": artifact_id, + "path": relative_path, + "sha256": f"sha256:{_sha256(path)}", + } + self._save(manifest) + self._event(run_id, "checkpoint_provenance_attached", artifact_id=artifact_id) + return manifest + def record_dataset( self, run_id: str, diff --git a/pyproject.toml b/pyproject.toml index eaa2180..0bfa78c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ dev = [ "build==1.2.2.post1", "hypothesis==6.165.3", + "jsonschema==4.25.1", "mypy==1.13.0", "pytest==9.1.1", "pytest-cov==7.1.0", @@ -118,6 +119,9 @@ markers = [ "operator_ui: tests that require the optional UI runtime", ] +[tool.coverage.run] +patch = ["subprocess"] + [tool.mutmut] source_paths = ["obliteratus/", "scripts/"] only_mutate = [ diff --git a/scripts/check_checkpoint_docs.py b/scripts/check_checkpoint_docs.py new file mode 100644 index 0000000..ae9180a --- /dev/null +++ b/scripts/check_checkpoint_docs.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""Validate checkpoint documentation and support claims without network or model work.""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +import re +import shlex +from pathlib import Path +from typing import Any +from urllib.parse import unquote + + +ROOT = Path(__file__).resolve().parents[1] +DOCS_DIR = ROOT / "docs/checkpoints" +MATRIX_PATH = DOCS_DIR / "support-matrix-v1.json" +SCHEMA_PATH = DOCS_DIR / "schemas/support-matrix-v1.schema.json" +STATUS_VOCABULARY = ["supported", "conditional", "deferred", "out_of_scope"] +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +ROW_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$") +MARKDOWN_LINK = re.compile(r"(? dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f"cannot read {label}: {exc}") + return {} + if not isinstance(value, dict): + errors.append(f"{label} root must be an object") + return {} + return value + + +def _exact_keys( + value: object, + *, + required: set[str], + label: str, + errors: list[str], +) -> dict[str, Any]: + if not isinstance(value, dict): + errors.append(f"{label} must be an object") + return {} + keys = set(value) + missing = sorted(required - keys) + unknown = sorted(keys - required) + if missing: + errors.append(f"{label} is missing fields: {', '.join(missing)}") + if unknown: + errors.append(f"{label} has unknown fields: {', '.join(unknown)}") + return value + + +def _nonempty_string(value: object, label: str, errors: list[str]) -> bool: + if not isinstance(value, str) or not value.strip(): + errors.append(f"{label} must be a non-empty string") + return False + return True + + +def _string_list( + value: object, + *, + label: str, + errors: list[str], + nonempty: bool = False, +) -> list[str]: + if not isinstance(value, list) or (nonempty and not value): + qualifier = "non-empty " if nonempty else "" + errors.append(f"{label} must be a {qualifier}array") + return [] + result: list[str] = [] + for index, item in enumerate(value): + if _nonempty_string(item, f"{label}[{index}]", errors): + result.append(item) + return result + + +def validate_matrix(matrix_path: Path = MATRIX_PATH, schema_path: Path = SCHEMA_PATH) -> list[str]: + """Validate the strict support-matrix shape and evidence promotion gate.""" + + errors: list[str] = [] + schema = _load_object(schema_path, "support-matrix schema", errors) + matrix = _load_object(matrix_path, "support matrix", errors) + if errors: + return errors + + schema_properties = schema.get("properties") + schema_required = schema.get("required") + if not isinstance(schema_properties, dict) or not isinstance(schema_required, list): + return ["support-matrix schema must declare root properties and required fields"] + root = _exact_keys( + matrix, + required=set(schema_required), + label="support matrix", + errors=errors, + ) + if set(schema_properties) != set(schema_required): + errors.append("support-matrix schema root properties must all be required") + if root.get("schema_id") != "obliteratus.checkpoint-support-matrix": + errors.append("support matrix has an unsupported schema_id") + if root.get("schema_version") != "1.0.0": + errors.append("support matrix has an unsupported schema_version") + if not isinstance(root.get("generated_from"), str) or not SHA_PATTERN.fullmatch( + root["generated_from"], + ): + errors.append("support matrix generated_from must be a 40-character commit SHA") + if root.get("status_vocabulary") != STATUS_VOCABULARY: + errors.append("support matrix status_vocabulary must match the canonical ordered list") + + definitions = schema.get("$defs") + if not isinstance(definitions, dict) or not isinstance(definitions.get("row"), dict): + errors.append("support-matrix schema must declare the row definition") + return errors + row_schema = definitions["row"] + row_required = row_schema.get("required") + row_properties = row_schema.get("properties") + if not isinstance(row_required, list) or not isinstance(row_properties, dict): + errors.append("support-matrix row schema must declare properties and required fields") + return errors + if set(row_required) != set(row_properties): + errors.append("support-matrix row properties must all be required") + + rows = root.get("rows") + if not isinstance(rows, list) or not rows: + errors.append("support matrix rows must be a non-empty array") + return errors + seen_ids: set[str] = set() + capability_names = { + "detect", + "safe_inspect", + "trusted_inspect", + "weights_canonicalize", + "topology_reshard", + "surgery", + "exact_resume", + "live_multi_node", + } + evidence_names = { + "references", + "candidate_commit", + "fixture_digest", + "environment", + "topology", + "retained_result", + } + for index, candidate in enumerate(rows): + label = f"support matrix row {index}" + row = _exact_keys(candidate, required=set(row_required), label=label, errors=errors) + row_id = row.get("id") + if not isinstance(row_id, str) or not ROW_ID_PATTERN.fullmatch(row_id): + errors.append(f"{label} has an invalid id") + row_id = str(index) + elif row_id in seen_ids: + errors.append(f"support matrix has duplicate row id: {row_id}") + seen_ids.add(row_id) + label = f"support matrix row {row_id}" + + for field in ("subject", "format", "model_mapping", "safety_level"): + _nonempty_string(row.get(field), f"{label}.{field}", errors) + for field in ("producer_versions", "state_scopes", "optional_extras", "limits"): + _string_list( + row.get(field), + label=f"{label}.{field}", + errors=errors, + nonempty=field == "limits", + ) + for field in ("adapter", "canonical_output"): + if row.get(field) is not None and not isinstance(row.get(field), str): + errors.append(f"{label}.{field} must be a string or null") + + capabilities = _exact_keys( + row.get("capabilities"), + required=capability_names, + label=f"{label}.capabilities", + errors=errors, + ) + supported = False + for name in sorted(capability_names): + status = _exact_keys( + capabilities.get(name), + required={"value", "basis"}, + label=f"{label}.capabilities.{name}", + errors=errors, + ) + if status.get("value") not in STATUS_VOCABULARY: + errors.append(f"{label}.capabilities.{name}.value is not canonical") + supported = supported or status.get("value") == "supported" + _nonempty_string(status.get("basis"), f"{label}.capabilities.{name}.basis", errors) + + evidence = _exact_keys( + row.get("evidence"), + required=evidence_names, + label=f"{label}.evidence", + errors=errors, + ) + references = _string_list( + evidence.get("references"), + label=f"{label}.evidence.references", + errors=errors, + nonempty=True, + ) + candidate_commit = evidence.get("candidate_commit") + if candidate_commit is not None and ( + not isinstance(candidate_commit, str) or not SHA_PATTERN.fullmatch(candidate_commit) + ): + errors.append(f"{label}.evidence.candidate_commit must be a commit SHA or null") + fixture_digest = evidence.get("fixture_digest") + if fixture_digest is not None and ( + not isinstance(fixture_digest, str) or not DIGEST_PATTERN.fullmatch(fixture_digest) + ): + errors.append(f"{label}.evidence.fixture_digest must be a sha256 digest or null") + for field in ("environment", "topology", "retained_result"): + if evidence.get(field) is not None and not isinstance(evidence.get(field), str): + errors.append(f"{label}.evidence.{field} must be a string or null") + + if supported: + exact_versions = row.get("producer_versions") + vague = re.compile(r"\b(?:compatible|varies|unknown|latest|planned)\b", re.IGNORECASE) + if not isinstance(exact_versions, list) or not exact_versions or any( + not isinstance(version, str) or vague.search(version) for version in exact_versions + ): + errors.append(f"{label} supported claims require exact producer versions") + required_evidence = { + "candidate_commit": candidate_commit, + "fixture_digest": fixture_digest, + "environment": evidence.get("environment"), + "topology": evidence.get("topology"), + "retained_result": evidence.get("retained_result"), + } + for field, value in required_evidence.items(): + if not isinstance(value, str) or not value.strip(): + errors.append(f"{label} supported claims require evidence.{field}") + if not references: + errors.append(f"{label} supported claims require evidence references") + if not row.get("limits"): + errors.append(f"{label} supported claims require limitations") + return errors + + +def _heading_slug(value: str) -> str: + value = re.sub(r"<[^>]+>", "", value).strip().lower() + value = re.sub(r"[^\w\- ]", "", value, flags=re.UNICODE) + return re.sub(r"[ ]+", "-", value) + + +def _anchors(path: Path) -> set[str]: + anchors: set[str] = set() + counts: dict[str, int] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.match(r"^#{1,6}\s+(.+?)\s*#*\s*$", line) + if not match: + continue + base = _heading_slug(match.group(1)) + count = counts.get(base, 0) + counts[base] = count + 1 + anchors.add(base if count == 0 else f"{base}-{count}") + return anchors + + +def validate_local_links(docs_dir: Path = DOCS_DIR, root: Path = ROOT) -> list[str]: + """Validate repository-local Markdown links and heading anchors.""" + + errors: list[str] = [] + for document in sorted(docs_dir.glob("*.md")): + text = document.read_text(encoding="utf-8") + for raw_target in MARKDOWN_LINK.findall(text): + target = raw_target.strip().split(maxsplit=1)[0].strip("<>") + if re.match(r"^[a-z][a-z0-9+.-]*:", target, re.IGNORECASE): + continue + path_text, separator, fragment = target.partition("#") + resolved = (document.parent / unquote(path_text)).resolve() if path_text else document + try: + resolved.relative_to(root.resolve()) + except ValueError: + errors.append(f"{document.relative_to(root)} link escapes the repository: {target}") + continue + if not resolved.is_file(): + errors.append(f"{document.relative_to(root)} has missing local link: {target}") + continue + if separator: + if resolved.suffix.lower() != ".md": + errors.append(f"{document.relative_to(root)} anchors non-Markdown target: {target}") + elif unquote(fragment).lower() not in _anchors(resolved): + errors.append(f"{document.relative_to(root)} has missing anchor: {target}") + return errors + + +def documented_cli_commands(docs_dir: Path = DOCS_DIR) -> list[tuple[Path, str]]: + """Return actual command examples, excluding prose about planned option names.""" + + commands: list[tuple[Path, str]] = [] + for document in sorted(docs_dir.glob("*.md")): + text = document.read_text(encoding="utf-8") + candidates = INLINE_CODE.findall(text) + for block in FENCED_BLOCK.findall(text): + candidates.extend(line.strip().removeprefix("$ ") for line in block.splitlines()) + for candidate in candidates: + try: + parts = shlex.split(candidate) + except ValueError: + continue + if not parts: + continue + is_module = ( + len(parts) >= 3 + and re.fullmatch(r"python(?:3(?:\.\d+)?)?", Path(parts[0]).name) + and parts[1:3] == ["-m", "obliteratus"] + ) + if parts[0] == "obliteratus" or is_module: + commands.append((document, candidate)) + return commands + + +class _ParserCompleted(Exception): + """Stop CLI execution immediately after argparse accepts an example.""" + + +def _parse_without_dispatch(argv: list[str]) -> None: + from obliteratus import cli + + original = argparse.ArgumentParser.parse_args + + def stop_after_parse(parser, args=None, namespace=None): + original(parser, args, namespace) + raise _ParserCompleted + + argparse.ArgumentParser.parse_args = stop_after_parse + try: + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + try: + cli.main(argv) + except _ParserCompleted: + return + except SystemExit as exc: + if exc.code in (None, 0): + return + raise ValueError(f"parser exited with status {exc.code}") from exc + raise ValueError("CLI returned before the parser boundary was captured") + finally: + argparse.ArgumentParser.parse_args = original + + +def validate_cli_examples(docs_dir: Path = DOCS_DIR, root: Path = ROOT) -> list[str]: + """Parse documentation commands while stopping before command dispatch.""" + + errors: list[str] = [] + for document, command in documented_cli_commands(docs_dir): + parts = shlex.split(command) + argv = parts[3:] if parts[0] != "obliteratus" else parts[1:] + try: + _parse_without_dispatch(argv) + except ValueError as exc: + errors.append(f"{document.relative_to(root)} invalid CLI example {command!r}: {exc}") + return errors + + +def validate_all( + *, + matrix_path: Path = MATRIX_PATH, + schema_path: Path = SCHEMA_PATH, + docs_dir: Path = DOCS_DIR, + root: Path = ROOT, +) -> list[str]: + return [ + *validate_matrix(matrix_path, schema_path), + *validate_local_links(docs_dir, root), + *validate_cli_examples(docs_dir, root), + ] + + +def main() -> int: + errors = validate_all() + if errors: + for error in errors: + print(f"checkpoint docs validation failed: {error}") + return 1 + print("checkpoint docs validation passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_checkpoint_fixtures.py b/scripts/generate_checkpoint_fixtures.py new file mode 100644 index 0000000..18a2c06 --- /dev/null +++ b/scripts/generate_checkpoint_fixtures.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +"""Generate the deterministic, synthetic Wave 2 checkpoint fixture corpus.""" + +from __future__ import annotations + +import argparse +import json +from hashlib import sha256 +from pathlib import Path +from typing import Any + +import torch +from safetensors.torch import save_file + +from obliteratus.checkpoint_fragments import ( + Padding, + Replica, + TensorFragment, + reconstruct_logical_tensor, + validate_fragments, +) + + +GENERATOR_VERSION = "1.0.0" +CORPUS_LIMITS = { + "max_case_bytes": 65536, + "max_cases": 16, + "max_files_per_case": 16, + "max_tensors_per_case": 16, +} + + +def _json_bytes(value: object) -> bytes: + return ( + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=True, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _write_json(path: Path, value: object) -> None: + path.write_bytes(_json_bytes(value)) + + +def _tensor_digest(tensor: torch.Tensor) -> str: + raw = tensor.detach().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + return f"sha256:{sha256(raw).hexdigest()}" + + +def _fragment( + fragment_id: str, + payload: torch.Tensor, + *, + payload_file: str, + global_shape: tuple[int, ...], + offset: tuple[int, ...], + extent: tuple[int, ...] | None = None, + logical_tensor_id: str, + component_id: str = "model", + role: str = "parameter", + padding: Padding | None = None, + replica: Replica | None = None, + partition_axes: tuple[int, ...] = (0,), + tie_group_id: str | None = None, + topology_coordinates: tuple[tuple[str, int], ...] = (("rank", 0),), +) -> tuple[TensorFragment, str]: + extent = extent if extent is not None else tuple(payload.shape) + padding = padding or Padding.zeros(len(global_shape)) + replica = replica or Replica.unique() + logical_payload = payload + if global_shape: + logical_payload = payload[ + tuple( + slice(before, before + size) + for before, size in zip(padding.before, extent, strict=True) + ) + ] + fragment = TensorFragment( + fragment_id=fragment_id, + component_id=component_id, + fqn=logical_tensor_id, + role=role, + dtype=str(payload.dtype).removeprefix("torch."), + global_shape=global_shape, + local_shape=tuple(payload.shape), + element_offset=offset, + element_extent=extent, + padding=padding, + shard_file_id=payload_file, + shard_digest_ref=f"source-digest:{payload_file}", + fragment_digest=_tensor_digest(logical_payload), + replica=replica, + partition_axes=partition_axes if global_shape else (), + logical_tensor_id=logical_tensor_id, + tie_group_id=tie_group_id, + shared_storage_id=tie_group_id, + topology_coordinates=topology_coordinates, + evidence_refs=(f"synthetic:{fragment_id}",), + payload=payload, + ) + return fragment, payload_file + + +def _case_definitions() -> list[dict[str, Any]]: + world1: list[tuple[TensorFragment, str]] = [] + world1.append( + _fragment( + "w1-weight", + torch.arange(6, dtype=torch.float32).reshape(2, 3), + payload_file="rank-00000.safetensors", + global_shape=(2, 3), + offset=(0, 0), + logical_tensor_id="model.weight", + partition_axes=(), + ) + ) + world1.append( + _fragment( + "w1-scalar", + torch.tensor(3, dtype=torch.int64), + payload_file="rank-00000.safetensors", + global_shape=(), + offset=(), + logical_tensor_id="model.step", + role="persistent_buffer", + partition_axes=(), + ) + ) + world1.append( + _fragment( + "w1-buffer", + torch.tensor([0.25, 0.5], dtype=torch.float32), + payload_file="rank-00000.safetensors", + global_shape=(2,), + offset=(0,), + logical_tensor_id="model.running_mean", + role="persistent_buffer", + partition_axes=(), + ) + ) + tied = torch.tensor([1.0, 2.0, 3.0, 4.0]) + for fragment_id, tensor_id in (("w1-embed", "model.embed.weight"), ("w1-head", "lm_head.weight")): + world1.append( + _fragment( + fragment_id, + tied.clone(), + payload_file="rank-00000.safetensors", + global_shape=(4,), + offset=(0,), + logical_tensor_id=tensor_id, + partition_axes=(), + tie_group_id="tie-embedding-head", + ) + ) + world1.append( + _fragment( + "w1-expert", + torch.arange(4, dtype=torch.float32).reshape(2, 2), + payload_file="rank-00000.safetensors", + global_shape=(2, 2), + offset=(0, 0), + logical_tensor_id="model.experts.0.weight", + partition_axes=(), + topology_coordinates=(("ep", 0), ("rank", 0)), + ) + ) + + uneven = torch.arange(7, dtype=torch.int64) + world2 = [ + _fragment( + "w2-r0", + torch.cat((uneven[:3], torch.tensor([-1], dtype=torch.int64))), + payload_file="rank-00000.safetensors", + global_shape=(7,), + offset=(0,), + extent=(3,), + logical_tensor_id="model.weight", + padding=Padding((0,), (1,), "producer_declared"), + topology_coordinates=(("rank", 0), ("tp", 0)), + ), + _fragment( + "w2-r1", + uneven[3:].clone(), + payload_file="rank-00001.safetensors", + global_shape=(7,), + offset=(3,), + logical_tensor_id="model.weight", + topology_coordinates=(("rank", 1), ("tp", 1)), + ), + ] + + matrix = torch.arange(35, dtype=torch.float32).reshape(5, 7) + world4: list[tuple[TensorFragment, str]] = [] + rank = 0 + for row, (top, bottom) in enumerate(((0, 2), (2, 5))): + for column, (left, right) in enumerate(((0, 3), (3, 7))): + world4.append( + _fragment( + f"w4-r{rank}", + matrix[top:bottom, left:right].clone(), + payload_file=f"rank-{rank:05d}.safetensors", + global_shape=(5, 7), + offset=(top, left), + logical_tensor_id="model.weight", + partition_axes=(0, 1), + topology_coordinates=(("rank", rank), ("tp_row", row), ("tp_col", column)), + ) + ) + rank += 1 + + replica_value = torch.tensor([5.0, 6.0, 7.0]) + replicas = [ + _fragment( + f"dp-r{rank}", + replica_value.clone(), + payload_file=f"rank-{rank:05d}.safetensors", + global_shape=(3,), + offset=(0,), + logical_tensor_id="model.weight", + partition_axes=(), + replica=Replica("dp-full", rank, 4), + topology_coordinates=(("dp", rank), ("rank", rank)), + ) + for rank in range(4) + ] + + topology_value = torch.arange(16, dtype=torch.float32).reshape(4, 4) + tp_pp: list[tuple[TensorFragment, str]] = [] + rank = 0 + for pp, (top, bottom) in enumerate(((0, 2), (2, 4))): + for tp, (left, right) in enumerate(((0, 2), (2, 4))): + tp_pp.append( + _fragment( + f"tp-pp-r{rank}", + topology_value[top:bottom, left:right].clone(), + payload_file=f"rank-{rank:05d}.safetensors", + global_shape=(4, 4), + offset=(top, left), + logical_tensor_id="model.weight", + partition_axes=(0, 1), + topology_coordinates=(("pp", pp), ("rank", rank), ("tp", tp)), + ) + ) + rank += 1 + + mixed = [ + _fragment( + "mixed-model", + torch.arange(4, dtype=torch.float32).reshape(2, 2), + payload_file="rank-00000.safetensors", + global_shape=(2, 2), + offset=(0, 0), + logical_tensor_id="model.weight", + component_id="full-model", + partition_axes=(), + topology_coordinates=(("rank", 0),), + ), + _fragment( + "mixed-adapter", + torch.tensor([[0.5, -0.5]], dtype=torch.float32), + payload_file="rank-00001.safetensors", + global_shape=(1, 2), + offset=(0, 0), + logical_tensor_id="adapter.lora_A.weight", + component_id="peft-adapter", + partition_axes=(), + topology_coordinates=(("rank", 1),), + ), + ] + + return [ + { + "case_id": "world1-complete", + "world_size": 1, + "features": ["buffer", "expert", "scalar", "tied_weight"], + "components": ["full_model"], + "topology": {"source": {"world_size": 1}, "target": {"world_size": 1}}, + "fragments": world1, + }, + { + "case_id": "world2-uneven-1d", + "world_size": 2, + "features": ["padding", "uneven_1d"], + "components": ["full_model"], + "topology": {"source": {"tp": 2}, "target": {"world_size": 1}}, + "fragments": world2, + }, + { + "case_id": "world4-uneven-2d", + "world_size": 4, + "features": ["uneven_2d"], + "components": ["full_model"], + "topology": {"source": {"tp_rows": 2, "tp_columns": 2}, "target": {"world_size": 1}}, + "fragments": world4, + }, + { + "case_id": "world4-dp-replicas", + "world_size": 4, + "features": ["dp_replica"], + "components": ["full_model"], + "topology": {"source": {"dp": 4}, "target": {"world_size": 1}}, + "fragments": replicas, + }, + { + "case_id": "tp2-pp2-to-single", + "world_size": 4, + "features": ["pipeline_parallel", "topology_a_to_b"], + "components": ["full_model"], + "topology": {"source": {"pp": 2, "tp": 2}, "target": {"world_size": 1}}, + "fragments": tp_pp, + }, + { + "case_id": "mixed-model-peft", + "world_size": 2, + "features": ["mixed_full_model_peft"], + "components": ["full_model", "peft_adapter"], + "topology": {"source": {"world_size": 2}, "target": {"world_size": 1}}, + "fragments": mixed, + }, + ] + + +def _write_case(root: Path, definition: dict[str, Any]) -> dict[str, Any]: + case_id = definition["case_id"] + case_root = root / "cases" / case_id + case_root.mkdir(parents=True) + fragment_pairs: list[tuple[TensorFragment, str]] = definition["fragments"] + by_file: dict[str, dict[str, torch.Tensor]] = {} + fragments: list[TensorFragment] = [] + fragment_records: list[dict[str, object]] = [] + for fragment, payload_file in fragment_pairs: + fragments.append(fragment) + by_file.setdefault(payload_file, {})[fragment.fragment_id] = fragment.payload + record = fragment.manifest_record() + record["payload_file"] = payload_file + record["payload_key"] = fragment.fragment_id + fragment_records.append(record) + for filename, tensors in sorted(by_file.items()): + save_file(dict(sorted(tensors.items())), case_root / filename) + validation = validate_fragments(fragments) + oracle_tensors: dict[str, torch.Tensor] = {} + oracle_records: list[dict[str, object]] = [] + for index, logical in enumerate(validation.logical_tensors): + value = reconstruct_logical_tensor(validation, logical.logical_tensor_id) + payload_key = f"tensor_{index:03d}" + oracle_tensors[payload_key] = value + oracle_records.append( + { + "logical_tensor_id": logical.logical_tensor_id, + "payload_key": payload_key, + "shape": list(value.shape), + "dtype": str(value.dtype).removeprefix("torch."), + "sha256": _tensor_digest(value), + } + ) + save_file(dict(sorted(oracle_tensors.items())), case_root / "oracles.safetensors") + _write_json( + case_root / "case.json", + { + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "case_id": case_id, + "world_size": definition["world_size"], + "features": sorted(definition["features"]), + "components": definition["components"], + "topology": definition["topology"], + "fragments": sorted(fragment_records, key=lambda item: item["fragment_id"]), + "oracle_file": "oracles.safetensors", + "oracles": oracle_records, + "expected_manifest_digest": validation.manifest_digest, + }, + ) + files = [] + for path in sorted(item for item in case_root.iterdir() if item.is_file()): + payload = path.read_bytes() + files.append( + { + "relative_path": path.name, + "size_bytes": len(payload), + "sha256": f"sha256:{sha256(payload).hexdigest()}", + } + ) + if len(files) > CORPUS_LIMITS["max_files_per_case"]: + raise ValueError(f"fixture case exceeds file limit: {case_id}") + if sum(item["size_bytes"] for item in files) > CORPUS_LIMITS["max_case_bytes"]: + raise ValueError(f"fixture case exceeds byte limit: {case_id}") + return { + "case_id": case_id, + "relative_path": f"cases/{case_id}", + "world_size": definition["world_size"], + "features": sorted(definition["features"]), + "files": files, + } + + +def _negative_catalog() -> dict[str, object]: + failures = { + "dimension_mismatch": "DCI_VALIDATION_FAILED", + "extra_shard": "DCI_SOURCE_BOUNDARY_VIOLATION", + "fragment_out_of_bounds": "DCI_VALIDATION_FAILED", + "integer_overflow": "DCI_VALIDATION_FAILED", + "missing_shard": "DCI_SOURCE_BOUNDARY_VIOLATION", + "negative_integer": "DCI_VALIDATION_FAILED", + "padding_shape_mismatch": "DCI_VALIDATION_FAILED", + "path_traversal": "DCI_SOURCE_BOUNDARY_VIOLATION", + "payload_dtype_mismatch": "DCI_VALIDATION_FAILED", + "payload_shape_mismatch": "DCI_VALIDATION_FAILED", + "replica_digest_mismatch": "DCI_VALIDATION_FAILED", + "resource_manifest_bomb": "DCI_RESOURCE_LIMIT", + "source_special_file": "DCI_SOURCE_BOUNDARY_VIOLATION", + "source_symlink": "DCI_SOURCE_BOUNDARY_VIOLATION", + "truncated_shard": "DCI_SOURCE_BOUNDARY_VIOLATION", + "coverage_gap": "DCI_VALIDATION_FAILED", + "coverage_overlap": "DCI_VALIDATION_FAILED", + } + return { + "schema_id": "obliteratus.checkpoint-negative-fixtures", + "schema_version": "1.0.0", + "cases": [ + { + "case_id": f"negative-{index:02d}", + "failure": failure, + "expected_code": code, + "mutation": f"deterministic:{failure}", + } + for index, (failure, code) in enumerate(sorted(failures.items()), start=1) + ], + } + + +def generate_corpus(destination: Path | str) -> Path: + """Create a new bounded corpus; existing paths are never overwritten.""" + root = Path(destination) + if root.exists() or root.is_symlink(): + raise FileExistsError(f"fixture destination already exists: {root}") + root.mkdir(parents=True) + definitions = _case_definitions() + if len(definitions) > CORPUS_LIMITS["max_cases"]: + raise ValueError("fixture corpus exceeds case limit") + cases = [_write_case(root, definition) for definition in definitions] + _write_json(root / "negative-cases.json", _negative_catalog()) + _write_json( + root / "fixture-corpus.json", + { + "schema_id": "obliteratus.checkpoint-fixture-corpus", + "schema_version": "1.0.0", + "generator": { + "path": "scripts/generate_checkpoint_fixtures.py", + "version": GENERATOR_VERSION, + }, + "license": "AGPL-3.0-or-later", + "provenance": { + "kind": "deterministic_synthetic", + "seed": 0, + "third_party_data": False, + "third_party_weights": False, + }, + "limits": CORPUS_LIMITS, + "cases": sorted(cases, key=lambda item: item["case_id"]), + }, + ) + return root + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("destination", type=Path) + arguments = parser.parse_args() + generate_corpus(arguments.destination) + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI wrapper + raise SystemExit(main()) diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/case.json b/tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/case.json new file mode 100644 index 0000000..0e3ea94 --- /dev/null +++ b/tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/case.json @@ -0,0 +1,161 @@ +{ + "case_id": "mixed-model-peft", + "components": [ + "full_model", + "peft_adapter" + ], + "expected_manifest_digest": "sha256:a37bcf41c60410c78430f767fcd0ab4c413ceca7fb9ef9da9bbb35f542b0043f", + "features": [ + "mixed_full_model_peft" + ], + "fragments": [ + { + "component_id": "peft-adapter", + "dtype": "float32", + "element_extent": [ + 1, + 2 + ], + "element_offset": [ + 0, + 0 + ], + "evidence_refs": [ + "synthetic:mixed-adapter" + ], + "fqn": "adapter.lora_A.weight", + "fragment_digest": "sha256:deea3b24add66f9c401d38a758eb5cb664db0596a3113b5ceaf8c5e774faa321", + "fragment_id": "mixed-adapter", + "global_shape": [ + 1, + 2 + ], + "local_shape": [ + 1, + 2 + ], + "logical_tensor_id": "adapter.lora_A.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00001.safetensors", + "payload_key": "mixed-adapter", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00001.safetensors", + "shard_file_id": "rank-00001.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 1 + ] + ] + }, + { + "component_id": "full-model", + "dtype": "float32", + "element_extent": [ + 2, + 2 + ], + "element_offset": [ + 0, + 0 + ], + "evidence_refs": [ + "synthetic:mixed-model" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe", + "fragment_id": "mixed-model", + "global_shape": [ + 2, + 2 + ], + "local_shape": [ + 2, + 2 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00000.safetensors", + "payload_key": "mixed-model", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00000.safetensors", + "shard_file_id": "rank-00000.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 0 + ] + ] + } + ], + "oracle_file": "oracles.safetensors", + "oracles": [ + { + "dtype": "float32", + "logical_tensor_id": "adapter.lora_A.weight", + "payload_key": "tensor_000", + "sha256": "sha256:deea3b24add66f9c401d38a758eb5cb664db0596a3113b5ceaf8c5e774faa321", + "shape": [ + 1, + 2 + ] + }, + { + "dtype": "float32", + "logical_tensor_id": "model.weight", + "payload_key": "tensor_001", + "sha256": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe", + "shape": [ + 2, + 2 + ] + } + ], + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "topology": { + "source": { + "world_size": 2 + }, + "target": { + "world_size": 1 + } + }, + "world_size": 2 +} diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/oracles.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/oracles.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..777e8be71d9d6e13498f733bddde1c230694dfb9 GIT binary patch literal 168 zcmeBRfPiYHlGME7{GxaR0|O#DbF4A|b3dB{yRuB^{;Wj6@JG+EB+RR!1o% fu_Q4*KP{~|wWJs*XrN;eTU)CD1VGd5fp|XvEJ7F9 literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/case.json b/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/case.json new file mode 100644 index 0000000..4ea058a --- /dev/null +++ b/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/case.json @@ -0,0 +1,310 @@ +{ + "case_id": "tp2-pp2-to-single", + "components": [ + "full_model" + ], + "expected_manifest_digest": "sha256:03da2d1dafe858e225174f12394d243dcaf70516c2649bca85c57dce91dff323", + "features": [ + "pipeline_parallel", + "topology_a_to_b" + ], + "fragments": [ + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 2 + ], + "element_offset": [ + 0, + 0 + ], + "evidence_refs": [ + "synthetic:tp-pp-r0" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:36f52612ab7fad5ef2adca0d34f41471c0c38f50cc92d9a47fa662631d3e1268", + "fragment_id": "tp-pp-r0", + "global_shape": [ + 4, + 4 + ], + "local_shape": [ + 2, + 2 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00000.safetensors", + "payload_key": "tp-pp-r0", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00000.safetensors", + "shard_file_id": "rank-00000.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "pp", + 0 + ], + [ + "rank", + 0 + ], + [ + "tp", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 2 + ], + "element_offset": [ + 0, + 2 + ], + "evidence_refs": [ + "synthetic:tp-pp-r1" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:762921576e058fd1360006105a48a8225d6baa27085f7d30abed455c288f9308", + "fragment_id": "tp-pp-r1", + "global_shape": [ + 4, + 4 + ], + "local_shape": [ + 2, + 2 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00001.safetensors", + "payload_key": "tp-pp-r1", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00001.safetensors", + "shard_file_id": "rank-00001.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "pp", + 0 + ], + [ + "rank", + 1 + ], + [ + "tp", + 1 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 2 + ], + "element_offset": [ + 2, + 0 + ], + "evidence_refs": [ + "synthetic:tp-pp-r2" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:5cca39bfe5a82135bad95ae98f2df321fe08ea75b68ea8f939ffa90845138146", + "fragment_id": "tp-pp-r2", + "global_shape": [ + 4, + 4 + ], + "local_shape": [ + 2, + 2 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00002.safetensors", + "payload_key": "tp-pp-r2", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00002.safetensors", + "shard_file_id": "rank-00002.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "pp", + 1 + ], + [ + "rank", + 2 + ], + [ + "tp", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 2 + ], + "element_offset": [ + 2, + 2 + ], + "evidence_refs": [ + "synthetic:tp-pp-r3" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:dbb7087f7cd13a205e0dcec5093a66dcfac4f6c237afd8dcafd64d166227083a", + "fragment_id": "tp-pp-r3", + "global_shape": [ + 4, + 4 + ], + "local_shape": [ + 2, + 2 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00003.safetensors", + "payload_key": "tp-pp-r3", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00003.safetensors", + "shard_file_id": "rank-00003.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "pp", + 1 + ], + [ + "rank", + 3 + ], + [ + "tp", + 1 + ] + ] + } + ], + "oracle_file": "oracles.safetensors", + "oracles": [ + { + "dtype": "float32", + "logical_tensor_id": "model.weight", + "payload_key": "tensor_000", + "sha256": "sha256:58dda328598e2f7fe472621bfc54935aaa354d1a6ebcaf9562cd743fd575eb19", + "shape": [ + 4, + 4 + ] + } + ], + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "topology": { + "source": { + "pp": 2, + "tp": 2 + }, + "target": { + "world_size": 1 + } + }, + "world_size": 4 +} diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/oracles.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/oracles.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abe5be1a9864107c0b20a08775b2586085da96f2 GIT binary patch literal 144 zcmeZZfPiYHlGME7{GxaR0|O#Q}(b_y7<;Z~y?|@)rI8 literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00002.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..09cb3c31cd741605f9bc6f4987d5ac58863f767d GIT binary patch literal 88 zcmZ=@fPiYHk^#gTzQ0EiucIKUAA+*K9B literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00003.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8fe94277c6e8d08d3493566d780fb32ca22fedcd GIT binary patch literal 88 zcmZ=@fPiYHk^?2H!{&NF+kC8gj>InjuEB< zObm2P%}`7*#$y7eep53YGgKd#;?b{TjA??IiH-#@2x}D}fEf}UK(fIeNH_oy5I2A@ U8XqW&DrN^10I2~2bhQf{08zSvqyPW_ literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/rank-00000.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/rank-00000.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5dc30d8ea9fc7775cbf82b7680a8cb1da79782bc GIT binary patch literal 472 zcmb7=?+Su25WtPzWS=j{ZH&=}@>9?=gv87(gCcPY4H0~l9<{R+6#GNR?#Ax;@jF&b zyl0do_h)vmPAVgdlEm(oYO)|3!3hwXsOGf|fyBzG-NhKIoyE=^WLQDNMpt9>fL$}% zW9b-{;ER^l=SatW@et-?*cZTPM;EEi+-ID^8#|-GyX6zD{429?TlEi9zqzJn+1G@BoN8e+&Qs literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/case.json b/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/case.json new file mode 100644 index 0000000..8a3cd84 --- /dev/null +++ b/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/case.json @@ -0,0 +1,150 @@ +{ + "case_id": "world2-uneven-1d", + "components": [ + "full_model" + ], + "expected_manifest_digest": "sha256:d0e2e96c6bdfb6ebd29470ef1f789b1a6a8f1ebc89624538cf56600ad3f4cb88", + "features": [ + "padding", + "uneven_1d" + ], + "fragments": [ + { + "component_id": "model", + "dtype": "int64", + "element_extent": [ + 3 + ], + "element_offset": [ + 0 + ], + "evidence_refs": [ + "synthetic:w2-r0" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:ab25350e3e65efebe24584461683ecda68725576e825e550038b90e7b1479946", + "fragment_id": "w2-r0", + "global_shape": [ + 7 + ], + "local_shape": [ + 4 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 1 + ], + "before": [ + 0 + ], + "semantic": "producer_declared" + }, + "partition_axes": [ + 0 + ], + "payload_file": "rank-00000.safetensors", + "payload_key": "w2-r0", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00000.safetensors", + "shard_file_id": "rank-00000.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 0 + ], + [ + "tp", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "int64", + "element_extent": [ + 4 + ], + "element_offset": [ + 3 + ], + "evidence_refs": [ + "synthetic:w2-r1" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:6df0e128a82e6d8c332ed546fca7d48406970ea60531b4d64ee41bdffc3d5da4", + "fragment_id": "w2-r1", + "global_shape": [ + 7 + ], + "local_shape": [ + 4 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0 + ], + "before": [ + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0 + ], + "payload_file": "rank-00001.safetensors", + "payload_key": "w2-r1", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00001.safetensors", + "shard_file_id": "rank-00001.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 1 + ], + [ + "tp", + 1 + ] + ] + } + ], + "oracle_file": "oracles.safetensors", + "oracles": [ + { + "dtype": "int64", + "logical_tensor_id": "model.weight", + "payload_key": "tensor_000", + "sha256": "sha256:81845a01dafa45c9b26e10a7af52a92e8604d5d8ef690f1e3ccdcfe3b5c6ae98", + "shape": [ + 7 + ] + } + ], + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "topology": { + "source": { + "tp": 2 + }, + "target": { + "world_size": 1 + } + }, + "world_size": 2 +} diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/oracles.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/oracles.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bf989f55e50c5321aecfd9e472e205f4588dd4ed GIT binary patch literal 128 zcmZ=@fPiYHlGME7{GxaR0|Ob3dB~LRGB^{;Wj6@JG+9Xy-DJ8KaF+M*ntvI!$ b7${_*V{8;#TdM#9P-7Svp$sM{{T~Vd3CbH9 literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00001.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00001.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56fcbdbed04782e91330ede29dea237a9fc6c365 GIT binary patch literal 104 zcmZ=@fPiYHawFX$LnW(frIeD&f>b3dB~LRGB^{;Wj6@JG+9Xy-DJ8KaF+M*ntvI!$ b7${_*V{8;#TdM#9%usV!pfoF#W`oiI=Rp%P literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/case.json b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/case.json new file mode 100644 index 0000000..39e54d1 --- /dev/null +++ b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/case.json @@ -0,0 +1,255 @@ +{ + "case_id": "world4-dp-replicas", + "components": [ + "full_model" + ], + "expected_manifest_digest": "sha256:d45ab4d7db81d4f7bd55e7ccb04bfb7c936b0bd320f1b57e77d7d02d3d488ff3", + "features": [ + "dp_replica" + ], + "fragments": [ + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3 + ], + "element_offset": [ + 0 + ], + "evidence_refs": [ + "synthetic:dp-r0" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655", + "fragment_id": "dp-r0", + "global_shape": [ + 3 + ], + "local_shape": [ + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0 + ], + "before": [ + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00000.safetensors", + "payload_key": "dp-r0", + "replica": { + "group_id": "dp-full", + "member_count": 4, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00000.safetensors", + "shard_file_id": "rank-00000.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "dp", + 0 + ], + [ + "rank", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3 + ], + "element_offset": [ + 0 + ], + "evidence_refs": [ + "synthetic:dp-r1" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655", + "fragment_id": "dp-r1", + "global_shape": [ + 3 + ], + "local_shape": [ + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0 + ], + "before": [ + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00001.safetensors", + "payload_key": "dp-r1", + "replica": { + "group_id": "dp-full", + "member_count": 4, + "member_index": 1 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00001.safetensors", + "shard_file_id": "rank-00001.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "dp", + 1 + ], + [ + "rank", + 1 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3 + ], + "element_offset": [ + 0 + ], + "evidence_refs": [ + "synthetic:dp-r2" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655", + "fragment_id": "dp-r2", + "global_shape": [ + 3 + ], + "local_shape": [ + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0 + ], + "before": [ + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00002.safetensors", + "payload_key": "dp-r2", + "replica": { + "group_id": "dp-full", + "member_count": 4, + "member_index": 2 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00002.safetensors", + "shard_file_id": "rank-00002.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "dp", + 2 + ], + [ + "rank", + 2 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3 + ], + "element_offset": [ + 0 + ], + "evidence_refs": [ + "synthetic:dp-r3" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655", + "fragment_id": "dp-r3", + "global_shape": [ + 3 + ], + "local_shape": [ + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0 + ], + "before": [ + 0 + ], + "semantic": "none" + }, + "partition_axes": [], + "payload_file": "rank-00003.safetensors", + "payload_key": "dp-r3", + "replica": { + "group_id": "dp-full", + "member_count": 4, + "member_index": 3 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00003.safetensors", + "shard_file_id": "rank-00003.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "dp", + 3 + ], + [ + "rank", + 3 + ] + ] + } + ], + "oracle_file": "oracles.safetensors", + "oracles": [ + { + "dtype": "float32", + "logical_tensor_id": "model.weight", + "payload_key": "tensor_000", + "sha256": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655", + "shape": [ + 3 + ] + } + ], + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "topology": { + "source": { + "dp": 4 + }, + "target": { + "world_size": 1 + } + }, + "world_size": 4 +} diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/oracles.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/oracles.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b6bb995b5c82d238868296bb146b3a56ef037186 GIT binary patch literal 84 zcmZ=@fPiYHlGME7{GxaR0|OKBFp literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00000.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00000.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fd612fb6a04537722b963723fd203923d829299f GIT binary patch literal 84 zcmZ=@fPiYHlmgu%10^dUqolGRRmn=p&DcmuN2xd?5yXo&j@400Ni0c>&reG$PAw@0 a3K{4a8pYPuDu4h3!vY5eh66zSzySb(gcd&l literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00001.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00001.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d3492095ab5432ac46b07d405b4d417b398e1109 GIT binary patch literal 84 zcmZ=@fPiYHlmgu%LnSL9qolGRRmn=p&DcmuN2xd?5yXo&j@400Ni0c>&reG$PAw@0 a3K{4a8pYPuDu4h3!vY5eh66zSzySb($`(KX literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00002.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b3f5a348d56167ac5104a8b7ba06154cf913395a GIT binary patch literal 84 zcmZ=@fPiYHlmgu%BPA;!qolGRRmn=p&DcmuN2xd?5yXo&j@400Ni0c>&reG$PAw@0 a3K{4a8pYPuDu4h3!vY5eh66zSzySb)5EelI literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00003.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f15d89bdf164ba903b57036745db57acd0497739 GIT binary patch literal 84 zcmZ=@fPiYHlmgu%V&reG$PAw@0 a3K{4a8pYPuDu4h3!vY5eh66zSzySb)Ru)14 literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/case.json b/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/case.json new file mode 100644 index 0000000..f322259 --- /dev/null +++ b/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/case.json @@ -0,0 +1,309 @@ +{ + "case_id": "world4-uneven-2d", + "components": [ + "full_model" + ], + "expected_manifest_digest": "sha256:356bddfa143562828982c2fd80b83d4a2e0c278814d844568b99aeed1f6cb310", + "features": [ + "uneven_2d" + ], + "fragments": [ + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 3 + ], + "element_offset": [ + 0, + 0 + ], + "evidence_refs": [ + "synthetic:w4-r0" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:e902ad65c5c6702e7ca1da427d13ec926431470bc155ff2983b362cb10d2dff5", + "fragment_id": "w4-r0", + "global_shape": [ + 5, + 7 + ], + "local_shape": [ + 2, + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00000.safetensors", + "payload_key": "w4-r0", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00000.safetensors", + "shard_file_id": "rank-00000.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 0 + ], + [ + "tp_col", + 0 + ], + [ + "tp_row", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 2, + 4 + ], + "element_offset": [ + 0, + 3 + ], + "evidence_refs": [ + "synthetic:w4-r1" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:8dc8137a498004d656f20c18b4726bf12aab21d7010b9980480317d6df38e8c8", + "fragment_id": "w4-r1", + "global_shape": [ + 5, + 7 + ], + "local_shape": [ + 2, + 4 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00001.safetensors", + "payload_key": "w4-r1", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00001.safetensors", + "shard_file_id": "rank-00001.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 1 + ], + [ + "tp_col", + 1 + ], + [ + "tp_row", + 0 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3, + 3 + ], + "element_offset": [ + 2, + 0 + ], + "evidence_refs": [ + "synthetic:w4-r2" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:cd2d91a4484f0cd6236034895436acbee187e02fd96900593a2512d9882f7e0e", + "fragment_id": "w4-r2", + "global_shape": [ + 5, + 7 + ], + "local_shape": [ + 3, + 3 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00002.safetensors", + "payload_key": "w4-r2", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00002.safetensors", + "shard_file_id": "rank-00002.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 2 + ], + [ + "tp_col", + 0 + ], + [ + "tp_row", + 1 + ] + ] + }, + { + "component_id": "model", + "dtype": "float32", + "element_extent": [ + 3, + 4 + ], + "element_offset": [ + 2, + 3 + ], + "evidence_refs": [ + "synthetic:w4-r3" + ], + "fqn": "model.weight", + "fragment_digest": "sha256:3c4e748352f778507efe297633562c96b36d3e68030f1c230266d63fd2994179", + "fragment_id": "w4-r3", + "global_shape": [ + 5, + 7 + ], + "local_shape": [ + 3, + 4 + ], + "logical_tensor_id": "model.weight", + "padding": { + "after": [ + 0, + 0 + ], + "before": [ + 0, + 0 + ], + "semantic": "none" + }, + "partition_axes": [ + 0, + 1 + ], + "payload_file": "rank-00003.safetensors", + "payload_key": "w4-r3", + "replica": { + "group_id": null, + "member_count": 1, + "member_index": 0 + }, + "role": "parameter", + "shard_digest_ref": "source-digest:rank-00003.safetensors", + "shard_file_id": "rank-00003.safetensors", + "shared_storage_id": null, + "tie_group_id": null, + "topology_coordinates": [ + [ + "rank", + 3 + ], + [ + "tp_col", + 1 + ], + [ + "tp_row", + 1 + ] + ] + } + ], + "oracle_file": "oracles.safetensors", + "oracles": [ + { + "dtype": "float32", + "logical_tensor_id": "model.weight", + "payload_key": "tensor_000", + "sha256": "sha256:471d327907fc83cb6703d3424393e5caeefd627fa86d8b1b2f07d3045b6e1433", + "shape": [ + 5, + 7 + ] + } + ], + "schema_id": "obliteratus.checkpoint-fixture-case", + "schema_version": "1.0.0", + "topology": { + "source": { + "tp_columns": 2, + "tp_rows": 2 + }, + "target": { + "world_size": 1 + } + }, + "world_size": 4 +} diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/oracles.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/oracles.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9487fcc4f899579c7004f755a9c555fe235e2d2b GIT binary patch literal 220 zcmXBMuM5IZ6o%m&3`YNiaq|t=UkDcGf*9>G=3pBpH@vsV7%Uo%#zmvixM(yQ7mY@v z@7TwA_&6t=wdj&d8y`%147ApAp3BIdvRKaLYCMukntj-A+o_uQN=Bg#gEUD@Yz;@P zh7;`b3dB{yRuB^{;Wj6@JG+DOMZR!1o%u_Q4*KP{~| kwWJs*XrNb3dB{yRuB^{;Wj6@JG+DOMFR!1o%u_Q4*KP{~| swWJs*XrNfcO9qD>yPR7yz*Y5C=E{05q8xJOBUy literal 0 HcmV?d00001 diff --git a/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00002.safetensors b/tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d05672f79e06476e4fe8ea48db2fdd85b63203cd GIT binary patch literal 108 zcmZ=@fPiYHaueMmBPFY9rIeD&f>b3dB{yRuB^{;Wj6@JG+E~XpR!1o%u_Q4*KP{~| twWJs*XrNb3dB{yRuB^{;Wj6@JG+E~XVR!1o%u_Q4*KP{~| zwWJs*XrN AdapterCapability: + return AdapterCapability( + adapter_id=adapter_id, + adapter_version="0.1.0", + producer=distribution, + producer_version=version, + formats=(checkpoint_format,), + required_extras=("checkpoint-example",), + required_dependencies=(ExactDependency(distribution, version),), + ) + + +def test_empty_registry_does_not_guess_an_adapter_or_dependency(): + resolution = AdapterRegistry().resolve( + "pytorch_dcp", + version_provider=lambda _name: pytest.fail("package metadata queried"), + ) + + assert resolution.to_dict() == { + "status": "missing", + "adapter_id": None, + "adapter_version": None, + "capability_digest": None, + "reason": ( + "No approved exact capability is registered; structural recognition " + "does not select an adapter or dependency set." + ), + } + + +@pytest.mark.parametrize("observed", [None, "1.2.2"]) +def test_missing_or_incompatible_dependency_has_exact_bounded_diagnostic(observed): + capability = _capability() + resolution = AdapterRegistry((capability,)).resolve( + "pytorch_dcp", + version_provider=lambda name: observed if name == "example-producer" else None, + project_version="9.8.7", + observed_producer="example-producer", + observed_producer_version="1.2.3", + ) + + assert resolution.status == "missing" + assert resolution.adapter_id == "example-dcp" + assert resolution.adapter_version == "0.1.0" + assert resolution.capability_digest == capability.capability_digest + assert resolution.reason == ( + "source_identity=verified; dependency_status=missing_or_incompatible; " + "install_extra=obliteratus[checkpoint-example]==9.8.7; " + "required_versions=example-producer==1.2.3; " + f"observed_versions=example-producer={observed or ''}" + ) + + +def test_present_exact_dependency_matches_without_granting_trust(): + capability = _capability() + resolution = AdapterRegistry((capability,)).resolve( + "pytorch_dcp", + version_provider=lambda _name: "1.2.3", + observed_producer="example-producer", + observed_producer_version="1.2.3", + ) + + assert resolution.status == "matched" + assert resolution.adapter_id == capability.adapter_id + assert resolution.capability_digest == capability.capability_digest + assert "trust authorization is still required" in resolution.reason + + +def test_format_only_candidate_never_becomes_an_exact_match(): + resolution = AdapterRegistry((_capability(),)).resolve( + "pytorch_dcp", + version_provider=lambda _name: "1.2.3", + ) + + assert resolution.status == "missing" + assert resolution.adapter_id == "example-dcp" + assert resolution.dependency_unavailable is False + assert resolution.reason.startswith("source_identity=unverified") + + +def test_observed_source_identity_must_be_complete_and_match_exactly(): + registry = AdapterRegistry((_capability(),)) + with pytest.raises(ValueError, match="provided together"): + registry.resolve("pytorch_dcp", observed_producer="example-producer") + + resolution = registry.resolve( + "pytorch_dcp", + observed_producer="example-producer", + observed_producer_version="1.2.4", + version_provider=lambda _name: pytest.fail("package metadata queried"), + ) + assert resolution.status == "missing" + assert resolution.adapter_id is None + assert "No exact capability matches" in resolution.reason + + +def test_resolution_and_digest_are_deterministic_for_explicit_records(): + first = _capability( + "adapter-b", + checkpoint_format="deepspeed_zero", + distribution="deepspeed", + version="0.16.1", + ) + second = _capability( + "adapter-a", + checkpoint_format="megatron_torch_dist", + distribution="megatron-core", + version="0.16.1", + ) + registry = registry_from([first, second]) + + assert registry.capabilities == (second, first) + assert first.capability_digest == _capability( + "adapter-b", + checkpoint_format="deepspeed_zero", + distribution="deepspeed", + version="0.16.1", + ).capability_digest + + +def test_multiple_format_matches_fail_closed_without_dependency_queries(): + registry = AdapterRegistry((_capability("adapter-a"), _capability("adapter-b"))) + resolution = registry.resolve( + "pytorch_dcp", + version_provider=lambda _name: pytest.fail("package metadata queried"), + ) + + assert resolution.status == "ambiguous" + assert resolution.adapter_id is None + assert resolution.reason == "Multiple exact capabilities match: adapter-a,adapter-b." + + +@pytest.mark.parametrize( + ("factory", "message"), + [ + (lambda: ExactDependency("bad name", "1.0.0"), "distribution"), + (lambda: ExactDependency("producer", "unselected"), "version"), + ( + lambda: AdapterCapability( + "adapter", + "1.0.0", + "producer", + "1.0.0", + ("hf_safetensors",), + ("extra",), + (ExactDependency("producer", "1.0.0"),), + ), + "trust-required", + ), + ( + lambda: AdapterRegistry((_capability(), _capability())), + "identifiers must be unique", + ), + ], +) +def test_invalid_or_non_exact_capabilities_are_rejected(factory, message): + with pytest.raises(ValueError, match=message): + factory() + + +def test_untrusted_observed_version_is_redacted_from_diagnostic(): + resolution = AdapterRegistry((_capability(),)).resolve( + "pytorch_dcp", + version_provider=lambda _name: "bad\nlocal-path=/secret", + ) + + assert "" in resolution.reason + assert "/secret" not in resolution.reason + + +def test_registry_and_capability_size_limits_are_bounded(): + dependency = ExactDependency("producer", "1.0.0") + with pytest.raises(ValueError, match="required_extras exceeds"): + AdapterCapability( + "adapter", + "1.0.0", + "producer", + "1.0.0", + ("pytorch_dcp",), + tuple(f"extra-{index}" for index in range(9)), + (dependency,), + ) + with pytest.raises(ValueError, match="registry exceeds"): + AdapterRegistry( + tuple(_capability(f"adapter-{index}") for index in range(17)) + ) + + +def test_capability_and_registry_require_immutable_typed_records(): + dependency = ExactDependency("producer", "1.0.0") + with pytest.raises(TypeError, match="immutable tuples"): + AdapterCapability( + "adapter", + "1.0.0", + "producer", + "1.0.0", + ["pytorch_dcp"], # type: ignore[arg-type] + ("extra",), + (dependency,), + ) + with pytest.raises(TypeError, match="ExactDependency"): + AdapterCapability( + "adapter", + "1.0.0", + "producer", + "1.0.0", + ("pytorch_dcp",), + ("extra",), + ("not-a-record",), # type: ignore[arg-type] + ) + with pytest.raises(TypeError, match="immutable tuple"): + AdapterRegistry([_capability()]) # type: ignore[arg-type] + with pytest.raises(TypeError, match="AdapterCapability"): + AdapterRegistry(("not-a-capability",)) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("formats", "extras", "dependencies", "message"), + [ + ((), ("extra",), (ExactDependency("producer", "1.0.0"),), "formats must"), + (("pytorch_dcp",), (), (ExactDependency("producer", "1.0.0"),), "extras must"), + (("pytorch_dcp",), ("extra",), (), "dependencies must"), + (("pytorch_dcp",) * 9, ("extra",), (ExactDependency("producer", "1.0.0"),), "formats exceeds"), + ( + ("pytorch_dcp",), + ("extra",), + tuple(ExactDependency(f"producer-{index}", "1.0.0") for index in range(5)), + "dependencies exceeds", + ), + ( + ("pytorch_dcp", "pytorch_dcp"), + ("extra",), + (ExactDependency("producer", "1.0.0"),), + "formats must be unique", + ), + ( + ("pytorch_dcp",), + ("extra", "extra"), + (ExactDependency("producer", "1.0.0"),), + "extras must be unique", + ), + ( + ("pytorch_dcp",), + ("extra",), + (ExactDependency("producer", "1.0.0"),) * 2, + "distributions must be unique", + ), + ( + ("pytorch_dcp",), + ("bad extra",), + (ExactDependency("producer", "1.0.0"),), + "required_extra", + ), + ], +) +def test_capability_collections_are_strict_and_bounded( + formats, + extras, + dependencies, + message, +): + with pytest.raises(ValueError, match=message): + AdapterCapability( + "adapter", + "1.0.0", + "producer", + "1.0.0", + formats, + extras, + dependencies, + ) + + +def test_installed_version_probe_returns_metadata_or_absence(monkeypatch): + monkeypatch.setattr(capability_module.metadata, "version", lambda _name: "1.2.3") + assert capability_module._installed_version("example") == "1.2.3" + + def missing(_name): + raise capability_module.metadata.PackageNotFoundError + + monkeypatch.setattr(capability_module.metadata, "version", missing) + assert capability_module._installed_version("example") is None diff --git a/tests/test_checkpoint_contract_schemas.py b/tests/test_checkpoint_contract_schemas.py new file mode 100644 index 0000000..982b34c --- /dev/null +++ b/tests/test_checkpoint_contract_schemas.py @@ -0,0 +1,101 @@ +"""Executable Draft 2020-12 checks for distributed-checkpoint contracts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator, FormatChecker + + +ROOT = Path(__file__).resolve().parents[1] +CONTRACTS = ROOT / "docs/checkpoints/schemas" +FIXTURES = CONTRACTS / "fixtures/v1" + + +def _load(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +def _validators() -> dict[str, Draft202012Validator]: + result = {} + for path in sorted(CONTRACTS.glob("*.schema.json")): + schema = _load(path) + Draft202012Validator.check_schema(schema) + schema_id = schema["properties"]["schema_id"]["const"] + assert schema_id not in result + result[schema_id] = Draft202012Validator(schema, format_checker=FormatChecker()) + return result + + +def test_every_checkpoint_schema_is_valid_and_has_a_unique_contract_id(): + validators = _validators() + assert { + "obliteratus.adapter-capability", + "obliteratus.checkpoint-descriptor", + "obliteratus.checkpoint-error-registry", + "obliteratus.checkpoint-support-matrix", + "obliteratus.checkpoint-trust-policy", + "obliteratus.conversion-manifest", + "obliteratus.trusted-worker-message", + } <= set(validators) + + +@pytest.mark.parametrize("path", sorted((FIXTURES / "valid").glob("*.json"))) +def test_declared_valid_contract_fixture_passes(path): + instance = _load(path) + _validators()[instance["schema_id"]].validate(instance) + + +@pytest.mark.parametrize("path", sorted((FIXTURES / "invalid").glob("*.json"))) +def test_declared_invalid_contract_fixture_fails(path): + instance = _load(path) + errors = list(_validators()[instance["schema_id"]].iter_errors(instance)) + assert errors + + +def test_trust_policy_forbids_persisted_environment_or_secret_fields(): + policy = _load(FIXTURES / "valid/trusted-metadata-policy.json") + policy["environment"] = {"TOKEN": "must-not-persist"} + + errors = list(_validators()[policy["schema_id"]].iter_errors(policy)) + + assert any("Additional properties are not allowed" in error.message for error in errors) + + +def test_support_matrix_and_error_registry_validate_against_their_schemas(): + validators = _validators() + for path in ( + ROOT / "docs/checkpoints/support-matrix-v1.json", + CONTRACTS / "checkpoint-error-codes-v1.json", + ): + instance = _load(path) + validators[instance["schema_id"]].validate(instance) + + +def test_error_registry_covers_every_fail_closed_degraded_mode_once_or_more(): + registry = _load(CONTRACTS / "checkpoint-error-codes-v1.json") + entries = registry["entries"] + assert len({entry["code"] for entry in entries}) == len(entries) + covered = {mode for entry in entries for mode in entry["degraded_modes"]} + assert covered == {f"F{number:02d}" for number in range(1, 21)} + + +def test_descriptor_blockers_accept_the_canonical_error_registry_vocabulary(): + descriptor = _load(CONTRACTS / "checkpoint-descriptor-v1.schema.json") + registry = _load(CONTRACTS / "checkpoint-error-codes-v1.json") + blocker = descriptor["$defs"]["blocker"]["properties"] + + assert {entry["category"] for entry in registry["entries"]} <= set( + blocker["category"]["enum"] + ) + assert {entry["phase"] for entry in registry["entries"]} <= set(blocker["phase"]["enum"]) + + +def test_descriptor_can_name_legacy_hf_pickle_without_treating_it_as_safetensors(): + descriptor = _load(CONTRACTS / "checkpoint-descriptor-v1.schema.json") + + assert "hf_pytorch_pickle" in descriptor["$defs"]["checkpointFormat"]["enum"] diff --git a/tests/test_checkpoint_docs_contracts.py b/tests/test_checkpoint_docs_contracts.py new file mode 100644 index 0000000..50c5824 --- /dev/null +++ b/tests/test_checkpoint_docs_contracts.py @@ -0,0 +1,108 @@ +"""Offline contracts for checkpoint documentation and support claims.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from scripts import check_checkpoint_docs + + +ROOT = Path(__file__).resolve().parents[1] + + +def _write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value), encoding="utf-8") + + +def _matrix() -> dict: + return json.loads( + (ROOT / "docs/checkpoints/support-matrix-v1.json").read_text(encoding="utf-8"), + ) + + +def test_checkpoint_documentation_contract_passes_offline(): + assert check_checkpoint_docs.validate_all() == [] + + +def test_support_matrix_rejects_unknown_fields_and_duplicate_rows(tmp_path): + matrix = _matrix() + matrix["unexpected"] = True + matrix["rows"].append(matrix["rows"][0]) + path = tmp_path / "matrix.json" + _write_json(path, matrix) + + errors = check_checkpoint_docs.validate_matrix(path) + + assert "support matrix has unknown fields: unexpected" in errors + assert any("duplicate row id" in error for error in errors) + + +def test_supported_claim_requires_exact_retained_evidence(tmp_path): + matrix = _matrix() + row = matrix["rows"][0] + row["capabilities"]["detect"]["value"] = "supported" + row["producer_versions"] = ["Transformers compatible; exact version varies"] + row["evidence"].update( + candidate_commit=None, + fixture_digest=None, + retained_result=None, + ) + path = tmp_path / "matrix.json" + _write_json(path, matrix) + + errors = check_checkpoint_docs.validate_matrix(path) + + assert any("exact producer versions" in error for error in errors) + assert any("evidence.candidate_commit" in error for error in errors) + assert any("evidence.fixture_digest" in error for error in errors) + assert any("evidence.retained_result" in error for error in errors) + + +def test_local_link_validator_rejects_missing_file_and_anchor(tmp_path): + docs = tmp_path / "docs/checkpoints" + docs.mkdir(parents=True) + (docs / "guide.md").write_text( + "# Guide\n\n[missing](missing.md) [anchor](target.md#absent)\n", + encoding="utf-8", + ) + (docs / "target.md").write_text("# Present\n", encoding="utf-8") + + errors = check_checkpoint_docs.validate_local_links(docs, tmp_path) + + assert any("missing local link" in error for error in errors) + assert any("missing anchor" in error for error in errors) + + +def test_documented_cli_examples_stop_before_dispatch(monkeypatch, tmp_path): + docs = tmp_path / "docs/checkpoints" + docs.mkdir(parents=True) + (docs / "guide.md").write_text( + "Run `python3 -m obliteratus --help` to inspect current syntax.\n", + encoding="utf-8", + ) + dispatched = False + + def forbidden_dispatch(_args): + nonlocal dispatched + dispatched = True + raise AssertionError("model command dispatch must not run") + + monkeypatch.setattr("obliteratus.cli._apply_gpu_selection", forbidden_dispatch) + + assert check_checkpoint_docs.validate_cli_examples(docs, tmp_path) == [] + assert dispatched is False + + +def test_invalid_documented_cli_example_fails_closed(tmp_path): + docs = tmp_path / "docs/checkpoints" + docs.mkdir(parents=True) + (docs / "guide.md").write_text( + "Run `obliteratus --not-a-real-option` for diagnostics.\n", + encoding="utf-8", + ) + + errors = check_checkpoint_docs.validate_cli_examples(docs, tmp_path) + + assert len(errors) == 1 + assert "invalid CLI example" in errors[0] diff --git a/tests/test_checkpoint_errors.py b/tests/test_checkpoint_errors.py new file mode 100644 index 0000000..c623b71 --- /dev/null +++ b/tests/test_checkpoint_errors.py @@ -0,0 +1,38 @@ +"""Stable runtime failures remain aligned with the accepted error registry.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from obliteratus.checkpoint_errors import CheckpointContractError + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_runtime_error_contract_covers_every_registered_code_exactly(): + registry = json.loads( + ( + ROOT / "docs/checkpoints/schemas/checkpoint-error-codes-v1.json" + ).read_text() + ) + + for entry in registry["entries"]: + error = CheckpointContractError( + entry["code"], + detail="bounded_detail", + affected_refs=("b", "a", "a"), + ) + assert error.code == entry["code"] + assert error.category == entry["category"] + assert error.phase == entry["phase"] + assert error.affected_refs == ("a", "b") + assert set(error.to_blocker()) == { + "code", + "category", + "phase", + "affected_refs", + "retryable", + "next_action", + } diff --git a/tests/test_checkpoint_evaluation.py b/tests/test_checkpoint_evaluation.py index e9851b4..569709b 100644 --- a/tests/test_checkpoint_evaluation.py +++ b/tests/test_checkpoint_evaluation.py @@ -2,6 +2,8 @@ from __future__ import annotations import hashlib import json +from pathlib import Path +from types import SimpleNamespace import pytest @@ -43,11 +45,221 @@ def test_checkpoint_inventory_verifies_size_hash_and_managed_path(tmp_path): ), encoding="utf-8", ) - manifest = { - "result": {"checkpoint": str(checkpoint), "inventory": str(inventory)} - } + manifest = {"result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}} assert _verify_checkpoint_inventory(run_dir, manifest) == checkpoint.resolve() weights.write_bytes(b"tampered weights") with pytest.raises(ValueError, match="hash changed"): _verify_checkpoint_inventory(run_dir, manifest) + + +def _inventory_candidate(tmp_path): + run_dir = tmp_path / ("run-" + "b" * 32) + checkpoint = run_dir / "checkpoint" + checkpoint.mkdir(parents=True) + weights = checkpoint / "weights.bin" + weights.write_bytes(b"verified weights") + inventory = run_dir / "artifact-inventory.json" + artifact = { + "path": "checkpoint/weights.bin", + "bytes": weights.stat().st_size, + "sha256": hashlib.sha256(weights.read_bytes()).hexdigest(), + } + inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8") + manifest = {"result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}} + return run_dir, checkpoint, inventory, artifact, manifest + + +def test_checkpoint_inventory_rejects_unmanaged_empty_and_noncheckpoint_records(tmp_path): + run_dir, checkpoint, inventory, artifact, manifest = _inventory_candidate(tmp_path) + + manifest["result"]["checkpoint"] = str(tmp_path / "outside") + with pytest.raises(ValueError, match="outside its managed"): + _verify_checkpoint_inventory(run_dir, manifest) + + manifest["result"]["checkpoint"] = str(checkpoint) + for value, message in (([], "empty"), ([{"path": "notes.md"}], "no checkpoint")): + inventory.write_text(json.dumps({"artifacts": value}), encoding="utf-8") + with pytest.raises(ValueError, match=message): + _verify_checkpoint_inventory(run_dir, manifest) + + artifact["path"] = "checkpoint/../notes.md" + inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8") + with pytest.raises(ValueError, match="invalid checkpoint artifact"): + _verify_checkpoint_inventory(run_dir, manifest) + + +def test_checkpoint_inventory_rejects_size_change(tmp_path): + run_dir, _checkpoint, inventory, artifact, manifest = _inventory_candidate(tmp_path) + artifact["bytes"] += 1 + inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8") + + with pytest.raises(ValueError, match="size changed"): + _verify_checkpoint_inventory(run_dir, manifest) + + +class _FakeLifecycle: + def __init__(self): + self.events = [] + + def loading(self, checkpoint): + self.events.append(("loading", checkpoint)) + + def resize(self, memory): + self.events.append(("resize", memory)) + + def ready(self, memory): + self.events.append(("ready", memory)) + + def release(self, *, reason): + self.events.append(("release", reason)) + + +class _FakeArchive: + def __init__(self, run_dir: Path, manifest): + self.run_dir = run_dir + self.manifest = manifest + self.finishes = [] + + def begin_evaluation(self, run_id, *, partition, evaluator): + assert run_id == self.manifest["run_id"] + self.reservation = (partition, evaluator) + return {"evaluation_id": "eval-" + "c" * 32} + + def result(self, run_id): + assert run_id == self.manifest["run_id"] + return self.manifest + + def _run_dir(self, run_id): + assert run_id == self.manifest["run_id"] + return self.run_dir + + def finish_evaluation(self, run_id, evaluation_id, **kwargs): + self.finishes.append((run_id, evaluation_id, kwargs)) + + +@pytest.mark.parametrize( + ("refusal_rate", "coherence", "expected"), + [(0.2, 0.9, 0), (0.3, 0.9, 2), (0.2, 0.79, 2)], +) +def test_evaluate_reloads_verifies_and_records_objective( + tmp_path, + monkeypatch, + refusal_rate, + coherence, + expected, +): + import obliteratus.checkpoint_evaluation as checkpoint_evaluation + + run_dir, checkpoint, _inventory, _artifact, manifest = _inventory_candidate(tmp_path) + run_id = run_dir.name + manifest["run_id"] = run_id + manifest["result"]["metrics"] = { + "baseline_perplexity": 3.5, + "baseline_coherence": 0.95, + } + archive = _FakeArchive(run_dir, manifest) + lifecycle = _FakeLifecycle() + pipelines = [] + + class FakePipeline: + def __init__(self, **kwargs): + self.kwargs = kwargs + self._quality_metrics = { + "refusal_rate": refusal_rate, + "coherence": coherence, + } + self.cleaned = False + pipelines.append(self) + + def _summon(self): + self.kwargs["on_log"]("loaded") + + def _verify(self): + self.kwargs["on_log"]("verified") + + def cleanup_failed_run(self): + self.cleaned = True + + memory = SimpleNamespace(reserved_bytes=1) + monkeypatch.setattr(checkpoint_evaluation, "RunArchive", lambda _root: archive) + monkeypatch.setattr(checkpoint_evaluation, "from_environment", lambda: lifecycle) + monkeypatch.setattr(checkpoint_evaluation, "AbliterationPipeline", FakePipeline) + monkeypatch.setattr(checkpoint_evaluation, "_partition_pairs", lambda _part: (("h", "s"),)) + monkeypatch.setattr(checkpoint_evaluation, "measure_torch_memory", lambda _torch: memory) + + assert checkpoint_evaluation.evaluate(run_id, "optimizer_tune", str(tmp_path)) == expected + assert archive.reservation == ( + "optimizer_tune", + checkpoint_evaluation.EVALUATOR_VERSION, + ) + assert pipelines[0].kwargs["model_name"] == str(checkpoint.resolve()) + assert pipelines[0]._stock_baseline == {"perplexity": 3.5, "coherence": 0.95} + assert pipelines[0].cleaned is True + metrics = archive.finishes[0][2]["metrics"] + assert metrics["passes_objective"] is (expected == 0) + assert archive.finishes[0][2]["log"] == ["loaded", "verified"] + assert lifecycle.events[-1] == ("release", "evaluation_optimizer_tune_complete") + + +def test_evaluate_records_failure_and_releases_lifecycle(tmp_path, monkeypatch): + import obliteratus.checkpoint_evaluation as checkpoint_evaluation + + run_dir, _checkpoint, _inventory, _artifact, manifest = _inventory_candidate(tmp_path) + run_id = run_dir.name + manifest["run_id"] = run_id + manifest["result"]["metrics"] = { + "baseline_perplexity": 3.5, + "baseline_coherence": 0.95, + } + archive = _FakeArchive(run_dir, manifest) + lifecycle = _FakeLifecycle() + + class FailedPipeline: + _quality_metrics = {} + + def __init__(self, **_kwargs): + pass + + def _summon(self): + raise RuntimeError("summon failed") + + def cleanup_failed_run(self): + self.cleaned = True + + monkeypatch.setattr(checkpoint_evaluation, "RunArchive", lambda _root: archive) + monkeypatch.setattr(checkpoint_evaluation, "from_environment", lambda: lifecycle) + monkeypatch.setattr(checkpoint_evaluation, "AbliterationPipeline", FailedPipeline) + monkeypatch.setattr(checkpoint_evaluation, "_partition_pairs", lambda _part: (("h", "s"),)) + + with pytest.raises(RuntimeError, match="summon failed"): + checkpoint_evaluation.evaluate(run_id, "final_test", str(tmp_path)) + + assert isinstance(archive.finishes[0][2]["failure"], RuntimeError) + assert lifecycle.events[-1] == ("release", "evaluation_final_test_complete") + + +def test_main_delegates_parsed_evaluation_arguments(monkeypatch): + import obliteratus.checkpoint_evaluation as checkpoint_evaluation + + observed = [] + monkeypatch.setattr( + "sys.argv", + [ + "checkpoint-evaluation", + "--archive-root", + "/archive", + "--run-id", + "run-" + "d" * 32, + "--partition", + "final_test", + ], + ) + monkeypatch.setattr( + checkpoint_evaluation, + "evaluate", + lambda *args: observed.append(args) or 2, + ) + + assert checkpoint_evaluation.main() == 2 + assert observed == [("run-" + "d" * 32, "final_test", "/archive")] diff --git a/tests/test_checkpoint_fixture_corpus.py b/tests/test_checkpoint_fixture_corpus.py new file mode 100644 index 0000000..5bef20d --- /dev/null +++ b/tests/test_checkpoint_fixture_corpus.py @@ -0,0 +1,207 @@ +"""Deterministic, project-owned distributed-checkpoint fixture corpus.""" + +from __future__ import annotations + +import json +from hashlib import sha256 +from pathlib import Path +import shutil + +import pytest +import torch + +from obliteratus.checkpoint_fixtures import load_fixture_case +from obliteratus.checkpoint_fragments import reconstruct_logical_tensor, validate_fragments +from scripts.generate_checkpoint_fixtures import generate_corpus + + +ROOT = Path(__file__).resolve().parents[1] +COMMITTED = ROOT / "tests/fixtures/distributed_checkpoints/v1" + + +def _tree_digest(root: Path) -> str: + digest = sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def _manifest(root: Path = COMMITTED) -> dict: + value = json.loads((root / "fixture-corpus.json").read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +def test_generator_is_byte_deterministic_and_committed_corpus_is_current(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + + generate_corpus(first) + generate_corpus(second) + + assert _tree_digest(first) == _tree_digest(second) + assert _tree_digest(first) == _tree_digest(COMMITTED) + + +def test_manifest_is_project_owned_self_hashing_and_bounded(): + manifest = _manifest() + + assert manifest["schema_id"] == "obliteratus.checkpoint-fixture-corpus" + assert manifest["schema_version"] == "1.0.0" + assert manifest["generator"] == { + "path": "scripts/generate_checkpoint_fixtures.py", + "version": "1.0.0", + } + assert manifest["license"] == "AGPL-3.0-or-later" + assert manifest["provenance"] == { + "kind": "deterministic_synthetic", + "seed": 0, + "third_party_data": False, + "third_party_weights": False, + } + assert manifest["limits"] == { + "max_case_bytes": 65536, + "max_cases": 16, + "max_files_per_case": 16, + "max_tensors_per_case": 16, + } + assert len(manifest["cases"]) <= manifest["limits"]["max_cases"] + for case in manifest["cases"]: + case_root = COMMITTED / case["relative_path"] + assert len(case["files"]) <= manifest["limits"]["max_files_per_case"] + assert sum(item["size_bytes"] for item in case["files"]) <= manifest["limits"][ + "max_case_bytes" + ] + for item in case["files"]: + payload = (case_root / item["relative_path"]).read_bytes() + assert len(payload) == item["size_bytes"] + assert f"sha256:{sha256(payload).hexdigest()}" == item["sha256"] + + +def test_valid_cases_cover_wave_two_neutral_topologies_and_features(): + cases = _manifest()["cases"] + + assert {case["world_size"] for case in cases} == {1, 2, 4} + assert {case["case_id"] for case in cases} == { + "mixed-model-peft", + "tp2-pp2-to-single", + "world1-complete", + "world2-uneven-1d", + "world4-dp-replicas", + "world4-uneven-2d", + } + features = {feature for case in cases for feature in case["features"]} + assert { + "buffer", + "dp_replica", + "expert", + "mixed_full_model_peft", + "padding", + "pipeline_parallel", + "scalar", + "tied_weight", + "topology_a_to_b", + "uneven_1d", + "uneven_2d", + } <= features + + +def test_every_valid_case_reconstructs_the_independent_value_oracle(): + for case_record in _manifest()["cases"]: + case = load_fixture_case(COMMITTED / case_record["relative_path"]) + result = validate_fragments(case.fragments) + + assert result.manifest_digest == case.expected_manifest_digest + assert len(result.logical_tensors) <= _manifest()["limits"]["max_tensors_per_case"] + for logical_tensor_id, oracle in case.tensor_oracles.items(): + tensor = reconstruct_logical_tensor(result, logical_tensor_id) + assert tuple(tensor.shape) == oracle.shape + assert str(tensor.dtype).removeprefix("torch.") == oracle.dtype + assert torch.equal(tensor, oracle.values) + raw = tensor.contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + assert f"sha256:{sha256(raw).hexdigest()}" == oracle.sha256 + + +def test_negative_catalog_covers_each_required_corruption_family(): + catalog = json.loads((COMMITTED / "negative-cases.json").read_text(encoding="utf-8")) + + assert catalog["schema_id"] == "obliteratus.checkpoint-negative-fixtures" + assert {case["failure"] for case in catalog["cases"]} == { + "dimension_mismatch", + "extra_shard", + "fragment_out_of_bounds", + "integer_overflow", + "missing_shard", + "negative_integer", + "padding_shape_mismatch", + "path_traversal", + "payload_dtype_mismatch", + "payload_shape_mismatch", + "replica_digest_mismatch", + "resource_manifest_bomb", + "source_special_file", + "source_symlink", + "truncated_shard", + "coverage_gap", + "coverage_overlap", + } + assert all(case["expected_code"].startswith("DCI_") for case in catalog["cases"]) + + +def _mutable_case(tmp_path: Path) -> Path: + destination = tmp_path / "case" + shutil.copytree(COMMITTED / "cases/world1-complete", destination) + return destination + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda record: record.update(schema_id="unknown"), "unsupported fixture case schema"), + (lambda record: record.update(fragments={}), "fragment list is invalid"), + (lambda record: record.update(fragments=[None]), "fragment must be an object"), + ( + lambda record: record["fragments"][0].update(payload_file="../escape"), + "unsafe fixture payload file", + ), + ( + lambda record: record["fragments"][0].update(payload_key="missing"), + "fixture payload key is missing", + ), + ], +) +def test_fixture_loader_rejects_invalid_case_contracts(tmp_path, mutate, message): + case = _mutable_case(tmp_path) + record = json.loads((case / "case.json").read_text(encoding="utf-8")) + mutate(record) + (case / "case.json").write_text(json.dumps(record), encoding="utf-8") + + with pytest.raises(ValueError, match=message): + load_fixture_case(case) + + +def test_fixture_loader_rejects_non_object_oversized_and_nonregular_json(tmp_path): + case = _mutable_case(tmp_path) + (case / "case.json").write_text("[]", encoding="utf-8") + with pytest.raises(ValueError, match="not an object"): + load_fixture_case(case) + + (case / "case.json").write_bytes(b" " * (256 * 1024 + 1)) + with pytest.raises(ValueError, match="exceeds"): + load_fixture_case(case) + + (case / "case.json").unlink() + (case / "case.json").mkdir() + with pytest.raises(ValueError, match="not a regular file"): + load_fixture_case(case) + + +def test_fixture_loader_rejects_symlink_root(tmp_path): + link = tmp_path / "linked" + link.symlink_to(COMMITTED / "cases/world1-complete", target_is_directory=True) + + with pytest.raises(ValueError, match="non-symlink directory"): + load_fixture_case(link) diff --git a/tests/test_checkpoint_fragments.py b/tests/test_checkpoint_fragments.py new file mode 100644 index 0000000..20e3fcd --- /dev/null +++ b/tests/test_checkpoint_fragments.py @@ -0,0 +1,536 @@ +"""Producer-neutral tensor-fragment validation and reconstruction oracles.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError, replace +from hashlib import sha256 + +import pytest +import torch +from hypothesis import given, strategies as st + +from obliteratus.checkpoint_errors import CheckpointContractError +from obliteratus.checkpoint_fragments import ( + FragmentLimits, + Padding, + Replica, + TensorFragment, + reconstruct_logical_tensor, + validate_fragments, +) + + +def _digest(tensor: torch.Tensor) -> str: + payload = ( + tensor.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + ) + return f"sha256:{sha256(payload).hexdigest()}" + + +def _fragment( + fragment_id: str, + payload: torch.Tensor, + *, + global_shape: tuple[int, ...], + offset: tuple[int, ...], + extent: tuple[int, ...] | None = None, + logical_tensor_id: str = "tensor.weight", + padding: Padding | None = None, + replica: Replica | None = None, + tie_group_id: str | None = None, + partition_axes: tuple[int, ...] = (0,), +) -> TensorFragment: + extent = extent if extent is not None else tuple(payload.shape) + padding = padding or Padding.zeros(len(global_shape)) + replica = replica or Replica.unique() + return TensorFragment( + fragment_id=fragment_id, + component_id="model", + fqn=logical_tensor_id, + role="parameter", + dtype=str(payload.dtype).removeprefix("torch."), + global_shape=global_shape, + local_shape=tuple(payload.shape), + element_offset=offset, + element_extent=extent, + padding=padding, + shard_file_id=f"shard-{fragment_id}", + shard_digest_ref=f"digest-{fragment_id}", + fragment_digest=_digest( + payload[ + tuple( + slice(before, before + size) + for before, size in zip(padding.before, extent, strict=True) + ) + ] + if global_shape + else payload + ), + replica=replica, + partition_axes=partition_axes if global_shape else (), + logical_tensor_id=logical_tensor_id, + tie_group_id=tie_group_id, + shared_storage_id=None, + topology_coordinates=(("tp", 0), ("pp", 1)), + evidence_refs=("evidence-1",), + payload=payload, + ) + + +def _assert_refused(fragments: list[TensorFragment], detail: str) -> None: + with pytest.raises(CheckpointContractError) as caught: + validate_fragments(fragments) + assert caught.value.code == "DCI_VALIDATION_FAILED" + assert caught.value.detail == detail + assert any(reference.startswith("tensor") for reference in caught.value.affected_refs) + + +def test_records_are_frozen_and_scalar_round_trips(): + fragment = _fragment( + "scalar", + torch.tensor(7.0), + global_shape=(), + offset=(), + partition_axes=(), + ) + + with pytest.raises(FrozenInstanceError): + fragment.dtype = "float16" # type: ignore[misc] + + result = validate_fragments([fragment]) + actual = reconstruct_logical_tensor(result, "tensor.weight") + + assert actual.shape == torch.Size([]) + assert actual.item() == 7.0 + assert result.logical_tensors[0].topology_coordinates == (("pp", 1), ("tp", 0)) + + +def test_uneven_fragments_reconstruct_independently_of_record_order(): + expected = torch.arange(15, dtype=torch.float32).reshape(3, 5) + fragments = [ + _fragment( + "right", + expected[:, 2:].clone(), + global_shape=(3, 5), + offset=(0, 2), + partition_axes=(1,), + ), + _fragment( + "left", + expected[:, :2].clone(), + global_shape=(3, 5), + offset=(0, 0), + partition_axes=(1,), + ), + ] + + forward = validate_fragments(fragments) + reverse = validate_fragments(list(reversed(fragments))) + + assert torch.equal(reconstruct_logical_tensor(forward, "tensor.weight"), expected) + assert torch.equal(reconstruct_logical_tensor(reverse, "tensor.weight"), expected) + assert forward.manifest_digest == reverse.manifest_digest + assert [item.fragment_id for item in forward.logical_tensors[0].fragments] == [ + "left", + "right", + ] + + +def test_two_dimensional_tiles_cover_the_logical_tensor_exactly(): + expected = torch.arange(24, dtype=torch.int64).reshape(4, 6) + fragments = [] + for row, (start, stop) in enumerate(((0, 1), (1, 4))): + for column, (left, right) in enumerate(((0, 2), (2, 6))): + fragments.append( + _fragment( + f"tile-{row}-{column}", + expected[start:stop, left:right].clone(), + global_shape=(4, 6), + offset=(start, left), + partition_axes=(0, 1), + ) + ) + + result = validate_fragments(fragments) + + assert torch.equal(reconstruct_logical_tensor(result, "tensor.weight"), expected) + + +def test_declared_padding_is_removed_before_reconstruction(): + payload = torch.tensor([-1, 10, 11, -2], dtype=torch.int32) + fragment = _fragment( + "padded", + payload, + global_shape=(2,), + offset=(0,), + extent=(2,), + padding=Padding(before=(1,), after=(1,), semantic="producer_declared"), + ) + + result = validate_fragments([fragment]) + + assert torch.equal( + reconstruct_logical_tensor(result, "tensor.weight"), + torch.tensor([10, 11], dtype=torch.int32), + ) + + +def test_explicit_replicas_are_deduplicated_only_after_digest_agreement(): + payload = torch.tensor([1.0, 2.0]) + fragments = [ + _fragment( + "replica-0", + payload.clone(), + global_shape=(2,), + offset=(0,), + replica=Replica("dp-0", 0, 2), + ), + _fragment( + "replica-1", + payload.clone(), + global_shape=(2,), + offset=(0,), + replica=Replica("dp-0", 1, 2), + ), + ] + + result = validate_fragments(fragments) + + assert len(result.logical_tensors[0].fragments) == 1 + assert result.logical_tensors[0].replica_members == (("replica-0", "replica-1"),) + + +@pytest.mark.parametrize( + ("mutator", "detail"), + [ + (lambda item: replace(item, element_offset=(-1,)), "negative_integer"), + ( + lambda item: replace(item, element_offset=((1 << 63) - 1,)), + "integer_overflow", + ), + (lambda item: replace(item, element_offset=(1,)), "fragment_out_of_bounds"), + (lambda item: replace(item, local_shape=(3,)), "padding_shape_mismatch"), + (lambda item: replace(item, partition_axes=(1,)), "partition_axis_out_of_bounds"), + ], +) +def test_invalid_fragment_geometry_fails_closed(mutator, detail): + valid = _fragment( + "fragment", + torch.tensor([1.0, 2.0]), + global_shape=(2,), + offset=(0,), + ) + + _assert_refused([mutator(valid)], detail) + + +def test_gap_and_overlap_are_distinct_refusals(): + left = _fragment("left", torch.tensor([1.0]), global_shape=(3,), offset=(0,)) + right = _fragment("right", torch.tensor([3.0]), global_shape=(3,), offset=(2,)) + _assert_refused([left, right], "coverage_gap") + + overlap = _fragment("overlap", torch.tensor([2.0, 3.0]), global_shape=(3,), offset=(1,)) + _assert_refused([replace(left, payload=torch.tensor([1.0, 2.0]), local_shape=(2,), element_extent=(2,), fragment_digest=None), overlap], "coverage_overlap") + + +def test_replica_membership_and_content_disagreement_fail_closed(): + payload = torch.tensor([1.0, 2.0]) + first = _fragment( + "replica-0", + payload, + global_shape=(2,), + offset=(0,), + replica=Replica("dp-0", 0, 2), + ) + _assert_refused([first], "replica_members_missing") + + disagreeing = _fragment( + "replica-1", + torch.tensor([1.0, 3.0]), + global_shape=(2,), + offset=(0,), + replica=Replica("dp-0", 1, 2), + ) + _assert_refused([first, disagreeing], "replica_digest_mismatch") + + +def test_tied_tensors_require_matching_shape_dtype_and_values(): + first = _fragment( + "embedding", + torch.tensor([1.0, 2.0]), + global_shape=(2,), + offset=(0,), + logical_tensor_id="model.embed.weight", + tie_group_id="tie-0", + ) + second = _fragment( + "lm-head", + torch.tensor([1.0, 3.0]), + global_shape=(2,), + offset=(0,), + logical_tensor_id="lm_head.weight", + tie_group_id="tie-0", + ) + + with pytest.raises(CheckpointContractError) as caught: + validate_fragments([first, second]) + + assert caught.value.detail == "tie_group_content_mismatch" + assert caught.value.affected_refs == ("lm_head.weight", "model.embed.weight") + + +def test_fragment_and_overlap_limits_refuse_before_expensive_work(): + first = _fragment("first", torch.tensor([1.0]), global_shape=(2,), offset=(0,)) + second = _fragment("second", torch.tensor([2.0]), global_shape=(2,), offset=(1,)) + + with pytest.raises(CheckpointContractError) as count_error: + validate_fragments([first, second], limits=FragmentLimits(max_fragments=1)) + assert count_error.value.code == "DCI_RESOURCE_LIMIT" + assert count_error.value.detail == "max_fragments" + + with pytest.raises(CheckpointContractError) as work_error: + validate_fragments([first, second], limits=FragmentLimits(max_overlap_checks=0)) + assert work_error.value.code == "DCI_RESOURCE_LIMIT" + assert work_error.value.detail == "max_overlap_checks" + + +@given( + size=st.integers(min_value=2, max_value=64), + split=st.integers(min_value=1, max_value=63), +) +def test_one_dimensional_partition_property(size: int, split: int): + split = min(split, size - 1) + expected = torch.arange(size, dtype=torch.int64) + fragments = [ + _fragment("a", expected[:split].clone(), global_shape=(size,), offset=(0,)), + _fragment("b", expected[split:].clone(), global_shape=(size,), offset=(split,)), + ] + + result = validate_fragments(fragments) + + assert torch.equal(reconstruct_logical_tensor(result, "tensor.weight"), expected) + + +def test_limits_and_top_level_fragment_contracts_fail_closed(): + with pytest.raises(ValueError, match="non-negative integer"): + FragmentLimits(max_fragments=-1) + + with pytest.raises(CheckpointContractError) as empty: + validate_fragments([]) + assert empty.value.detail == "fragment_set_empty" + + with pytest.raises(CheckpointContractError) as wrong_type: + validate_fragments([object()]) # type: ignore[list-item] + assert wrong_type.value.detail == "fragment_type_invalid" + + valid = _fragment("same", torch.ones(1), global_shape=(1,), offset=(0,)) + with pytest.raises(CheckpointContractError) as duplicate: + validate_fragments([valid, valid]) + assert duplicate.value.detail == "fragment_id_duplicate" + + result = validate_fragments([valid]) + with pytest.raises(KeyError, match="absent"): + result.get("absent") + + +@pytest.mark.parametrize( + ("mutator", "detail", "code"), + [ + (lambda item: replace(item, global_shape=[1]), "shape_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type] + ( + lambda item: replace(item, global_shape=(1, 1), local_shape=(1,)), + "dimension_mismatch", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, global_shape=(1, 1)), + "max_dimensions", + "DCI_RESOURCE_LIMIT", + ), + (lambda item: replace(item, element_extent=(True,)), "integer_type_invalid", "DCI_VALIDATION_FAILED"), + ( + lambda item: replace(item, global_shape=((1 << 63),)), + "integer_overflow", + "DCI_VALIDATION_FAILED", + ), + (lambda item: replace(item, component_id=""), "identifier_invalid", "DCI_VALIDATION_FAILED"), + (lambda item: replace(item, fqn="x" * 4097), "identifier_invalid", "DCI_VALIDATION_FAILED"), + (lambda item: replace(item, role="unknown"), "role_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type] + (lambda item: replace(item, dtype="string"), "dtype_unsupported", "DCI_VALIDATION_FAILED"), + (lambda item: replace(item, fragment_digest="bad"), "fragment_digest_invalid", "DCI_VALIDATION_FAILED"), + (lambda item: replace(item, padding=object()), "padding_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type] + (lambda item: replace(item, replica=object()), "replica_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type] + ( + lambda item: replace(item, padding=Padding((0,), (0,), "invalid")), # type: ignore[arg-type] + "padding_semantic_invalid", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, padding=Padding((1,), (0,), "none"), local_shape=(2,)), + "undeclared_padding", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, partition_axes=(0, 0)), + "partition_axis_duplicate", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, replica=Replica(None, 1, 1)), + "replica_declaration_invalid", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, replica=Replica("group", 0, 1)), + "replica_declaration_invalid", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, replica=Replica("group", 2, 2)), + "replica_member_out_of_bounds", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, topology_coordinates=(("tp", -1),)), + "topology_coordinate_invalid", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, topology_coordinates=(("tp", 0), ("tp", 1))), + "topology_coordinate_duplicate", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, topology_coordinates=(("tp",),)), + "topology_coordinate_invalid", + "DCI_VALIDATION_FAILED", + ), + ( + lambda item: replace(item, evidence_refs=(object(),)), + "evidence_ref_invalid", + "DCI_VALIDATION_FAILED", + ), + ], +) +def test_fragment_metadata_validation_covers_each_fail_closed_family(mutator, detail, code): + valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,)) + limits = FragmentLimits(max_dimensions=1) if detail == "max_dimensions" else FragmentLimits() + + with pytest.raises(CheckpointContractError) as caught: + validate_fragments([mutator(valid)], limits=limits) + + assert caught.value.code == code + assert caught.value.detail == detail + + +@pytest.mark.parametrize( + ("payload", "local_shape", "dtype", "digest", "detail"), + [ + (object(), (1,), "float32", None, "payload_type_invalid"), + (torch.ones(2), (1,), "float32", None, "payload_shape_mismatch"), + (torch.ones(1), (1,), "float64", None, "payload_dtype_mismatch"), + (torch.ones(1), (1,), "float32", "sha256:" + "0" * 64, "fragment_digest_mismatch"), + ], +) +def test_payload_contract_refuses_wrong_type_shape_dtype_and_digest( + payload, + local_shape, + dtype, + digest, + detail, +): + valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,)) + candidate = replace( + valid, + payload=payload, + local_shape=local_shape, + dtype=dtype, + fragment_digest=digest, + ) + _assert_refused([candidate], detail) + + +def test_payload_contract_rejects_non_strided_tensor_layout(): + valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,)) + sparse = torch.sparse_coo_tensor( + torch.tensor([[0]]), + torch.tensor([1.0]), + size=(1,), + check_invariants=True, + ) + + _assert_refused( + [replace(valid, payload=sparse, fragment_digest=None)], + "payload_layout_unsupported", + ) + + +def test_replica_metadata_count_and_digest_availability_are_mandatory(): + payload = torch.ones(1) + first = _fragment( + "a", + payload, + global_shape=(1,), + offset=(0,), + replica=Replica("group", 0, 2), + ) + second = _fragment( + "b", + payload, + global_shape=(1,), + offset=(0,), + replica=Replica("group", 1, 3), + ) + _assert_refused([first, second], "replica_count_mismatch") + + second = replace(second, replica=Replica("group", 1, 2), component_id="other") + _assert_refused([first, second], "replica_metadata_mismatch") + + first = replace(first, payload=None, fragment_digest=None) + second = replace(second, payload=None, fragment_digest=None, component_id="model") + _assert_refused([first, second], "replica_digest_unavailable") + + +def test_logical_metadata_resource_zero_and_payload_absence_boundaries(): + first = _fragment("a", torch.ones(1), global_shape=(1,), offset=(0,)) + left = _fragment("left", torch.ones(1), global_shape=(2,), offset=(0,)) + right = _fragment("right", torch.ones(1), global_shape=(2,), offset=(1,)) + _assert_refused([left, replace(right, fqn="other")], "logical_tensor_metadata_mismatch") + + with pytest.raises(CheckpointContractError) as limit: + validate_fragments([first], limits=FragmentLimits(max_elements_per_tensor=0)) + assert limit.value.code == "DCI_RESOURCE_LIMIT" + assert limit.value.detail == "max_elements_per_tensor" + + zero_a = _fragment("zero-a", torch.empty(0), global_shape=(0,), offset=(0,)) + zero_b = _fragment("zero-b", torch.empty(0), global_shape=(0,), offset=(0,)) + _assert_refused([zero_a, zero_b], "zero_tensor_representation_ambiguous") + + absent = replace(first, payload=None, fragment_digest=_digest(torch.ones(1))) + result = validate_fragments([absent]) + with pytest.raises(CheckpointContractError) as unavailable: + reconstruct_logical_tensor(result, "tensor.weight") + assert unavailable.value.detail == "payload_unavailable" + + +def test_tie_groups_require_two_members_and_identical_metadata(): + single = _fragment( + "single", + torch.ones(1), + global_shape=(1,), + offset=(0,), + tie_group_id="tie", + ) + _assert_refused([single], "tie_group_member_missing") + + second = _fragment( + "second", + torch.ones(2), + global_shape=(2,), + offset=(0,), + logical_tensor_id="tensor.other", + tie_group_id="tie", + ) + with pytest.raises(CheckpointContractError) as mismatch: + validate_fragments([single, second]) + assert mismatch.value.detail == "tie_group_metadata_mismatch" diff --git a/tests/test_checkpoint_inspection.py b/tests/test_checkpoint_inspection.py new file mode 100644 index 0000000..8d9789a --- /dev/null +++ b/tests/test_checkpoint_inspection.py @@ -0,0 +1,753 @@ +"""Offline, structure-only checkpoint inspection tests.""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +from pathlib import Path + +import pytest +import torch +from jsonschema import Draft202012Validator, FormatChecker +from safetensors.torch import save_file + +import obliteratus.checkpoint_inspection as inspection_module +from obliteratus.checkpoint_capabilities import ( + AdapterCapability, + AdapterRegistry, + ExactDependency, +) +from obliteratus.checkpoint_errors import CheckpointContractError +from obliteratus.checkpoint_inspection import InspectionLimits, inspect_checkpoint + + +ROOT = Path(__file__).resolve().parents[1] +DESCRIPTOR_SCHEMA = json.loads( + (ROOT / "docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json").read_text() +) + + +def _save(path: Path, **tensors: torch.Tensor) -> None: + save_file(dict(sorted(tensors.items())), path) + + +def _assert_contract(report) -> dict: + descriptor = report.to_dict() + Draft202012Validator( + DESCRIPTOR_SCHEMA, + format_checker=FormatChecker(), + ).validate(descriptor) + assert json.loads(report.to_json()) == descriptor + return descriptor + + +def test_direct_hf_safetensors_is_inventory_backed_and_canonical_ready(tmp_path): + _save(tmp_path / "model.safetensors", weight=torch.arange(6).reshape(2, 3)) + (tmp_path / "config.json").write_text('{"model_type":"tiny"}\n', encoding="utf-8") + + first = inspect_checkpoint(tmp_path) + second = inspect_checkpoint(tmp_path) + descriptor = _assert_contract(first) + + assert first.primary_format == "hf_safetensors" + assert first.support_decision == "canonical_hf_ready" + assert first.descriptor_id == second.descriptor_id + assert descriptor["classification_confidence"] == "verified" + assert descriptor["safety"] == { + "inspection_level": "safe_structure", + "trust_required": False, + "inventory_revalidated": True, + "unsafe_serialization_findings": [], + "violations": [], + } + assert descriptor["state"] == { + "observed_scopes": ["model_weights"], + "classification": "weights_only", + } + assert descriptor["resource_estimate"]["tensor_count"] == 1 + assert descriptor["resource_estimate"]["logical_bytes"] == 48 + assert not descriptor["blockers"] + + +def test_indexed_safetensors_validates_safe_weight_map_and_shards(tmp_path): + _save(tmp_path / "model-00001-of-00002.safetensors", a=torch.tensor([1.0])) + _save(tmp_path / "model-00002-of-00002.safetensors", b=torch.tensor([2.0, 3.0])) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps( + { + "metadata": {"total_size": 12}, + "weight_map": { + "a": "model-00001-of-00002.safetensors", + "b": "model-00002-of-00002.safetensors", + }, + } + ), + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "hf_safetensors" + assert descriptor["resource_estimate"]["tensor_count"] == 2 + assert descriptor["resource_estimate"]["shard_count"] == 2 + assert descriptor["support_decision"] == "canonical_hf_ready" + + +def test_direct_and_indexed_hf_signatures_are_an_ambiguous_collision(tmp_path): + _save(tmp_path / "model.safetensors", direct=torch.ones(1)) + _save(tmp_path / "model-00001-of-00001.safetensors", indexed=torch.ones(1)) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps( + { + "weight_map": { + "indexed": "model-00001-of-00001.safetensors", + } + } + ), + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "ambiguous" + assert descriptor["support_decision"] == "blocked" + assert "hf_layout_collision" in descriptor["safety"]["violations"] + + +@pytest.mark.parametrize("mode", ["missing", "extra"]) +def test_indexed_safetensors_missing_or_extra_shards_are_blocked(tmp_path, mode): + _save(tmp_path / "model-00001-of-00001.safetensors", a=torch.ones(1)) + referenced = ( + "missing.safetensors" + if mode == "missing" + else "model-00001-of-00001.safetensors" + ) + if mode == "extra": + _save(tmp_path / "model-00002-of-00002.safetensors", extra=torch.ones(1)) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {"a": referenced}}), + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["support_decision"] == "blocked" + assert "hf_weight_map_shard_mismatch" in descriptor["safety"]["violations"] + assert descriptor["blockers"][0]["code"] == "DCI_VALIDATION_FAILED" + + +@pytest.mark.parametrize( + ("header", "payload", "violation"), + [ + ({"weight": {"dtype": "F32", "shape": [2], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_range_invalid"), + ({"weight": {"dtype": "UNKNOWN", "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"), + ({"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 8]}}, b"\0" * 4, "safetensors_range_invalid"), + ({"weight": {"dtype": "F32", "shape": [1], "data_offsets": [4, 8]}}, b"\0" * 8, "safetensors_range_gap"), + ], +) +def test_safetensors_dtype_shape_and_range_corruption_is_blocked( + tmp_path, + header, + payload, + violation, +): + encoded = json.dumps(header, separators=(",", ":")).encode() + (tmp_path / "model.safetensors").write_bytes( + len(encoded).to_bytes(8, "little") + encoded + payload + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["support_decision"] == "blocked" + assert violation in descriptor["safety"]["violations"] + + +@pytest.mark.parametrize( + ("files", "expected_format"), + [ + ({"pytorch_model.bin": b"pickle"}, "hf_pytorch_pickle"), + ({".metadata": b"opaque-dcp-metadata", "__0_0.distcp": b"payload"}, "pytorch_dcp"), + ( + { + ".metadata": b"opaque-dcp-metadata", + "fsdp_metadata.json": b'{"state_dict_type":"SHARDED_STATE_DICT"}', + }, + "fsdp_state_dict", + ), + ( + { + "metadata.json": b'{"sharded_backend":"torch_dist","version":"1.0"}', + "common.pt": b"pickle", + }, + "megatron_torch_dist", + ), + ({"zero_pp_rank_0_mp_rank_00_optim_states.pt": b"pickle"}, "deepspeed_zero"), + ( + { + "universal_checkpoint_info.json": b'{"type":"universal"}', + "zero_pp_rank_0_mp_rank_00_model_states.pt": b"pickle", + }, + "deepspeed_universal", + ), + ], +) +def test_vendor_and_pickle_layouts_are_classified_without_payload_access( + tmp_path, + files, + expected_format, +): + for name, payload in files.items(): + (tmp_path / name).write_bytes(payload) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == expected_format + assert descriptor["support_decision"] == "trusted_inspection_required" + assert descriptor["safety"]["trust_required"] is True + assert descriptor["adapter_resolution"]["status"] == "missing" + assert {item["code"] for item in descriptor["blockers"]} == { + "DCI_TRUST_POLICY_REQUIRED" + } + + +def test_exact_registered_capability_reports_missing_extra_and_version(tmp_path): + (tmp_path / ".metadata").write_bytes(b"opaque-dcp-metadata") + (tmp_path / "__0_0.distcp").write_bytes(b"payload-never-read-as-a-tensor") + registry = AdapterRegistry( + ( + AdapterCapability( + adapter_id="example-dcp", + adapter_version="0.1.0", + producer="example-producer", + producer_version="1.2.3", + formats=("pytorch_dcp",), + required_extras=("checkpoint-example",), + required_dependencies=( + ExactDependency("obliteratus-test-package-that-does-not-exist", "1.2.3"), + ), + ), + ) + ) + + descriptor = _assert_contract( + inspect_checkpoint(tmp_path, adapter_registry=registry) + ) + + assert descriptor["primary_format"] == "pytorch_dcp" + assert descriptor["support_decision"] == "trusted_inspection_required" + assert descriptor["adapter_resolution"]["status"] == "missing" + assert descriptor["adapter_resolution"]["adapter_id"] == "example-dcp" + assert "install_extra=obliteratus[checkpoint-example]==0.1.3" in descriptor[ + "adapter_resolution" + ]["reason"] + assert ( + "required_versions=obliteratus-test-package-that-does-not-exist==1.2.3" + in descriptor["adapter_resolution"]["reason"] + ) + assert {item["code"] for item in descriptor["blockers"]} == { + "DCI_TRUST_POLICY_REQUIRED", + "DCI_TRUST_RUNTIME_UNAVAILABLE", + } + + +def test_ambiguous_registered_capabilities_fail_closed(tmp_path): + (tmp_path / ".metadata").write_bytes(b"opaque-dcp-metadata") + dependency = ExactDependency("example-producer", "1.2.3") + registry = AdapterRegistry( + tuple( + AdapterCapability( + adapter_id=f"example-{index}", + adapter_version="0.1.0", + producer="example-producer", + producer_version="1.2.3", + formats=("pytorch_dcp",), + required_extras=("checkpoint-example",), + required_dependencies=(dependency,), + ) + for index in range(2) + ) + ) + + descriptor = _assert_contract( + inspect_checkpoint(tmp_path, adapter_registry=registry) + ) + + assert descriptor["adapter_resolution"]["status"] == "ambiguous" + assert descriptor["support_decision"] == "blocked" + assert [item["code"] for item in descriptor["blockers"]].count( + "DCI_UNSUPPORTED_FORMAT_OR_VERSION" + ) == 1 + + +def test_peft_layout_is_safe_safetensors_but_keeps_adapter_identity(tmp_path): + _save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1, 2)) + (tmp_path / "adapter_config.json").write_text( + '{"base_model_name_or_path":"local/base","peft_type":"LORA"}\n', + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "peft_safetensors" + assert descriptor["components"][0]["kind"] == "peft_adapter" + assert descriptor["components"][0]["format"] == "peft_safetensors" + assert descriptor["state"]["observed_scopes"] == ["adapter_weights"] + assert descriptor["support_decision"] == "canonical_hf_ready" + + +def test_mixed_model_and_adapter_components_are_preserved_and_blocked(tmp_path): + _save(tmp_path / "model.safetensors", weight=torch.ones(2, 2)) + _save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1, 2)) + (tmp_path / "adapter_config.json").write_text('{"peft_type":"LORA"}\n') + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "ambiguous" + assert [(item["kind"], item["format"]) for item in descriptor["components"]] == [ + ("model", "hf_safetensors"), + ("peft_adapter", "peft_safetensors"), + ] + assert descriptor["state"]["observed_scopes"] == ["adapter_weights", "model_weights"] + assert descriptor["support_decision"] == "blocked" + assert descriptor["blockers"][0]["code"] == "DCI_UNSUPPORTED_FORMAT_OR_VERSION" + + +def test_unknown_layout_is_a_stable_blocked_descriptor(tmp_path): + (tmp_path / "notes.txt").write_text("not a checkpoint\n") + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "unknown" + assert descriptor["classification_confidence"] == "unknown" + assert descriptor["components"][0]["kind"] == "unknown" + assert descriptor["support_decision"] == "blocked" + + +def test_legacy_pickle_payload_is_never_executed(tmp_path): + marker = tmp_path / "payload-executed" + command = f"touch {marker}".encode("utf-8") + malicious = b"cos\nsystem\n(S'" + command + b"'\ntR." + (tmp_path / "pytorch_model.bin").write_bytes(malicious) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "hf_pytorch_pickle" + assert not marker.exists() + + +def test_remote_code_declaration_is_inert_and_not_imported(tmp_path, monkeypatch): + _save(tmp_path / "model.safetensors", weight=torch.ones(1)) + (tmp_path / "config.json").write_text( + json.dumps({"auto_map": {"AutoModel": "must_not_import.Model"}}), + encoding="utf-8", + ) + imported: list[str] = [] + original_import = __import__ + + def guarded_import(name, *args, **kwargs): + if name.startswith("must_not_import"): + imported.append(name) + raise AssertionError("remote code import attempted") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr("builtins.__import__", guarded_import) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["primary_format"] == "hf_safetensors" + assert imported == [] + + +def test_default_inspection_invokes_no_reader_network_process_group_or_plugin( + tmp_path, + monkeypatch, +): + _save(tmp_path / "model.safetensors", weight=torch.ones(1)) + before = { + path.name: path.read_bytes() + for path in tmp_path.iterdir() + if path.is_file() + } + monkeypatch.setattr(torch, "load", lambda *_a, **_k: pytest.fail("torch.load called")) + monkeypatch.setattr( + torch.distributed, + "init_process_group", + lambda *_a, **_k: pytest.fail("process group initialized"), + ) + monkeypatch.setattr( + socket, + "create_connection", + lambda *_a, **_k: pytest.fail("network opened"), + ) + monkeypatch.setattr( + subprocess, + "run", + lambda *_a, **_k: pytest.fail("subprocess started"), + ) + monkeypatch.setattr( + "importlib.metadata.entry_points", + lambda *_a, **_k: pytest.fail("plugin discovery attempted"), + ) + monkeypatch.setattr( + "importlib.metadata.version", + lambda *_a, **_k: pytest.fail("package metadata queried without a capability"), + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert descriptor["support_decision"] == "canonical_hf_ready" + assert { + path.name: path.read_bytes() + for path in tmp_path.iterdir() + if path.is_file() + } == before + + +def test_duplicate_recognized_basenames_fail_closed(tmp_path): + for directory in (tmp_path / "a", tmp_path / "b"): + directory.mkdir() + _save(directory / "model.safetensors", weight=torch.ones(1)) + + with pytest.raises(CheckpointContractError) as caught: + inspect_checkpoint(tmp_path) + + assert caught.value.code == "DCI_VALIDATION_FAILED" + assert caught.value.detail == "duplicate_basename" + + +def test_symlink_and_special_file_sources_fail_before_classification(tmp_path): + target = tmp_path / "target.safetensors" + target.write_bytes(b"not relevant") + (tmp_path / "model.safetensors").symlink_to(target.name) + + with pytest.raises(CheckpointContractError) as symlink_error: + inspect_checkpoint(tmp_path) + assert symlink_error.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION" + assert symlink_error.value.detail == "source_symlink" + + (tmp_path / "model.safetensors").unlink() + if hasattr(os, "mkfifo"): + os.mkfifo(tmp_path / "special") + with pytest.raises(CheckpointContractError) as special_error: + inspect_checkpoint(tmp_path) + assert special_error.value.detail == "source_special_file" + + +def test_resource_limits_apply_before_header_or_json_allocation(tmp_path): + (tmp_path / "a").write_bytes(b"a") + (tmp_path / "b").write_bytes(b"b") + with pytest.raises(CheckpointContractError) as file_error: + inspect_checkpoint(tmp_path, limits=InspectionLimits(max_files=1)) + assert file_error.value.code == "DCI_RESOURCE_LIMIT" + assert file_error.value.detail == "max_files" + + for path in tmp_path.iterdir(): + path.unlink() + (tmp_path / "model.safetensors").write_bytes((1024).to_bytes(8, "little") + b"{}") + with pytest.raises(CheckpointContractError) as header_error: + inspect_checkpoint( + tmp_path, + limits=InspectionLimits(max_safetensors_header_bytes=64), + ) + assert header_error.value.detail == "max_safetensors_header_bytes" + + +def test_inventory_race_is_detected_and_never_returned_as_success(tmp_path, monkeypatch): + weights = tmp_path / "model.safetensors" + _save(weights, weight=torch.ones(1)) + original = inspection_module._hash_regular_file + changed = False + + def race(path, expected, limits): + nonlocal changed + result = original(path, expected, limits) + if not changed: + changed = True + path.write_bytes(path.read_bytes() + b"changed") + return result + + monkeypatch.setattr(inspection_module, "_hash_regular_file", race) + + with pytest.raises(CheckpointContractError) as caught: + inspect_checkpoint(tmp_path) + + assert caught.value.code == "DCI_SOURCE_CHANGED" + assert caught.value.detail == "source_changed" + + +def _raw_safetensors(path: Path, header: object, payload: bytes = b"") -> None: + encoded = json.dumps(header, separators=(",", ":")).encode("utf-8") + path.write_bytes(len(encoded).to_bytes(8, "little") + encoded + payload) + + +def test_inspection_limits_and_source_root_types_fail_closed(tmp_path): + with pytest.raises(ValueError, match="positive integer"): + InspectionLimits(max_files=0) + + with pytest.raises(CheckpointContractError) as missing: + inspect_checkpoint(tmp_path / "missing") + assert missing.value.detail == "source_missing" + + target = tmp_path / "target" + target.mkdir() + root_link = tmp_path / "root-link" + root_link.symlink_to(target, target_is_directory=True) + with pytest.raises(CheckpointContractError) as symlink: + inspect_checkpoint(root_link) + assert symlink.value.detail == "source_symlink" + + if hasattr(os, "mkfifo"): + fifo = tmp_path / "root-fifo" + os.mkfifo(fifo) + with pytest.raises(CheckpointContractError) as special: + inspect_checkpoint(fifo) + assert special.value.detail == "source_special_file" + + +def test_ancestor_symlink_and_open_race_fail_with_stable_source_errors( + tmp_path, + monkeypatch, +): + actual = tmp_path / "actual" + source = actual / "checkpoint" + source.mkdir(parents=True) + _save(source / "model.safetensors", weight=torch.ones(1)) + alias = tmp_path / "alias" + alias.symlink_to(actual, target_is_directory=True) + + with pytest.raises(CheckpointContractError) as symlink: + inspect_checkpoint(alias / "checkpoint") + assert symlink.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION" + assert symlink.value.detail == "source_symlink" + + weights = source / "model.safetensors" + real_open = inspection_module.os.open + + def fail_observed_open(path, flags): + if Path(path) == weights: + raise OSError("injected source replacement") + return real_open(path, flags) + + monkeypatch.setattr(inspection_module.os, "open", fail_observed_open) + with pytest.raises(CheckpointContractError) as changed: + inspect_checkpoint(source) + assert changed.value.code == "DCI_SOURCE_CHANGED" + assert changed.value.detail == "source_changed" + + +def test_single_file_and_directory_byte_limits_are_enforced(tmp_path): + weights = tmp_path / "single.safetensors" + _save(weights, weight=torch.ones(1)) + descriptor = _assert_contract(inspect_checkpoint(weights)) + assert descriptor["source_inventory"]["files"][0]["relative_path"] == weights.name + + nested = tmp_path / "nested" + nested.mkdir() + (nested / "file").write_text("x", encoding="utf-8") + with pytest.raises(CheckpointContractError) as directories: + inspect_checkpoint(tmp_path, limits=InspectionLimits(max_directories=1)) + assert directories.value.detail == "max_directories" + + with pytest.raises(CheckpointContractError) as total: + inspect_checkpoint(weights, limits=InspectionLimits(max_total_bytes=1)) + assert total.value.detail == "max_total_bytes" + + +@pytest.mark.parametrize( + ("header", "payload", "violation"), + [ + ([], b"", "safetensors_header_invalid"), + ({"": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"), + ({"x": []}, b"", "safetensors_header_invalid"), + ({"x": {"dtype": "F32", "shape": [1], "data_offsets": [0]}}, b"\0" * 4, "safetensors_header_invalid"), + ({"x": {"dtype": 1, "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"), + ({"x": {"dtype": "F32", "shape": [-1], "data_offsets": [0, 0]}}, b"", "safetensors_header_invalid"), + ({"x": {"dtype": "F32", "shape": [1 << 62, 4], "data_offsets": [0, 0]}}, b"", "integer_overflow"), + ({"x": {"dtype": "C128", "shape": [1 << 62], "data_offsets": [0, 0]}}, b"", "integer_overflow"), + ( + { + "a": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}, + "b": {"dtype": "F32", "shape": [1], "data_offsets": [2, 6]}, + }, + b"\0" * 6, + "safetensors_range_overlap", + ), + ], +) +def test_additional_safetensors_header_corruptions_are_blocked( + tmp_path, + header, + payload, + violation, +): + _raw_safetensors(tmp_path / "model.safetensors", header, payload) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert violation in descriptor["safety"]["violations"] + assert descriptor["support_decision"] == "blocked" + + +def test_truncated_invalid_and_overlarge_safetensors_headers(tmp_path): + weights = tmp_path / "model.safetensors" + weights.write_bytes(b"tiny") + assert "safetensors_truncated" in _assert_contract(inspect_checkpoint(tmp_path))["safety"][ + "violations" + ] + + weights.write_bytes((20).to_bytes(8, "little") + b"{}") + assert "safetensors_truncated" in _assert_contract(inspect_checkpoint(tmp_path))["safety"][ + "violations" + ] + + weights.write_bytes((1).to_bytes(8, "little") + b"{") + assert "safetensors_header_invalid" in _assert_contract(inspect_checkpoint(tmp_path))[ + "safety" + ]["violations"] + + _raw_safetensors( + weights, + { + "a": {"dtype": "F32", "shape": [0], "data_offsets": [0, 0]}, + "b": {"dtype": "F32", "shape": [0], "data_offsets": [0, 0]}, + }, + ) + with pytest.raises(CheckpointContractError) as tensor_limit: + inspect_checkpoint(tmp_path, limits=InspectionLimits(max_tensors=1)) + assert tensor_limit.value.detail == "max_tensors" + + +@pytest.mark.parametrize( + ("weight_map", "violation"), + [ + ({}, "hf_weight_map_invalid"), + ({"": "model-00001-of-00001.safetensors"}, "hf_weight_map_invalid"), + ({"other": "model-00001-of-00001.safetensors"}, "hf_weight_map_tensor_mismatch"), + ], +) +def test_hf_index_shape_and_tensor_membership_are_validated(tmp_path, weight_map, violation): + _save(tmp_path / "model-00001-of-00001.safetensors", weight=torch.ones(1)) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": weight_map}), + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert violation in descriptor["safety"]["violations"] + + +def test_hf_index_rejects_unmapped_tensor_in_a_referenced_shard(tmp_path): + _save( + tmp_path / "model-00001-of-00001.safetensors", + declared=torch.ones(1), + undeclared=torch.ones(1), + ) + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps( + { + "weight_map": { + "declared": "model-00001-of-00001.safetensors", + } + } + ), + encoding="utf-8", + ) + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + + assert "hf_weight_map_tensor_mismatch" in descriptor["safety"]["violations"] + assert descriptor["support_decision"] == "blocked" + + +def test_duplicate_json_keys_and_aggregate_tensor_limit_fail_closed(tmp_path): + (tmp_path / ".metadata").write_bytes(b"opaque") + (tmp_path / "fsdp_metadata.json").write_text( + '{"state_dict_type":"SHARDED_STATE_DICT","state_dict_type":"FULL_STATE_DICT"}', + encoding="utf-8", + ) + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + assert "json_invalid" in descriptor["safety"]["violations"] + + for path in tuple(tmp_path.iterdir()): + path.unlink() + _save(tmp_path / "model.safetensors", weight=torch.ones(1)) + _save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1)) + (tmp_path / "adapter_config.json").write_text("{}", encoding="utf-8") + with pytest.raises(CheckpointContractError) as tensors: + inspect_checkpoint(tmp_path, limits=InspectionLimits(max_tensors=1)) + assert tensors.value.code == "DCI_RESOURCE_LIMIT" + assert tensors.value.detail == "max_tensors" + + +def test_invalid_bounded_json_is_reported_without_vendor_reader(tmp_path): + (tmp_path / ".metadata").write_bytes(b"opaque") + metadata = tmp_path / "fsdp_metadata.json" + metadata.write_text("[]", encoding="utf-8") + + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + assert "json_object_required" in descriptor["safety"]["violations"] + + metadata.write_text("{", encoding="utf-8") + descriptor = _assert_contract(inspect_checkpoint(tmp_path)) + assert "json_invalid" in descriptor["safety"]["violations"] + + metadata.write_text("{}", encoding="utf-8") + with pytest.raises(CheckpointContractError) as json_limit: + inspect_checkpoint(tmp_path, limits=InspectionLimits(max_json_bytes=1)) + assert json_limit.value.detail == "max_json_bytes" + + +def test_final_revalidation_detects_removed_source(tmp_path, monkeypatch): + weights = tmp_path / "model.safetensors" + _save(weights, weight=torch.ones(1)) + original = inspection_module._format_components + + def remove_after_probe(files, limits): + result = original(files, limits) + weights.unlink() + return result + + monkeypatch.setattr(inspection_module, "_format_components", remove_after_probe) + with pytest.raises(CheckpointContractError) as caught: + inspect_checkpoint(tmp_path) + assert caught.value.code == "DCI_SOURCE_CHANGED" + + +def test_final_revalidation_detects_nested_inventory_mutation(tmp_path, monkeypatch): + nested = tmp_path / "nested" + nested.mkdir() + _save(nested / "model.safetensors", weight=torch.ones(1)) + original = inspection_module._format_components + + def add_after_probe(files, limits): + result = original(files, limits) + (nested / "late-file").write_text("changed", encoding="utf-8") + return result + + monkeypatch.setattr(inspection_module, "_format_components", add_after_probe) + with pytest.raises(CheckpointContractError) as caught: + inspect_checkpoint(tmp_path) + assert caught.value.code == "DCI_SOURCE_CHANGED" + + +def test_descriptor_read_error_is_mapped_to_stable_source_change(tmp_path, monkeypatch): + weights = tmp_path / "model.safetensors" + _save(weights, weight=torch.ones(1)) + real_read = inspection_module.os.read + calls = 0 + + def fail_after_inventory(descriptor, size): + nonlocal calls + calls += 1 + if calls > 2: + raise OSError("injected read race") + return real_read(descriptor, size) + + monkeypatch.setattr(inspection_module.os, "read", fail_after_inventory) + with pytest.raises(CheckpointContractError) as caught: + inspect_checkpoint(tmp_path) + assert caught.value.code == "DCI_SOURCE_CHANGED" + assert caught.value.detail == "source_changed" diff --git a/tests/test_checkpoint_provenance.py b/tests/test_checkpoint_provenance.py new file mode 100644 index 0000000..1e332ea --- /dev/null +++ b/tests/test_checkpoint_provenance.py @@ -0,0 +1,524 @@ +"""Versioned checkpoint provenance, lineage, and resume-state truth tests.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from obliteratus.checkpoint_provenance import ( + AdapterIdentity, + ArtifactIdentity, + DatasetIdentity, + LineageEvent, + ProvenanceRecord, + ToolIdentity, + TrainingIdentity, + build_provenance, + classify_resume_state, + migrate_legacy_metadata, + sanitize_command, + verify_provenance_record, +) +from obliteratus.run_archive import RunArchive + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = json.loads( + (ROOT / "docs/checkpoints/schemas/artifact-provenance-v1.schema.json").read_text() +) +DIGEST_A = "sha256:" + "a" * 64 +DIGEST_B = "sha256:" + "b" * 64 +COMMIT = "c" * 40 + + +def _record(*, output_digests=(DIGEST_B,), command=("checkpoint", "convert")): + base = ArtifactIdentity("hub", "org/base", "0123456789abcdef", DIGEST_A) + return build_provenance( + sources=(ArtifactIdentity("local", "content-addressed-source", None, DIGEST_A),), + converter=ToolIdentity("obliteratus-neutral-writer", "1.0.0", COMMIT), + obliteratus_commit=COMMIT, + configuration_digest=DIGEST_A, + tokenizer=ArtifactIdentity("hub", "org/base", "0123456789abcdef", DIGEST_B), + base_model=base, + command=command, + environment={"python": "3.12.7", "platform": "linux", "packages": {"torch": "2.5"}}, + source_topology={"world_size": 4, "tp": 2, "pp": 2}, + lineage=( + LineageEvent( + event_id="event-consolidation", + event_type="consolidation", + parent_artifact_ids=("artifact-sha256:" + "d" * 64,), + tool="obliteratus-neutral-writer@1.0.0", + transformations=("deduplicate_declared_replicas",), + ), + ), + input_digests=(DIGEST_A,), + output_digests=output_digests, + transformations=("consolidation", "canonical_safetensors"), + observed_scopes=("model_weights",), + lost_state=("optimizer_state", "scheduler_state"), + adapter=AdapterIdentity( + adapter_type="lora", + base_model=base, + config_digest=DIGEST_B, + key_map_digest=DIGEST_A, + ), + dataset=DatasetIdentity( + identifier="dataset/name", + revision="rev-1", + digest=DIGEST_A, + split="train", + subset=None, + record_count=42, + ), + training=TrainingIdentity( + method="adapter_train", + framework="transformers", + framework_version="4.60.0", + hyperparameters_digest=DIGEST_B, + ), + unknowns=("optimizer_producer_version",), + ) + + +def test_provenance_is_strict_versioned_canonical_and_content_addressed(): + record = _record() + payload = record.to_dict() + + Draft202012Validator(SCHEMA).validate(payload) + assert payload["schema_id"] == "obliteratus.artifact-provenance" + assert payload["schema_version"] == "1.0.0" + assert payload["artifact_id"].startswith("artifact-sha256:") + assert payload["record_digest"].startswith("sha256:") + assert json.loads(record.to_json()) == payload + assert record.artifact_id == _record().artifact_id + assert record.to_json() == _record().to_json() + + +def test_canonical_sets_and_maps_do_not_depend_on_caller_order(): + first = _record(output_digests=(DIGEST_A, DIGEST_B)) + second = _record(output_digests=(DIGEST_B, DIGEST_A)) + + assert first.artifact_id == second.artifact_id + assert first.to_json() == second.to_json() + + +@pytest.mark.parametrize( + ("scopes", "expected"), + [ + (("model_weights",), "weights_only"), + (("model_weights", "optimizer_state"), "model_and_optimizer"), + ( + ( + "model_weights", + "optimizer_state", + "scheduler_state", + "rng_state", + "dataloader_state", + "framework_state", + ), + "exact_resume", + ), + (("optimizer_state",), "unknown"), + ((), "unknown"), + ], +) +def test_resume_classification_is_derived_only_from_observed_state(scopes, expected): + assert classify_resume_state(scopes) == expected + + +def test_lineage_vocabulary_keeps_surgery_distinct_from_finetuning(): + allowed = { + "consolidation", + "reshard", + "pretrain", + "full_finetune", + "adapter_train", + "adapter_merge", + "quantization", + "dequantization", + "surgery", + } + + for event_type in allowed: + assert LineageEvent("event", event_type, (), "tool@1", ()).event_type == event_type + with pytest.raises(ValueError, match="lineage event type"): + LineageEvent("event", "finetuning_surgery", (), "tool@1", ()) + + +def test_command_redacts_secrets_prompts_and_private_paths(): + sanitized = sanitize_command( + ( + "checkpoint", + "convert", + "/home/alice/private/model", + "--token", + "hf_abcdefghijklmnopqrstuvwxyz", + "--prompt=raw private prompt", + ) + ) + rendered = json.dumps(sanitized) + + assert "/home/alice" not in rendered + assert "hf_" not in rendered + assert "raw private prompt" not in rendered + assert "[REDACTED]" in sanitized + assert any(value.startswith("[LOCAL_PATH:sha256:") for value in sanitized) + + +def test_legacy_migration_preserves_declared_facts_and_never_invents_digests(): + facts = migrate_legacy_metadata( + { + "model": "org/base", + "model_revision": "rev-1", + "tokenizer_revision": None, + "seed": "42", + "dataset_inputs": [{"identifier": "builtin", "sha256": "e" * 64}], + "unmapped_private_field": "must not leak", + } + ) + payload = facts.to_dict() + + assert payload["base_model"] == { + "identity": "org/base", + "revision": "rev-1", + "digest": None, + } + assert payload["tokenizer"] == {"revision": None, "digest": None} + assert payload["seed"] == "42" + assert payload["datasets"] == [ + {"identifier": "builtin", "digest": "sha256:" + "e" * 64} + ] + assert "unmapped_private_field" not in json.dumps(payload) + assert "base_model_digest" in payload["unknowns"] + assert "tokenizer_digest" in payload["unknowns"] + + +def test_run_archive_attaches_same_artifact_identity_without_raw_sensitive_data(tmp_path): + archive = RunArchive(tmp_path) + run_id = archive.begin(["org/base"]) + record = _record( + command=("convert", "/home/alice/private/model", "--token", "hf_secretsecretsecret") + ) + + manifest = archive.attach_checkpoint_provenance(run_id, record) + provenance_path = tmp_path / run_id / "checkpoint-provenance.json" + raw = provenance_path.read_text(encoding="utf-8") + + assert manifest["artifact_id"] == record.artifact_id + assert manifest["checkpoint_provenance"]["artifact_id"] == record.artifact_id + assert json.loads(raw)["artifact_id"] == record.artifact_id + assert "/home/alice" not in raw + assert "hf_secret" not in raw + + +def test_provenance_rejects_secret_bearing_environment_keys(): + with pytest.raises(ValueError, match="sensitive key"): + build_provenance( + sources=(ArtifactIdentity("local", "source", None, DIGEST_A),), + converter=ToolIdentity("writer", "1", COMMIT), + obliteratus_commit=COMMIT, + configuration_digest=None, + tokenizer=None, + base_model=None, + command=("convert",), + environment={"API_TOKEN": "secret"}, + source_topology={}, + lineage=(), + input_digests=(DIGEST_A,), + output_digests=(DIGEST_B,), + transformations=(), + observed_scopes=("model_weights",), + lost_state=(), + ) + + +@pytest.mark.parametrize( + ("factory", "message"), + [ + (lambda: ArtifactIdentity("other", "source", None, DIGEST_A), "kind is invalid"), + (lambda: ArtifactIdentity("local", "", None, DIGEST_A), "non-empty bounded"), + ( + lambda: ArtifactIdentity("local", "hf_abcdefghijklmnopqrstuvwxyz", None, DIGEST_A), + "contains a secret", + ), + (lambda: ArtifactIdentity("local", "/private/source", None, DIGEST_A), "private local path"), + (lambda: ArtifactIdentity("local", "source", None, "bad"), "sha256 digest"), + (lambda: ToolIdentity("tool", "1", "A" * 40), "40-character lowercase commit"), + ( + lambda: LineageEvent("event", "surgery", ("bad-parent",), "tool@1", ()), + "parent artifact ID", + ), + ( + lambda: DatasetIdentity("dataset", None, DIGEST_A, None, None, -1), + "record count", + ), + (lambda: TrainingIdentity("invalid", None, None, None), "training method"), + (lambda: TrainingIdentity("unknown", None, None, "bad"), "sha256 digest"), + ], +) +def test_identity_records_reject_unverifiable_or_sensitive_fields(factory, message): + with pytest.raises(ValueError, match=message): + factory() + + +def _minimal_provenance(**overrides): + values = { + "sources": (ArtifactIdentity("local", "source", None, DIGEST_A),), + "converter": ToolIdentity("writer", "1", COMMIT), + "obliteratus_commit": COMMIT, + "configuration_digest": None, + "tokenizer": None, + "base_model": None, + "command": ("convert",), + "environment": {"python": "3.12", "platform": "linux", "packages": {}}, + "source_topology": {}, + "lineage": (), + "input_digests": (DIGEST_A,), + "output_digests": (DIGEST_B,), + "transformations": (), + "observed_scopes": ("model_weights",), + "lost_state": (), + } + values.update(overrides) + return build_provenance(**values) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"sources": ()}, "at least one source"), + ({"configuration_digest": "bad"}, "sha256 digest"), + ({"environment": {"python": "3", "platform": "linux", "packages": {}, "extra": 1}}, "unsupported fields"), + ({"source_topology": {"rank": 1 << 64}}, "outside int64"), + ({"source_topology": {"path": "/private/checkpoint"}}, "private local path"), + ({"source_topology": {"note": "hf_abcdefghijklmnopqrstuvwxyz"}}, "contains a secret"), + ({"source_topology": {1: "value"}}, "non-string key"), + ({"source_topology": {"prompt_text": "value"}}, "sensitive key"), + ({"source_topology": {"opaque": object()}}, "non-JSON value"), + ({"input_digests": ("bad",)}, "sha256 digest"), + ], +) +def test_provenance_builder_rejects_incomplete_or_unsafe_evidence(overrides, message): + with pytest.raises(ValueError, match=message): + _minimal_provenance(**overrides) + + +def test_normalization_accepts_explicit_json_scalars_and_sequences(): + record = _minimal_provenance( + source_topology={"active": True, "optional": None, "ranks": [0, 1]}, + ).to_dict() + + assert record["source_topology"] == { + "active": True, + "optional": None, + "ranks": [0, 1], + } + + +def test_command_redaction_covers_equals_and_bare_sensitive_values(): + sanitized = sanitize_command( + ( + "convert", + "--output=/private/output", + "--note=ok", + "--credential=secret-value", + "bearer abcdefghijklmnop", + ) + ) + + assert sanitized[1].startswith("--output=[LOCAL_PATH:sha256:") + assert sanitized[2] == "--note=ok" + assert sanitized[3] == "--credential=[REDACTED]" + assert sanitized[4] == "[REDACTED]" + + +def test_legacy_migration_marks_invalid_identity_and_ignores_invalid_dataset_rows(): + payload = migrate_legacy_metadata( + { + "model": "/private/model", + "model_revision": "hf_abcdefghijklmnopqrstuvwxyz", + "tokenizer_revision": 42, + "dataset_inputs": ["invalid", {"identifier": "/private", "sha256": "a" * 64}], + } + ).to_dict() + + assert payload["base_model"] == {"identity": None, "revision": None, "digest": None} + assert payload["tokenizer"]["revision"] is None + assert payload["datasets"] == [] + assert "base_model_identity" in payload["unknowns"] + + +def test_provenance_verifier_rejects_tampering_and_constructor_identity_disagreement(): + record = _record() + tampered = record.to_dict() + tampered["unknowns"] = ["changed"] + + with pytest.raises(ValueError, match="record digest mismatch"): + verify_provenance_record(tampered) + with pytest.raises(ValueError, match="identity fields disagree"): + ProvenanceRecord( + "artifact-sha256:" + "f" * 64, + record.record_digest, + record.to_json(), + ) + with pytest.raises(ValueError, match="JSON is not canonical"): + ProvenanceRecord( + record.artifact_id, + record.record_digest, + json.dumps(record.to_dict()), + ) + + +def test_public_metadata_bounds_mixed_keys_and_legacy_sensitive_values_fail_closed(): + with pytest.raises(ValueError, match="non-string key"): + _minimal_provenance(source_topology={"rank": 0, 1: "invalid"}) + with pytest.raises(ValueError, match="invalid key"): + _minimal_provenance(source_topology={"x" * 513: "invalid"}) + deeply_nested = {} + cursor = deeply_nested + for _ in range(18): + cursor["next"] = {} + cursor = cursor["next"] + with pytest.raises(ValueError, match="nesting is too deep"): + _minimal_provenance(source_topology=deeply_nested) + with pytest.raises(ValueError, match="command argument"): + _minimal_provenance(command=(object(),)) + + legacy = migrate_legacy_metadata( + { + "model": "org/base", + "model_revision": "/private/revision", + "tokenizer_revision": "hf_abcdefghijklmnopqrstuvwxyz", + "seed": "/private/seed", + "dataset_inputs": None, + } + ).to_dict() + rendered = json.dumps(legacy) + assert "/private" not in rendered + assert "hf_" not in rendered + assert legacy["seed"] is None + assert "seed" in legacy["unknowns"] + + +def test_public_provenance_parsers_reject_boundedness_and_structure_attacks(): + record = _record() + with pytest.raises(ValueError, match="provenance JSON is invalid"): + ProvenanceRecord(record.artifact_id, record.record_digest, '{"a": 1, "a": 2}') + with pytest.raises(ValueError, match="text is too large"): + _minimal_provenance(source_topology={"note": "x" * 1025}) + with pytest.raises(ValueError, match="too many fields"): + _minimal_provenance( + source_topology={f"field-{index}": index for index in range(4097)}, + ) + with pytest.raises(ValueError, match="too many items"): + _minimal_provenance(source_topology={"items": [None] * 4097}) + with pytest.raises(ValueError, match="must be an object"): + verify_provenance_record([]) + with pytest.raises(ValueError, match="bounded string collection"): + classify_resume_state("model_weights") + with pytest.raises(ValueError, match="collection is invalid"): + _minimal_provenance(input_digests=DIGEST_A) + with pytest.raises(ValueError, match="must contain strings"): + _minimal_provenance(input_digests=(1,)) + with pytest.raises(ValueError, match="sources collection"): + _minimal_provenance(sources="source") + with pytest.raises(ValueError, match="artifact identities"): + _minimal_provenance(sources=(object(),)) + with pytest.raises(ValueError, match="lineage events"): + _minimal_provenance(lineage=(object(),)) + with pytest.raises(ValueError, match="bounded mapping"): + migrate_legacy_metadata([]) + + +def test_provenance_verifier_rejects_each_identity_and_state_layer(): + payload = _record().to_dict() + + malformed = {**payload, "sources": []} + with pytest.raises(ValueError, match="structure is invalid"): + verify_provenance_record(malformed) + + malformed = {**payload, "artifact_id": "invalid"} + with pytest.raises(ValueError, match="artifact ID is invalid"): + verify_provenance_record(malformed) + + malformed = {**payload, "state": {}} + with pytest.raises(ValueError, match="state is invalid"): + verify_provenance_record(malformed) + + malformed = json.loads(json.dumps(payload)) + malformed["state"]["classification"] = "unknown" + with pytest.raises(ValueError, match="classification is not evidence-derived"): + verify_provenance_record(malformed) + + malformed = {**payload, "artifact_id": "artifact-sha256:" + "f" * 64} + digest_input = {key: value for key, value in malformed.items() if key != "record_digest"} + encoded = json.dumps( + digest_input, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + malformed["record_digest"] = f"sha256:{sha256(encoded).hexdigest()}" + with pytest.raises(ValueError, match="artifact ID mismatch"): + verify_provenance_record(malformed) + + +def test_command_and_legacy_scalar_paths_remain_public_and_canonical(): + assert sanitize_command(("convert", f"--note=hf_{'a' * 20}")) == ( + "convert", + "--note=[REDACTED]", + ) + assert migrate_legacy_metadata({"seed": 7}).to_dict()["seed"] == "7" + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda value: value["sources"][0].update(kind="invalid"), "source identity"), + (lambda value: value["converter"].update(commit="invalid"), "converter"), + (lambda value: value["command"].append("/private/source"), "command"), + (lambda value: value["environment"].update(packages=[]), "packages"), + (lambda value: value.update(source_topology=[]), "source topology"), + (lambda value: value["lineage"][0].update(extra=True), "lineage event fields"), + (lambda value: value.update(input_digests=[]), "input digests"), + ( + lambda value: value.update(output_digests=[DIGEST_B, DIGEST_B]), + "output digests", + ), + (lambda value: value["transformations"].reverse(), "transformations"), + ( + lambda value: value["state"].update( + observed_scopes=["model_weights", "model_weights"] + ), + "observed scopes", + ), + (lambda value: value["adapter"].update(config_digest="invalid"), "adapter identity"), + (lambda value: value["dataset"].update(record_count=True), "dataset identity"), + (lambda value: value["training"].update(framework=[]), "training framework"), + (lambda value: value.update(unknowns=["duplicate", "duplicate"]), "unknowns"), + ], +) +def test_provenance_verifier_rejects_noncanonical_nested_records(mutation, message): + payload = deepcopy(_record().to_dict()) + mutation(payload) + + with pytest.raises(ValueError, match=message): + verify_provenance_record(payload) + + +def test_builder_cannot_emit_schema_invalid_environment_or_duplicate_lineage(): + with pytest.raises(ValueError, match="environment python"): + _minimal_provenance( + environment={"python": [], "platform": "linux", "packages": {}}, + ) + + event = LineageEvent("event", "consolidation", (), "writer@1", ()) + with pytest.raises(ValueError, match="sorted and unique"): + _minimal_provenance(lineage=(event, event)) diff --git a/tests/test_checkpoint_service.py b/tests/test_checkpoint_service.py new file mode 100644 index 0000000..cafd9ef --- /dev/null +++ b/tests/test_checkpoint_service.py @@ -0,0 +1,29 @@ +"""Public producer-neutral checkpoint service contracts.""" + +from __future__ import annotations + +import torch +from safetensors.torch import save_file + +from obliteratus.checkpoint_capabilities import AdapterRegistry +from obliteratus.checkpoint_inspection import InspectionLimits +from obliteratus.checkpoint_service import CheckpointService + + +def test_service_inspects_canonical_checkpoint_with_injected_limits(tmp_path): + save_file({"weight": torch.ones(2)}, tmp_path / "model.safetensors") + service = CheckpointService(inspection_limits=InspectionLimits(max_files=4)) + + report = service.inspect(tmp_path) + + assert report.primary_format == "hf_safetensors" + assert report.support_decision == "canonical_hf_ready" + + +def test_service_has_no_trusted_reader_adapter_or_conversion_entrypoint(): + service = CheckpointService() + + assert service.adapter_registry == AdapterRegistry() + assert not hasattr(service, "trusted_inspect") + assert not hasattr(service, "load_vendor_checkpoint") + assert not hasattr(service, "convert") diff --git a/tests/test_checkpoint_writer.py b/tests/test_checkpoint_writer.py new file mode 100644 index 0000000..959b9b0 --- /dev/null +++ b/tests/test_checkpoint_writer.py @@ -0,0 +1,779 @@ +"""Bounded deterministic safetensors writer and transactional failure tests.""" + +from __future__ import annotations + +import json +from hashlib import sha256 +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from jsonschema import Draft202012Validator +from safetensors.torch import load_file + +import obliteratus.checkpoint_writer as writer_module +from obliteratus.checkpoint_errors import CheckpointContractError +from obliteratus.checkpoint_fixtures import load_fixture_case +from obliteratus.checkpoint_provenance import ( + ArtifactIdentity, + LineageEvent, + ToolIdentity, + build_provenance, +) +from obliteratus.checkpoint_writer import ( + ImmutableCopy, + VerifiedSourceFile, + WriterLimits, + write_canonical_checkpoint, +) + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests/fixtures/distributed_checkpoints/v1/cases" +MANIFEST_SCHEMA = json.loads( + (ROOT / "docs/checkpoints/schemas/conversion-manifest-v1.schema.json").read_text() +) +COMMIT = "c" * 40 + + +def _digest_bytes(payload: bytes) -> str: + return f"sha256:{sha256(payload).hexdigest()}" + + +def _digest_file(path: Path) -> str: + return _digest_bytes(path.read_bytes()) + + +def _tree(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): _digest_file(path) + for path in sorted(item for item in root.rglob("*") if item.is_file()) + } + + +def _inputs( + tmp_path: Path, + case_name: str = "world1-complete", + *, + source_topology: dict | None = None, +): + case_root = FIXTURES / case_name + case = load_fixture_case(case_root) + sources = tuple( + VerifiedSourceFile( + path=path, + relative_path=f"fixture/{case_name}/{path.name}", + expected_sha256=_digest_file(path), + ) + for path in sorted(item for item in case_root.iterdir() if item.is_file()) + ) + copy_root = tmp_path / "immutable-base" + copy_root.mkdir(parents=True) + config = copy_root / "config.json" + tokenizer = copy_root / "tokenizer_config.json" + config.write_text('{"architectures":["TinyModel"],"vocab_size":8}\n') + tokenizer.write_text('{"model_max_length":128,"tokenizer_class":"Tiny"}\n') + copies = ( + ImmutableCopy( + relative_path="config.json", + source_path=config, + expected_sha256=_digest_file(config), + kind="configuration", + ), + ImmutableCopy( + relative_path="tokenizer_config.json", + source_path=tokenizer, + expected_sha256=_digest_file(tokenizer), + kind="tokenizer", + ), + ) + input_digests = tuple(item.expected_sha256 for item in sources) + + def provenance_factory(output_digests: tuple[str, ...]): + return build_provenance( + sources=( + ArtifactIdentity( + "local", + f"synthetic-fixture-{case_name}", + "v1", + _digest_bytes("".join(sorted(input_digests)).encode()), + ), + ), + converter=ToolIdentity("obliteratus-neutral-writer", "1.0.0", COMMIT), + obliteratus_commit=COMMIT, + configuration_digest=copies[0].expected_sha256, + tokenizer=ArtifactIdentity( + "generated", + "fixture-tokenizer", + "v1", + copies[1].expected_sha256, + ), + base_model=ArtifactIdentity( + "generated", + "fixture-base-model", + "v1", + copies[0].expected_sha256, + ), + command=("checkpoint", "write", f"fixture:{case_name}"), + environment={"python": "test", "platform": "cpu", "packages": {}}, + source_topology=source_topology or {}, + lineage=( + LineageEvent( + "event-consolidate", + "consolidation", + (), + "obliteratus-neutral-writer@1.0.0", + ("canonical_safetensors",), + ), + ), + input_digests=input_digests, + output_digests=output_digests, + transformations=("canonical_safetensors", "consolidation"), + observed_scopes=("model_weights",), + lost_state=("optimizer_state", "scheduler_state", "rng_state"), + ) + + return case, sources, copies, provenance_factory + + +def _write(tmp_path: Path, destination: Path, **kwargs): + source_topology = {"world_size": 1} + case, sources, copies, provenance_factory = _inputs( + tmp_path, + kwargs.pop("case_name", "world1-complete"), + source_topology=source_topology, + ) + result = write_canonical_checkpoint( + destination, + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology=source_topology, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + **kwargs, + ) + return case, result + + +def test_writer_is_deterministic_sharded_reloadable_and_contract_valid(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + case, first_result = _write( + tmp_path / "one", + first, + limits=WriterLimits(max_shard_bytes=32), + ) + _, second_result = _write( + tmp_path / "two", + second, + limits=WriterLimits(max_shard_bytes=32), + ) + + assert _tree(first) == _tree(second) + assert first_result.artifact_id == second_result.artifact_id + assert first_result.output_path == first.resolve() + assert not (first / "pytorch_model.bin").exists() + index = json.loads((first / "model.safetensors.index.json").read_text()) + assert list(index["weight_map"]) == sorted(index["weight_map"]) + assert len(set(index["weight_map"].values())) > 1 + loaded = {} + for shard in sorted(set(index["weight_map"].values())): + loaded.update(load_file(first / shard, device="cpu")) + for logical_id, oracle in case.tensor_oracles.items(): + assert torch.equal(loaded[logical_id], oracle.values) + assert str(loaded[logical_id].dtype).removeprefix("torch.") == oracle.dtype + manifest = json.loads((first / "conversion-manifest.json").read_text()) + Draft202012Validator(MANIFEST_SCHEMA).validate(manifest) + provenance = json.loads((first / "checkpoint-provenance.json").read_text()) + metadata = json.loads((first / "abliteration_metadata.json").read_text()) + assert manifest["manifest_id"] == provenance["artifact_id"] == metadata["artifact_id"] + assert provenance["artifact_id"] == first_result.artifact_id + assert manifest["validation"] == { + "coverage": True, + "replicas": True, + "ties": True, + "hashes": True, + "index": True, + "safe_reload": True, + "source_unchanged": True, + "result": "passed", + } + assert manifest["resource_usage"]["actual_peak_ram_bytes"] is None + assert manifest["resource_usage"]["actual_temp_bytes"] is None + + +def test_single_shard_uses_canonical_direct_filename(tmp_path): + case, result = _write( + tmp_path / "work", + tmp_path / "output", + case_name="world2-uneven-1d", + limits=WriterLimits(max_shard_bytes=1024), + ) + + assert result.weight_files == ("model.safetensors",) + assert not (result.output_path / "model.safetensors.index.json").exists() + loaded = load_file(result.output_path / "model.safetensors", device="cpu") + assert torch.equal(loaded["model.weight"], case.tensor_oracles["model.weight"].values) + + +def test_writer_refuses_ties_without_an_explicit_validated_policy(tmp_path): + case, sources, copies, provenance_factory = _inputs(tmp_path) + + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + tmp_path / "output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + ) + + assert caught.value.code == "DCI_VALIDATION_FAILED" + assert caught.value.detail == "tie_policy_required" + + +def test_admission_denial_creates_no_staging_or_output(tmp_path, monkeypatch): + case, sources, copies, provenance_factory = _inputs(tmp_path) + destination = tmp_path / "output" + + def forbidden_transaction(*_args, **_kwargs): + raise AssertionError("staging began before admission") + + monkeypatch.setattr(writer_module, "atomic_checkpoint_directory", forbidden_transaction) + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + destination, + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + limits=WriterLimits(max_output_bytes=1), + ) + + assert caught.value.code == "DCI_ADMISSION_DENIED" + assert caught.value.detail == "max_output_bytes" + assert not destination.exists() + assert list(tmp_path.glob(".output.staging-*")) == [] + + +def test_copy_and_source_digests_are_verified_before_staging(tmp_path): + case, sources, copies, provenance_factory = _inputs(tmp_path) + copies[0].source_path.write_text("changed\n") + + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + tmp_path / "output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + ) + + assert caught.value.code == "DCI_SOURCE_CHANGED" + assert not (tmp_path / "output").exists() + + +def test_enospc_preserves_source_and_prior_output_and_cleans_staging(tmp_path, monkeypatch): + case, sources, copies, provenance_factory = _inputs(tmp_path) + destination = tmp_path / "output" + destination.mkdir() + sentinel = destination / "sentinel" + sentinel.write_text("prior") + source_before = {item.relative_path: _digest_file(item.path) for item in sources} + + def enospc(_tensors, path): + path.write_bytes(b"partial") + raise OSError(28, "No space left on device") + + monkeypatch.setattr(writer_module, "_save_safetensors_file", enospc) + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + destination, + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + ) + + assert caught.value.code == "DCI_MATERIALIZE_FAILED" + assert sentinel.read_text() == "prior" + assert {item.relative_path: _digest_file(item.path) for item in sources} == source_before + assert list(tmp_path.glob(".output.staging-*")) == [] + assert list(tmp_path.glob(".output.backup-*")) == [] + + +def test_cancellation_preserves_prior_output_and_cleans_staging(tmp_path, monkeypatch): + class Cancelled(BaseException): + pass + + case, sources, copies, provenance_factory = _inputs(tmp_path) + destination = tmp_path / "output" + destination.mkdir() + (destination / "sentinel").write_text("prior") + + def cancel(_tensors, _path): + raise Cancelled() + + monkeypatch.setattr(writer_module, "_save_safetensors_file", cancel) + with pytest.raises(Cancelled): + write_canonical_checkpoint( + destination, + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + ) + + assert (destination / "sentinel").read_text() == "prior" + assert list(tmp_path.glob(".output.staging-*")) == [] + + +def test_postwrite_validation_failure_never_promotes(tmp_path, monkeypatch): + case, sources, copies, provenance_factory = _inputs(tmp_path) + destination = tmp_path / "output" + destination.mkdir() + (destination / "sentinel").write_text("prior") + + def reject(*_args, **_kwargs): + raise CheckpointContractError( + "DCI_VALIDATION_FAILED", + detail="injected_postwrite_failure", + ) + + monkeypatch.setattr(writer_module, "_verify_staging", reject) + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + destination, + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + ) + + assert caught.value.detail == "injected_postwrite_failure" + assert (destination / "sentinel").read_text() == "prior" + assert list(tmp_path.glob(".output.staging-*")) == [] + + +@pytest.mark.parametrize("target", ["index", "manifest"]) +def test_corrupt_index_or_manifest_is_rejected_before_promotion( + tmp_path, + monkeypatch, + target, +): + case, sources, copies, provenance_factory = _inputs( + tmp_path / "inputs", + source_topology={}, + ) + original = writer_module._write_json + + def corrupt(path, value): + if target == "index" and path.name == "model.safetensors.index.json": + value = {"metadata": {"total_size": 0}, "weight_map": value["weight_map"]} + if target == "manifest" and path.name == "conversion-manifest.json": + value = {**value, "source_topology": {"tampered": True}} + return original(path, value) + + monkeypatch.setattr(writer_module, "_write_json", corrupt) + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + tmp_path / "output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=provenance_factory, + tie_policy="duplicate_validated", + limits=WriterLimits(max_shard_bytes=32), + ) + assert caught.value.code == "DCI_VALIDATION_FAILED" + assert caught.value.detail == ( + "output_index_mismatch" if target == "index" else "output_manifest_mismatch" + ) + assert not (tmp_path / "output").exists() + + +def _invoke(tmp_path, *, source_files=None, copies=None, factory=None, **kwargs): + source_topology = {"world_size": 1} + case, default_sources, default_copies, default_factory = _inputs( + tmp_path / "inputs", + source_topology=source_topology, + ) + return write_canonical_checkpoint( + tmp_path / "output", + case.fragments, + source_files=default_sources if source_files is None else source_files, + copies=default_copies if copies is None else copies, + descriptor_digest=case.expected_manifest_digest, + source_topology=source_topology, + provenance_factory=default_factory if factory is None else factory, + tie_policy="duplicate_validated", + **kwargs, + ) + + +def test_writer_limit_and_input_records_validate_before_io(tmp_path): + with pytest.raises(ValueError, match="positive integer"): + WriterLimits(max_shard_bytes=0) + with pytest.raises(ValueError, match="cannot exceed 100"): + WriterLimits(min_free_headroom_percent=101) + with pytest.raises(ValueError, match="relative path is unsafe"): + VerifiedSourceFile(tmp_path / "source", "../source", "sha256:" + "0" * 64) + with pytest.raises(ValueError, match="sha256 digest"): + VerifiedSourceFile(tmp_path / "source", "source", "bad") + with pytest.raises(ValueError, match="root-level"): + ImmutableCopy("nested/config.json", tmp_path / "config", "sha256:" + "0" * 64, "configuration") + with pytest.raises(ValueError, match="configuration or tokenizer"): + ImmutableCopy("config.json", tmp_path / "config", "sha256:" + "0" * 64, "other") + + +def test_writer_rejects_missing_duplicate_or_incomplete_evidence(tmp_path): + with pytest.raises(CheckpointContractError) as empty: + _invoke(tmp_path / "empty", source_files=()) + assert empty.value.detail == "source_inventory_empty" + + case, sources, copies, factory = _inputs(tmp_path / "duplicate-inputs") + duplicate_source = (*sources, sources[0]) + with pytest.raises(ValueError, match="source relative paths must be unique"): + write_canonical_checkpoint( + tmp_path / "duplicate-source-output", + case.fragments, + source_files=duplicate_source, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + + with pytest.raises(ValueError, match="copy output paths must be unique"): + write_canonical_checkpoint( + tmp_path / "duplicate-copy-output", + case.fragments, + source_files=sources, + copies=(*copies, copies[0]), + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + + with pytest.raises(CheckpointContractError) as missing_copy: + write_canonical_checkpoint( + tmp_path / "missing-copy-output", + case.fragments, + source_files=sources, + copies=(copies[0],), + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert missing_copy.value.detail == "canonical_config_or_tokenizer_missing" + + +@pytest.mark.parametrize( + ("limits", "detail"), + [ + (WriterLimits(max_temp_bytes=1), "max_temp_bytes"), + (WriterLimits(max_peak_ram_bytes=1), "max_peak_ram_bytes"), + ], +) +def test_every_writer_admission_limit_fails_before_staging(tmp_path, limits, detail): + with pytest.raises(CheckpointContractError) as caught: + _invoke(tmp_path, limits=limits) + assert caught.value.code == "DCI_ADMISSION_DENIED" + assert caught.value.detail == detail + assert not (tmp_path / "output").exists() + + +def test_tensor_larger_than_configured_shard_limit_is_refused(tmp_path): + with pytest.raises(CheckpointContractError) as caught: + _invoke(tmp_path, limits=WriterLimits(max_shard_bytes=1)) + assert caught.value.code == "DCI_ADMISSION_DENIED" + assert caught.value.detail == "tensor_exceeds_max_shard_bytes" + assert not (tmp_path / "output").exists() + + +def test_post_write_size_limit_is_rechecked_before_promotion(tmp_path, monkeypatch): + monkeypatch.setattr( + writer_module, + "_admit", + lambda *_args, **_kwargs: writer_module._Admission(0, 0, 0, 0), + ) + with pytest.raises(CheckpointContractError) as caught: + _invoke( + tmp_path, + limits=WriterLimits(max_output_bytes=1, max_temp_bytes=1), + ) + assert caught.value.code == "DCI_ADMISSION_DENIED" + assert caught.value.detail == "post_write_size_limit" + assert not (tmp_path / "output").exists() + + +def test_filesystem_admission_and_destination_symlink_fail_closed(tmp_path, monkeypatch): + monkeypatch.setattr(writer_module.shutil, "disk_usage", lambda _path: SimpleNamespace(free=0)) + with pytest.raises(CheckpointContractError) as capacity: + _invoke(tmp_path / "capacity") + assert capacity.value.detail == "filesystem_free_bytes" + + parent_target = tmp_path / "parent-target" + parent_target.mkdir() + parent_link = tmp_path / "parent-link" + parent_link.symlink_to(parent_target, target_is_directory=True) + case, sources, copies, factory = _inputs(tmp_path / "symlink-inputs") + with pytest.raises(CheckpointContractError) as boundary: + write_canonical_checkpoint( + parent_link / "output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert boundary.value.detail == "destination_symlink" + + +def test_source_ancestor_symlink_and_open_race_fail_before_staging(tmp_path, monkeypatch): + actual = tmp_path / "actual" + case, sources, copies, factory = _inputs(actual) + alias = tmp_path / "alias" + alias.symlink_to(actual, target_is_directory=True) + aliased_sources = ( + VerifiedSourceFile( + alias / copies[0].source_path.relative_to(actual), + "source/config.json", + copies[0].expected_sha256, + ), + ) + + with pytest.raises(CheckpointContractError) as symlink: + write_canonical_checkpoint( + tmp_path / "symlink-output", + case.fragments, + source_files=aliased_sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert symlink.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION" + assert symlink.value.detail == "source_symlink" + + real_open = writer_module.os.open + blocked_path = sources[0].path + + def fail_observed_open(path, flags): + if Path(path) == blocked_path: + raise OSError("injected source replacement") + return real_open(path, flags) + + monkeypatch.setattr(writer_module.os, "open", fail_observed_open) + with pytest.raises(CheckpointContractError) as changed: + write_canonical_checkpoint( + tmp_path / "race-output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert changed.value.code == "DCI_SOURCE_CHANGED" + assert not list(tmp_path.glob(".race-output.staging-*")) + + +def test_source_missing_and_nonregular_inputs_fail_before_staging(tmp_path): + case, sources, copies, factory = _inputs(tmp_path / "source-inputs") + sources = ( + VerifiedSourceFile( + tmp_path / "missing-source", + sources[0].relative_path, + sources[0].expected_sha256, + ), + *sources[1:], + ) + with pytest.raises(CheckpointContractError) as missing: + write_canonical_checkpoint( + tmp_path / "missing-output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert missing.value.code == "DCI_SOURCE_CHANGED" + + case, sources, copies, factory = _inputs(tmp_path / "directory-inputs") + source_directory = tmp_path / "source-directory" + source_directory.mkdir() + sources = ( + VerifiedSourceFile( + source_directory, + sources[0].relative_path, + sources[0].expected_sha256, + ), + *sources[1:], + ) + with pytest.raises(CheckpointContractError) as nonregular: + write_canonical_checkpoint( + tmp_path / "directory-output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology={}, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert nonregular.value.detail == "source_not_regular_file" + + +@pytest.mark.parametrize( + ("record", "detail"), + [ + ({}, "provenance_contract_invalid"), + ( + { + "schema_id": "obliteratus.artifact-provenance", + "output_digests": [], + "record_digest": "bad", + }, + "provenance_contract_invalid", + ), + ], +) +def test_provenance_contract_failures_never_promote(tmp_path, record, detail): + fake = SimpleNamespace(artifact_id="artifact-sha256:" + "0" * 64, to_dict=lambda: record) + with pytest.raises(CheckpointContractError) as caught: + _invoke(tmp_path, factory=lambda _digests: fake) + assert caught.value.code == "DCI_EVIDENCE_UNAVAILABLE" + assert caught.value.detail == detail + assert not (tmp_path / "output").exists() + + +@pytest.mark.parametrize( + ("mismatch", "detail"), + [ + ("input", "provenance_input_digest_mismatch"), + ("configuration", "provenance_configuration_digest_mismatch"), + ("topology", "provenance_source_topology_mismatch"), + ], +) +def test_writer_binds_source_config_and_topology_to_provenance( + tmp_path, + mismatch, + detail, +): + source_topology = {"world_size": 1} + case, sources, copies, _ = _inputs( + tmp_path / "inputs", + source_topology=source_topology, + ) + source_digests = tuple(item.expected_sha256 for item in sources) + + def factory(output_digests): + return build_provenance( + sources=(ArtifactIdentity("local", "fixture", "v1", source_digests[0]),), + converter=ToolIdentity("writer", "1", COMMIT), + obliteratus_commit=COMMIT, + configuration_digest=( + "sha256:" + "f" * 64 + if mismatch == "configuration" + else copies[0].expected_sha256 + ), + tokenizer=None, + base_model=None, + command=("checkpoint", "write"), + environment={"python": "test", "platform": "cpu", "packages": {}}, + source_topology=( + {"world_size": 2} if mismatch == "topology" else source_topology + ), + lineage=(), + input_digests=( + ("sha256:" + "e" * 64,) + if mismatch == "input" + else source_digests + ), + output_digests=output_digests, + transformations=("canonical_safetensors",), + observed_scopes=("model_weights",), + lost_state=(), + ) + + with pytest.raises(CheckpointContractError) as caught: + write_canonical_checkpoint( + tmp_path / "output", + case.fragments, + source_files=sources, + copies=copies, + descriptor_digest=case.expected_manifest_digest, + source_topology=source_topology, + provenance_factory=factory, + tie_policy="duplicate_validated", + ) + assert caught.value.code == "DCI_EVIDENCE_UNAVAILABLE" + assert caught.value.detail == detail + assert not (tmp_path / "output").exists() + + +def test_provenance_factory_and_promotion_errors_are_stable(tmp_path, monkeypatch): + with pytest.raises(CheckpointContractError) as factory_error: + _invoke(tmp_path / "factory", factory=lambda _digests: (_ for _ in ()).throw(RuntimeError("boom"))) + assert factory_error.value.detail == "provenance_factory_failed" + + def promotion_failure(*_args, **_kwargs): + raise OSError("promotion unavailable") + + monkeypatch.setattr(writer_module, "atomic_checkpoint_directory", promotion_failure) + with pytest.raises(CheckpointContractError) as promotion: + _invoke(tmp_path / "promotion") + assert promotion.value.code == "DCI_PROMOTION_FAILED" + + +def test_output_json_and_file_record_validation_rejects_unsafe_artifacts(tmp_path): + missing = tmp_path / "missing.json" + with pytest.raises(CheckpointContractError, match="output_json_invalid"): + writer_module._verify_json_object(missing) + + malformed = tmp_path / "malformed.json" + malformed.write_text("{", encoding="utf-8") + with pytest.raises(CheckpointContractError, match="output_json_invalid"): + writer_module._verify_json_object(malformed) + + array = tmp_path / "array.json" + array.write_text("[]", encoding="utf-8") + with pytest.raises(CheckpointContractError, match="output_json_invalid"): + writer_module._verify_json_object(array) + + directory = tmp_path / "directory" + directory.mkdir() + with pytest.raises(CheckpointContractError, match="output_not_regular_file"): + writer_module._output_record(directory) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2f15643..70e755f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -144,6 +144,58 @@ class TestCLIDispatch: main(["interactive"]) mock_cmd.assert_called_once() + def test_distributed_preflight_is_an_explicit_separate_command(self, tmp_path): + profile = tmp_path / "profile.json" + profile.write_text("{}", encoding="utf-8") + with patch("obliteratus.cli._cmd_distributed") as mock_cmd: + main(["distributed", "preflight", str(profile), "--json"]) + args_passed = mock_cmd.call_args.args[0] + assert args_passed.command == "distributed" + assert args_passed.distributed_command == "preflight" + assert args_passed.profile == profile + + def test_ordinary_command_never_infers_distributed_mode(self, monkeypatch): + monkeypatch.setenv("WORLD_SIZE", "2") + monkeypatch.setenv("RANK", "0") + monkeypatch.setenv("MASTER_ADDR", "10.10.0.10") + with ( + patch("obliteratus.cli._cmd_abliterate") as ordinary, + patch("obliteratus.cli._cmd_distributed") as distributed, + ): + main(["obliterate", "fake/model"]) + ordinary.assert_called_once() + distributed.assert_not_called() + + @pytest.mark.parametrize("option", ["--token", "--password", "--api-key", "--remote"]) + def test_distributed_preflight_rejects_unknown_options_without_echoing_value( + self, option, capsys + ): + secret = "hf_private_value_that_must_not_appear" + with pytest.raises(SystemExit) as error: + main(["distributed", "preflight", "profile.json", option, secret]) + assert error.value.code == 2 + captured = capsys.readouterr() + assert secret not in captured.out + assert secret not in captured.err + + @pytest.mark.parametrize( + "argv", + [ + ["--token", "{secret}", "distributed", "preflight", "profile.json"], + ["distributed", "preflight", "profile.json", "--token={secret}"], + ], + ) + def test_distributed_preflight_secret_prescan_cannot_be_bypassed( + self, argv, capsys + ): + secret = "hf_private_value_that_must_not_appear" + with pytest.raises(SystemExit) as error: + main([item.replace("{secret}", secret) for item in argv]) + assert error.value.code == 2 + captured = capsys.readouterr() + assert secret not in captured.out + assert secret not in captured.err + # 9. --contribute and --contribute-notes are accepted on obliterate def test_contribute_flags_on_obliterate(self): """Verify --contribute and --contribute-notes are accepted args.""" diff --git a/tests/test_cli_boundaries.py b/tests/test_cli_boundaries.py index 2ac834b..0133107 100644 --- a/tests/test_cli_boundaries.py +++ b/tests/test_cli_boundaries.py @@ -9,6 +9,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, Mock import pytest +import torch from obliteratus import cli @@ -21,6 +22,7 @@ def ns(**values): ("argv", "target"), [ (["gpu-calc", "--params", "1"], "_cmd_gpu_calc"), + (["checkpoint", "inspect", "local/checkpoint"], "_cmd_checkpoint"), (["run", "config.yml"], "_cmd_run"), (["interactive"], "_cmd_interactive"), (["models"], "_cmd_models"), @@ -144,6 +146,74 @@ def test_version_is_stable_and_does_not_dispatch(capsys): assert capsys.readouterr().out.endswith(f"obliteratus {__version__}\n") +def test_checkpoint_inspect_json_is_machine_readable_without_banner(tmp_path, capsys): + from safetensors.torch import save_file + + save_file({"weight": torch.ones(1)}, tmp_path / "model.safetensors") + + cli.main(["checkpoint", "inspect", str(tmp_path), "--json"]) + + payload = json.loads(capsys.readouterr().out) + assert payload["schema_id"] == "obliteratus.checkpoint-descriptor" + assert payload["primary_format"] == "hf_safetensors" + assert payload["safety"]["inspection_level"] == "safe_structure" + + +def test_checkpoint_inspect_boundary_error_is_stable_json(tmp_path, capsys): + target = tmp_path / "target" + target.write_bytes(b"payload") + (tmp_path / "model.safetensors").symlink_to(target.name) + + with pytest.raises(SystemExit) as caught: + cli.main(["checkpoint", "inspect", str(tmp_path), "--json"]) + + assert caught.value.code == 2 + payload = json.loads(capsys.readouterr().out) + assert payload["code"] == "DCI_SOURCE_BOUNDARY_VIOLATION" + assert payload["detail"] == "source_symlink" + assert str(tmp_path) not in json.dumps(payload) + + +def test_checkpoint_inspect_human_output_honors_explicit_limits(tmp_path, capsys): + cli.main( + [ + "checkpoint", + "inspect", + str(tmp_path), + "--max-files", + "10", + "--max-total-bytes", + "1024", + "--max-json-bytes", + "512", + "--max-header-bytes", + "256", + ] + ) + + output = capsys.readouterr().out + assert "format" in output + assert "confidence" in output + assert "unknown" in output + assert "blocked" in output + assert "DCI_UNSUPPORTED_FORMAT_OR_VERSION" in output + + +def test_checkpoint_inspect_human_boundary_error_is_actionable(tmp_path, capsys): + target = tmp_path / "target" + target.write_bytes(b"payload") + (tmp_path / "model.safetensors").symlink_to(target.name) + + with pytest.raises(SystemExit) as caught: + cli.main(["checkpoint", "inspect", str(tmp_path)]) + + assert caught.value.code == 2 + output = capsys.readouterr().out + assert "DCI_SOURCE_BOUNDARY_VIOLATION" in output + assert "source_symlink" in output + assert "Repair the immutable local source boundary" in output + + def test_gpu_selection_contract(monkeypatch): monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False) cli._apply_gpu_selection(ns(gpus=None, remote=None)) diff --git a/tests/test_distributed_contracts.py b/tests/test_distributed_contracts.py new file mode 100644 index 0000000..d9eced2 --- /dev/null +++ b/tests/test_distributed_contracts.py @@ -0,0 +1,425 @@ +"""Pure tests for bounded distributed runtime records and frames.""" + +from __future__ import annotations + +from dataclasses import FrozenInstanceError +from enum import Enum + +import pytest +import torch + +from obliteratus.distributed.consensus import ( + assert_rank_order, + decode_frame, + encode_frame, + gloo_all_gather_records, + require_consensus_digest, + require_record_consensus, + unanimous_vote, +) +from obliteratus.distributed.contracts import ( + MAX_CONSENSUS_BYTES, + ContractError, + LogicalPlacement, + PlacementKind, + RankInventory, + RunIdentity, + RuntimeStage, + StageMessage, + TopologyPlan, + Vote, + advance_stage, + canonical_record, + contract_digest, + validate_inventory_consensus, +) +from obliteratus.distributed.numerical import distributed_project_weight + + +def _digest(character: str = "a") -> str: + return character * 64 + + +def _identity(**overrides: object) -> RunIdentity: + values: dict[str, object] = { + "run_id": "1" * 32, + "config_digest": _digest("a"), + "source_digest": _digest("b"), + "model_digest": _digest("c"), + "tokenizer_digest": _digest("d"), + "commit_sha": "e" * 40, + "world_size": 2, + } + values.update(overrides) + return RunIdentity(**values) # type: ignore[arg-type] + + +def _inventory(rank: int = 0, **overrides: object) -> RankInventory: + values: dict[str, object] = { + "rank": rank, + "local_rank": rank, + "world_size": 2, + "host_digest": _digest(str(rank + 1)), + "device_digest": _digest(chr(ord("a") + rank)), + "device_kind": "cpu", + "total_memory_bytes": 1024, + "free_memory_bytes": 512, + "software_digest": _digest("e"), + "storage_digest": _digest("f"), + } + values.update(overrides) + return RankInventory(**values) # type: ignore[arg-type] + + +def test_contract_records_are_immutable_and_canonical(): + identity = _identity() + with pytest.raises(FrozenInstanceError): + identity.world_size = 3 # type: ignore[misc] + first = canonical_record({"z": 1, "identity": identity, "items": (Vote.ABORT,)}) + second = canonical_record({"items": ["abort"], "identity": identity, "z": 1}) + assert first == second + assert contract_digest({"value": 1}) == contract_digest({"value": 1}) + assert len(contract_digest(identity)) == 64 + + +def test_identity_and_topology_fields_are_digest_bound(): + identity = _identity() + for field, value in ( + ("run_id", "2" * 32), + ("config_digest", _digest("1")), + ("source_digest", _digest("2")), + ("model_digest", _digest("3")), + ("tokenizer_digest", _digest("4")), + ): + assert contract_digest(identity) != contract_digest(_identity(**{field: value})) + topology = TopologyPlan(2, 0, "gloo", _digest("5")) + assert contract_digest(topology) != contract_digest(TopologyPlan(2, 1, "gloo", _digest("5"))) + assert contract_digest(topology) != contract_digest(TopologyPlan(2, 0, "nccl", _digest("5"))) + + +def test_public_records_reject_unknown_fields(): + with pytest.raises(TypeError, match="unexpected keyword"): + _identity(unknown="value") + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"run_id": "not-a-run"}, "run_id has an invalid format"), + ({"config_digest": "A" * 64}, "config_digest has an invalid format"), + ({"commit_sha": "e" * 39}, "commit_sha has an invalid format"), + ({"world_size": True}, "world_size must be an integer"), + ({"world_size": 1}, "world_size must be between 2 and 4096"), + ], +) +def test_run_identity_rejects_malformed_or_single_rank_values(overrides, message): + with pytest.raises(ContractError, match=message): + _identity(**overrides) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"rank": 2}, "rank must be between 0 and 1"), + ({"local_rank": -1}, "local_rank must be between 0 and 1"), + ({"device_kind": "mps"}, "device_kind must be 'cpu' or 'cuda'"), + ({"free_memory_bytes": 2048}, "free_memory_bytes cannot exceed"), + ({"total_memory_bytes": 0}, "total_memory_bytes must be between"), + ({"host_digest": "x" * 64}, "host_digest has an invalid format"), + ], +) +def test_rank_inventory_is_bounded(overrides, message): + with pytest.raises(ContractError, match=message): + _inventory(**overrides) + + +def test_complete_homogeneous_inventory_is_accepted(): + validate_inventory_consensus(_identity(), (_inventory(0), _inventory(1))) + + +@pytest.mark.parametrize( + ("records", "message"), + [ + ((_inventory(0),), "exactly one record per rank"), + ((_inventory(0), _inventory(0)), "ranks do not exactly cover"), + ( + (_inventory(0), _inventory(1, world_size=3)), + "world_size disagrees", + ), + ( + (_inventory(0), _inventory(1, host_digest=_digest("1"), device_digest=_digest("a"))), + "unique host/device pair", + ), + ( + (_inventory(0), _inventory(1, host_digest=_digest("1"), local_rank=0)), + "local ranks must be unique", + ), + ( + (_inventory(0), _inventory(1, software_digest=_digest("0"))), + "software identities disagree", + ), + ( + (_inventory(0), _inventory(1, storage_digest=_digest("0"))), + "storage identities disagree", + ), + ], +) +def test_inventory_consensus_rejects_missing_duplicate_or_divergent_records(records, message): + with pytest.raises(ContractError, match=message): + validate_inventory_consensus(_identity(), records) + + +def test_inventory_consensus_rejects_wrong_record_types(): + with pytest.raises(ContractError, match="identity must"): + validate_inventory_consensus(object(), ()) # type: ignore[arg-type] + with pytest.raises(ContractError, match="invalid rank record"): + validate_inventory_consensus(_identity(), (_inventory(0), object())) # type: ignore[arg-type] + + +@pytest.mark.parametrize("backend", ["gloo", "nccl"]) +def test_topology_accepts_declared_backends(backend): + assert TopologyPlan(2, 0, backend, _digest()).backend == backend + + +@pytest.mark.parametrize( + ("args", "message"), + [ + ((1, 0, "gloo", _digest()), "world_size"), + ((2, 2, "gloo", _digest()), "coordinator_rank"), + ((2, 0, "mpi", _digest()), "backend must"), + ((2, 0, "gloo", "bad"), "placement_plan_digest"), + ], +) +def test_topology_rejects_unqualified_values(args, message): + with pytest.raises(ContractError, match=message): + TopologyPlan(*args) + + +def _placement(kind: PlacementKind, rank: int, **overrides: object) -> LogicalPlacement: + shard_dim = ( + None + if kind is PlacementKind.REPLICATED + else (0 if kind is PlacementKind.COLUMN_WISE else 1) + ) + start, end = (0, 0) if shard_dim is None else (rank * 2, (rank + 1) * 2) + values: dict[str, object] = { + "logical_name": "model.layers.0.weight", + "global_shape": (4, 4), + "dtype": "float32", + "kind": kind, + "rank": rank, + "world_size": 2, + "direction_axis": 1, + "shard_dim": shard_dim, + "shard_start": start, + "shard_end": end, + } + values.update(overrides) + return LogicalPlacement(**values) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("kind", "expected"), + [ + (PlacementKind.COLUMN_WISE, (2, 4)), + (PlacementKind.ROW_WISE, (4, 2)), + (PlacementKind.REPLICATED, (4, 4)), + ], +) +def test_logical_placements_report_exact_local_shapes(kind, expected): + assert _placement(kind, 0).local_shape == expected + + +@pytest.mark.parametrize( + ("kind", "overrides", "message"), + [ + (PlacementKind.COLUMN_WISE, {"shard_dim": 1}, "requires shard_dim=0"), + (PlacementKind.ROW_WISE, {"shard_dim": 0}, "requires shard_dim=1"), + (PlacementKind.REPLICATED, {"shard_dim": 0}, "cannot declare"), + (PlacementKind.COLUMN_WISE, {"shard_start": 1}, "does not match"), + (PlacementKind.COLUMN_WISE, {"global_shape": (5, 4)}, "equal shard"), + (PlacementKind.COLUMN_WISE, {"global_shape": (4,)}, "two-dimensional"), + (PlacementKind.COLUMN_WISE, {"logical_name": "bad name"}, "invalid format"), + (PlacementKind.COLUMN_WISE, {"direction_axis": 2}, "direction_axis"), + ], +) +def test_logical_placement_fails_closed_for_unknown_or_uneven_layouts(kind, overrides, message): + with pytest.raises(ContractError, match=message): + _placement(kind, 0, **overrides) + + +def test_logical_placement_rejects_unknown_dtype(): + with pytest.raises(ContractError, match="dtype is not supported"): + _placement(PlacementKind.COLUMN_WISE, 0, dtype="float8_e4m3fn") + + +def test_lifecycle_accepts_only_the_documented_happy_path_and_abort_path(): + happy = [ + RuntimeStage.CREATED, + RuntimeStage.PREFLIGHTED, + RuntimeStage.LOADED, + RuntimeStage.PROBED, + RuntimeStage.DISTILLED, + RuntimeStage.PREPARED, + RuntimeStage.MUTATING, + RuntimeStage.VERIFIED, + RuntimeStage.STAGED, + RuntimeStage.PUBLISHED, + ] + for current, requested in zip(happy, happy[1:]): + assert advance_stage(current, requested) is requested + assert advance_stage(RuntimeStage.LOADED, RuntimeStage.ABORTING) is RuntimeStage.ABORTING + assert advance_stage(RuntimeStage.ABORTING, RuntimeStage.ABORTED) is RuntimeStage.ABORTED + assert ( + advance_stage(RuntimeStage.ABORTING, RuntimeStage.QUARANTINED) is RuntimeStage.QUARANTINED + ) + + +@pytest.mark.parametrize( + ("current", "requested"), + [ + (RuntimeStage.CREATED, RuntimeStage.LOADED), + (RuntimeStage.MUTATING, RuntimeStage.STAGED), + (RuntimeStage.PUBLISHED, RuntimeStage.ABORTING), + (RuntimeStage.ABORTED, RuntimeStage.CREATED), + (RuntimeStage.QUARANTINED, RuntimeStage.PUBLISHED), + ], +) +def test_lifecycle_rejects_skipped_or_post_terminal_transitions(current, requested): + with pytest.raises(ContractError, match="invalid distributed stage transition"): + advance_stage(current, requested) + with pytest.raises(ContractError, match="must be RuntimeStage"): + advance_stage(current.value, requested) # type: ignore[arg-type] + + +def test_stage_message_requires_sequenced_typed_abort_evidence(): + message = StageMessage( + run_id="1" * 32, + identity_digest=_digest(), + rank=1, + sequence=7, + stage=RuntimeStage.ABORTING, + vote=Vote.ABORT, + error_code="LMS_TEST_FAILURE", + ) + assert contract_digest(message) + assert StageMessage.from_bytes(message.to_bytes()) == message + with pytest.raises(ContractError, match="not canonical"): + StageMessage.from_bytes(b" " + message.to_bytes()) + duplicate = message.to_bytes().replace(b'{"error_code":', b'{"rank":0,"error_code":', 1) + with pytest.raises(ContractError, match="duplicate"): + StageMessage.from_bytes(duplicate) + with pytest.raises(ContractError, match="requires an error_code"): + StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.ABORTING, Vote.ABORT) + with pytest.raises(ContractError, match="valid only with an abort vote"): + StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.LOADED, None, "LMS_FAIL") + with pytest.raises(ContractError, match="error_code has an invalid format"): + StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.ABORTING, Vote.ABORT, "bad") + + +@pytest.mark.parametrize( + "value", + [1.5, {1: "bad key"}, {"nested": object()}], +) +def test_canonical_record_rejects_ambiguous_types(value): + with pytest.raises(ContractError): + canonical_record(value) + + +def test_canonical_record_rejects_enum_type_bypass_cycles_and_resource_abuse(): + class FloatEnum(Enum): + VALUE = 1.5 + + with pytest.raises(ContractError, match="unsupported type float"): + canonical_record(FloatEnum.VALUE) + cyclic: dict[str, object] = {} + cyclic["cycle"] = cyclic + with pytest.raises(ContractError, match="reference cycle"): + canonical_record(cyclic) + nested: object = None + for _ in range(18): + nested = [nested] + with pytest.raises(ContractError, match="nesting depth"): + canonical_record(nested) + with pytest.raises(ContractError, match="4096 items"): + canonical_record([None] * 4097) + with pytest.raises(ContractError, match="signed 64-bit"): + canonical_record(2**63) + + +def test_record_gather_rejects_noncanonical_raw_bytes_before_group_use(): + with pytest.raises(ContractError, match="unsupported type bytes"): + gloo_all_gather_records(b"not-a-canonical-record") + + +def test_canonical_record_enforces_size_before_collective_allocation(): + with pytest.raises(ContractError, match="exceeds 16 bytes"): + canonical_record({"value": "x" * 20}, max_bytes=16) + with pytest.raises(ContractError, match="max_bytes"): + canonical_record({}, max_bytes=MAX_CONSENSUS_BYTES + 1) + + +def test_fixed_frame_round_trip_and_zero_padding(): + frame = encode_frame(b"record", capacity=16) + assert frame.dtype == torch.uint8 + assert frame.numel() == 20 + assert decode_frame(frame) == b"record" + + +@pytest.mark.parametrize( + ("payload", "capacity", "message"), + [ + ("not bytes", 16, "payload must be bytes"), + (b"too long", 2, "exceeds 2 bytes"), + (b"ok", 0, "capacity must be between"), + (b"ok", MAX_CONSENSUS_BYTES + 1, "capacity must be between"), + ], +) +def test_fixed_frame_rejects_invalid_input(payload, capacity, message): + with pytest.raises(ContractError, match=message): + encode_frame(payload, capacity=capacity) # type: ignore[arg-type] + + +def test_frame_decoder_rejects_type_shape_length_and_padding_corruption(): + with pytest.raises(ContractError, match="one-dimensional uint8"): + decode_frame(torch.zeros((2, 2))) + with pytest.raises(ContractError, match="invalid capacity"): + decode_frame(torch.zeros(4, dtype=torch.uint8)) + too_long = encode_frame(b"a", capacity=2) + too_long[:4] = torch.tensor(list((3).to_bytes(4, "big")), dtype=torch.uint8) + with pytest.raises(ContractError, match="length exceeds"): + decode_frame(too_long) + bad_padding = encode_frame(b"a", capacity=2) + bad_padding[-1] = 1 + with pytest.raises(ContractError, match="padding must be zero"): + decode_frame(bad_padding) + + +def test_collective_helpers_refuse_without_a_gloo_group(): + with pytest.raises(ContractError, match="must be initialized"): + gloo_all_gather_records({"rank": 0}) + with pytest.raises(ContractError, match="64 lowercase"): + require_consensus_digest("BAD") + with pytest.raises(ContractError, match="must be initialized"): + require_record_consensus(_identity()) + with pytest.raises(ContractError, match="vote sequence"): + unanimous_vote(-1, True) + with pytest.raises(ContractError, match="accepted must"): + unanimous_vote(1, 1) # type: ignore[arg-type] + with pytest.raises(ContractError, match="initialized process group"): + distributed_project_weight( + torch.eye(4), + torch.ones(4), + _placement(PlacementKind.REPLICATED, 0), + ) + with pytest.raises(ContractError, match="placement must"): + distributed_project_weight(torch.eye(4), torch.ones(4), object()) # type: ignore[arg-type] + + +def test_rank_order_rejects_missing_duplicate_and_reordered_records(): + records = (_inventory(0), _inventory(1)) + assert_rank_order(records, world_size=2) + with pytest.raises(ContractError, match="record count"): + assert_rank_order(records[:1], world_size=2) + with pytest.raises(ContractError, match="global-rank order"): + assert_rank_order(tuple(reversed(records)), world_size=2) diff --git a/tests/test_distributed_evidence.py b/tests/test_distributed_evidence.py new file mode 100644 index 0000000..c9ae7e6 --- /dev/null +++ b/tests/test_distributed_evidence.py @@ -0,0 +1,113 @@ +"""Redacted, atomic distributed-preflight evidence tests.""" + +from __future__ import annotations + +import json +import stat + +import pytest + +from obliteratus.distributed.contracts import ( + ContractError, + RuntimeStage, + StageMessage, + Vote, +) +from obliteratus.distributed.evidence import ( + PreflightEvidence, + read_evidence, + read_stage_message, + write_evidence, + write_stage_message, +) + + +def test_evidence_is_allowlisted_redacted_and_private(tmp_path): + hostile = "Bearer hf_secret password=/private/model 10.10.0.10 host.internal" + evidence = PreflightEvidence.failure( + run_id="1" * 32, + config_digest="2" * 64, + code="LMS_DIAGNOSTIC_REDACTION_FAILED", + world_size=2, + evidence_tier="protocol_cpu", + detail=hostile, + ) + path = tmp_path / "evidence.json" + write_evidence(path, evidence) + raw = path.read_text(encoding="utf-8") + parsed = json.loads(raw) + assert parsed["result"] == "failed" + assert parsed["error_code"] == "LMS_DIAGNOSTIC_REDACTION_FAILED" + assert hostile not in raw + assert "10.10.0.10" not in raw + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert read_evidence(path) == evidence + + +def test_cleanup_failure_is_quarantined(): + evidence = PreflightEvidence.failure( + run_id="1" * 32, + config_digest="2" * 64, + code="LMS_CLEANUP_INCOMPLETE", + world_size=2, + evidence_tier="candidate_preflight", + detail="secret", + ) + assert evidence.result == "quarantined" + assert evidence.error_code == "LMS_CLEANUP_INCOMPLETE" + + +def test_evidence_never_overwrites_an_existing_attempt_record(tmp_path): + evidence = PreflightEvidence.failure( + run_id="1" * 32, + config_digest="2" * 64, + code="LMS_EVIDENCE_UNAVAILABLE", + world_size=2, + evidence_tier="candidate_preflight", + ) + path = tmp_path / "evidence.json" + write_evidence(path, evidence) + with pytest.raises(ContractError, match="already exists"): + write_evidence(path, evidence) + + +def test_evidence_parent_must_be_private_and_cannot_be_a_symlink(tmp_path): + evidence = PreflightEvidence.failure( + run_id="1" * 32, + config_digest="2" * 64, + code="LMS_EVIDENCE_UNAVAILABLE", + world_size=2, + evidence_tier="candidate_preflight", + ) + public = tmp_path / "public" + public.mkdir(mode=0o755) + with pytest.raises(ContractError, match="private"): + write_evidence(public / "evidence.json", evidence) + private = tmp_path / "private" + private.mkdir(mode=0o700) + alias = tmp_path / "alias" + alias.symlink_to(private, target_is_directory=True) + with pytest.raises(ContractError): + write_evidence(alias / "evidence.json", evidence) + + +def test_evidence_decoder_rejects_noncanonical_and_duplicate_json(): + duplicate = b'{"schema_version":1,"schema_version":1,"result":"failed"}\n' + with pytest.raises(ContractError, match="duplicate"): + PreflightEvidence.from_bytes(duplicate) + + +def test_lifecycle_receipt_uses_a_distinct_nonterminal_schema(tmp_path): + message = StageMessage( + run_id="1" * 32, + identity_digest="2" * 64, + rank=0, + sequence=1, + stage=RuntimeStage.PREFLIGHTED, + vote=Vote.PREPARED, + ) + path = tmp_path / "prepared.stage.json" + write_stage_message(path, message) + assert read_stage_message(path) == message + with pytest.raises(ContractError): + read_evidence(path) diff --git a/tests/test_distributed_gloo.py b/tests/test_distributed_gloo.py new file mode 100644 index 0000000..25361a0 --- /dev/null +++ b/tests/test_distributed_gloo.py @@ -0,0 +1,871 @@ +"""Two-process CPU/Gloo semantic and failure tests for issue 58.""" + +from __future__ import annotations + +import hashlib +import multiprocessing as mp +import os +import socket +import time +from dataclasses import replace +from datetime import timedelta +from pathlib import Path +from queue import Empty +from typing import Any + +import pytest +import torch +import torch.distributed as dist + +from obliteratus.analysis.numerical_contracts import project_weight_against_direction +from obliteratus.distributed.config import DistributedPreflightConfig +from obliteratus.distributed.consensus import ( + gloo_all_gather_records, + require_consensus_digest, + require_record_consensus, + unanimous_vote, +) +from obliteratus.distributed.contracts import ( + ContractError, + LogicalPlacement, + PlacementKind, + RunIdentity, + RuntimeStage, + Vote, +) +from obliteratus.distributed.evidence import read_evidence, read_stage_message +from obliteratus.distributed.numerical import distributed_project_weight +from obliteratus.distributed.launcher import TorchrunEnvironment +from obliteratus.distributed.preflight import ( + LocalSnapshot, + SourceIdentity, + execute_preflight, + run_preflight, +) + + +WORLD_SIZE = 2 + + +def _preflight_config(root: Path) -> DistributedPreflightConfig: + root.joinpath("staging").mkdir(exist_ok=True) + software = ( + ("accelerate", "test"), + ("cuda", "unavailable"), + ("driver", "unavailable"), + ("machine", "test"), + ("nccl", "unavailable"), + ("platform", "test"), + ("python", "test"), + ("safetensors", "test"), + ("torch", "test"), + ("transformers", "test"), + ) + return DistributedPreflightConfig( + run_id="1" * 32, + rendezvous_id="2" * 32, + world_size=2, + local_world_size=1, + source_digest="a" * 64, + model_digest="b" * 64, + tokenizer_digest="c" * 64, + commit_sha="d" * 40, + code_digest="0" * 64, + tensor_parallel_size=2, + coordinator_rank=0, + placement_plan_digest="e" * 64, + dimension_divisors=(2, 4), + master_addr="10.10.0.10", + master_port=29500, + network_interface="eth0", + allowed_master_cidrs=("10.10.0.0/24",), + source_path=root / "source", + staging_path=root / "staging", + storage_digest="f" * 64, + min_free_device_memory_bytes=1, + min_free_host_memory_bytes=1, + min_free_staging_bytes=1, + max_source_files=1000, + max_source_bytes=1024, + max_source_file_bytes=1024, + source_timeout_seconds=2, + init_timeout_seconds=2, + collective_timeout_seconds=2, + teardown_timeout_seconds=2, + software_versions=software, + device_kind="cpu", + device_name="cpu", + compute_capability="none", + evidence_tier="protocol_cpu", + allowed_environment_keys=(), + local_files_only=True, + trust_remote_code=False, + allow_runtime_install=False, + allow_plugins=False, + allow_compilation=False, + allow_adapters=False, + allow_quantization=False, + evidence_path=root / "staging" / ("1" * 32) / "preflight.json", + digest="9" * 64, + ).validate() + + +class _FixedProbes: + def __init__(self, rank: int, software: tuple[tuple[str, str], ...]): + self.rank = rank + self.software = software + + def collect(self, config, launch): + return LocalSnapshot( + host_identity=f"host-{self.rank}", + device_identity=f"cpu-{self.rank}", + device_name="cpu", + compute_capability="none", + device_kind="cpu", + total_device_memory_bytes=8192, + free_device_memory_bytes=4096, + total_host_memory_bytes=16384, + free_host_memory_bytes=8192, + free_staging_bytes=8192, + storage_identity=config.storage_digest, + source=SourceIdentity( + config.source_digest, + config.model_digest, + config.tokenizer_digest, + 2, + 2, + ), + software_versions=self.software, + commit_sha=config.commit_sha, + code_digest=config.code_digest, + ) + + +def _execute_worker( + rank: int, + root_text: str, + mode: str, + port: int, + queue: mp.Queue, +) -> None: + root = Path(root_text) + config = replace( + _preflight_config(root), + master_addr="127.0.0.1", + master_port=port, + init_timeout_seconds=10, + collective_timeout_seconds=10, + ) + launch = TorchrunEnvironment( + rank=rank, + local_rank=0, + world_size=2, + local_world_size=1, + group_rank=rank, + role_rank=rank, + role_world_size=2, + master_endpoint_digest="8" * 64, + run_id=config.run_id, + rendezvous_id=config.rendezvous_id, + network_interface=config.network_interface, + ) + os.environ["MASTER_ADDR"] = config.master_addr + os.environ["MASTER_PORT"] = str(config.master_port) + probes = _FixedProbes(rank, config.software_versions) + if mode == "identity_failure" and rank == 1: + original_collect = probes.collect + + def collect_with_wrong_identity(candidate_config, candidate_launch): + snapshot = original_collect(candidate_config, candidate_launch) + return replace( + snapshot, + source=replace(snapshot.source, source_digest="7" * 64), + ) + + probes.collect = collect_with_wrong_identity # type: ignore[method-assign] + if mode == "storage_failure" and rank == 1: + original_collect = probes.collect + + def collect_with_wrong_storage(candidate_config, candidate_launch): + snapshot = original_collect(candidate_config, candidate_launch) + return replace(snapshot, storage_identity="7" * 64) + + probes.collect = collect_with_wrong_storage # type: ignore[method-assign] + if mode == "resource_failure" and rank == 1: + original_collect = probes.collect + + def collect_without_headroom(candidate_config, candidate_launch): + snapshot = original_collect(candidate_config, candidate_launch) + return replace(snapshot, free_device_memory_bytes=0) + + probes.collect = collect_without_headroom # type: ignore[method-assign] + if mode == "commit_failure" and rank == 1: + original_collect = probes.collect + + def collect_with_wrong_commit(candidate_config, candidate_launch): + snapshot = original_collect(candidate_config, candidate_launch) + return replace(snapshot, commit_sha="7" * 40) + + probes.collect = collect_with_wrong_commit # type: ignore[method-assign] + if mode == "code_failure" and rank == 1: + original_collect = probes.collect + + def collect_with_wrong_code(candidate_config, candidate_launch): + snapshot = original_collect(candidate_config, candidate_launch) + return replace(snapshot, code_digest="7" * 64) + + probes.collect = collect_with_wrong_code # type: ignore[method-assign] + if mode == "stage_timeout": + + def collect_after_timeout(candidate_config, candidate_launch): + del candidate_config, candidate_launch + raise TimeoutError("secret-bearing timeout detail") + + probes.collect = collect_after_timeout # type: ignore[method-assign] + if mode == "cancelled": + + def collect_after_cancel(candidate_config, candidate_launch): + del candidate_config, candidate_launch + raise KeyboardInterrupt + + probes.collect = collect_after_cancel # type: ignore[method-assign] + if mode == "native_stderr": + original_collect = probes.collect + + def collect_with_native_diagnostic(candidate_config, candidate_launch): + os.write(2, b"secret-bearing native child diagnostic\n") + return original_collect(candidate_config, candidate_launch) + + probes.collect = collect_with_native_diagnostic # type: ignore[method-assign] + if mode == "rank_exception" and rank == 1: + + def collect_after_error(candidate_config, candidate_launch): + del candidate_config, candidate_launch + raise RuntimeError("secret-bearing rank exception") + + probes.collect = collect_after_error # type: ignore[method-assign] + if mode == "execute_hang" and rank == 1: + original_collect = probes.collect + + def collect_after_hang(candidate_config, candidate_launch): + time.sleep(60) + return original_collect(candidate_config, candidate_launch) + + probes.collect = collect_after_hang # type: ignore[method-assign] + if mode == "execute_early_exit" and rank == 1: + + def exit_before_attestation(candidate_config, candidate_launch): + del candidate_config, candidate_launch + os._exit(17) + + probes.collect = exit_before_attestation # type: ignore[method-assign] + + import obliteratus.distributed.launcher as launcher_module + import obliteratus.distributed.preflight as preflight_module + + original_network_validator = launcher_module.validate_network_interface + original_writer = preflight_module.write_stage_message + original_destroy = dist.destroy_process_group + launcher_module.validate_network_interface = lambda *args, **kwargs: None + if mode == "evidence_failure" and rank == 0: + + def fail_prepared(path, evidence): + if Path(path).name == ".preflight.prepared.stage.json": + raise OSError("injected secret-bearing sink failure") + return original_writer(path, evidence) + + preflight_module.write_stage_message = fail_prepared + if mode == "teardown_failure" and rank == 1: + + def fail_destroy(): + raise RuntimeError("injected secret-bearing teardown failure") + + dist.destroy_process_group = fail_destroy # type: ignore[method-assign] + try: + evidence = execute_preflight(config, launch, probes=probes) + queue.put((rank, "ok", (evidence.result, evidence.error_code))) + except Exception as exc: + queue.put((rank, "error", (getattr(exc, "code", None), str(exc)))) + finally: + launcher_module.validate_network_interface = original_network_validator + preflight_module.write_stage_message = original_writer + dist.destroy_process_group = original_destroy # type: ignore[method-assign] + if dist.is_available() and dist.is_initialized(): + original_destroy() + + +def _run_execute_workers(tmp_path: Path, mode: str, *, timeout: float = 35.0): + context = mp.get_context("spawn") + queue = context.Queue() + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + listener.close() + processes = [ + context.Process( + target=_execute_worker, + args=(rank, str(tmp_path), mode, port, queue), + ) + for rank in range(WORLD_SIZE) + ] + for process in processes: + process.start() + for process in processes: + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join(5) + pytest.fail(f"{mode} execute worker {process.pid} did not terminate") + assert process.exitcode == 0 + results = sorted(queue.get(timeout=2) for _ in range(WORLD_SIZE)) + queue.close() + queue.join_thread() + return results + + +def _run_execute_disruption( + tmp_path: Path, + mode: str, + *, + ranks: tuple[int, ...] = (0, 1), + timeout: float = 40.0, +): + context = mp.get_context("spawn") + queue = context.Queue() + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + listener.close() + processes = [ + context.Process( + target=_execute_worker, + args=(rank, str(tmp_path), mode, port, queue), + ) + for rank in ranks + ] + for process in processes: + process.start() + for process in processes: + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join(5) + pytest.fail(f"{mode} execute worker {process.pid} did not terminate") + results = [queue.get(timeout=2)] + queue.close() + queue.join_thread() + return sorted(results), tuple(process.exitcode for process in processes) + + +def _placement(kind: PlacementKind, rank: int, *, direction_axis: int = 1) -> LogicalPlacement: + shard_dim = ( + None + if kind is PlacementKind.REPLICATED + else (0 if kind is PlacementKind.COLUMN_WISE else 1) + ) + global_shape = (4, 6) if direction_axis == 0 else (4, 4) + shard_size = 0 if shard_dim is None else global_shape[shard_dim] // WORLD_SIZE + start, end = (0, 0) if shard_dim is None else (rank * shard_size, (rank + 1) * shard_size) + return LogicalPlacement( + logical_name="model.layers.0.proj.weight", + global_shape=global_shape, + dtype="float64", + kind=kind, + rank=rank, + world_size=WORLD_SIZE, + direction_axis=direction_axis, + shard_dim=shard_dim, + shard_start=start, + shard_end=end, + ) + + +def _full_inputs(transposed: bool = False) -> tuple[torch.Tensor, torch.Tensor]: + shape = (4, 6) if transposed else (4, 4) + element_count = shape[0] * shape[1] + weight = torch.arange(1, element_count + 1, dtype=torch.float64).reshape(shape) + weight = (weight - (element_count + 1) / 2) / 7.0 + return weight, torch.tensor((1.0, -2.0, 0.5, 3.0), dtype=torch.float64) + + +def _projection_payload(rank: int, mode: str) -> dict[str, Any]: + transposed = mode.startswith("transposed") + full_weight, direction = _full_inputs(transposed) + if mode == "column": + placement = _placement(PlacementKind.COLUMN_WISE, rank) + local_weight = full_weight[rank * 2 : (rank + 1) * 2] + elif mode == "row": + placement = _placement(PlacementKind.ROW_WISE, rank) + local_weight = full_weight[:, rank * 2 : (rank + 1) * 2] + elif mode == "transposed_column": + placement = _placement(PlacementKind.COLUMN_WISE, rank, direction_axis=0) + local_weight = full_weight[rank * 2 : (rank + 1) * 2] + elif mode == "transposed_row": + placement = _placement(PlacementKind.ROW_WISE, rank, direction_axis=0) + local_weight = full_weight[:, rank * 3 : (rank + 1) * 3] + elif mode in {"nonfinite", "zero_direction", "zero_weight", "column_no_norm"}: + placement = _placement(PlacementKind.COLUMN_WISE, rank) + local_weight = full_weight[rank * 2 : (rank + 1) * 2].clone() + elif mode == "replicated": + placement = _placement(PlacementKind.REPLICATED, rank) + local_weight = full_weight.clone() + else: + raise AssertionError(f"unknown projection mode {mode}") + if mode == "nonfinite" and rank == 1: + local_weight[0, 0] = float("nan") + if mode == "zero_direction": + direction = torch.zeros_like(direction) + if mode == "zero_weight": + local_weight.zero_() + result = distributed_project_weight( + local_weight, + direction, + placement, + norm_preserve=mode != "column_no_norm", + regularization=0.2, + projection_row_fraction=0.5, + ) + return { + "weight": result.weight.tolist(), + "projected": result.projected, + "coefficient_norm_sq": result.coefficient_norm_sq, + "layout": result.layout, + } + + +def _validation_messages(rank: int) -> list[str]: + placement = _placement(PlacementKind.COLUMN_WISE, rank) + weight = _full_inputs()[0][rank * 2 : (rank + 1) * 2] + direction = _full_inputs()[1] + cases: tuple[tuple[torch.Tensor, object, dict[str, Any], str], ...] = ( + (weight[:1], direction, {}, "local weight shape"), + (torch.ones_like(weight, dtype=torch.int64), direction, {}, "floating-point"), + (weight.float(), direction, {}, "weight dtype"), + (weight, object(), {}, "direction does not match"), + (weight, direction[:3], {}, "direction does not match"), + (weight, torch.ones(4, dtype=torch.int64), {}, "direction does not match"), + (weight, direction, {"regularization": True}, "finite number"), + (weight, direction, {"regularization": 2.0}, "in [0, 1]"), + (weight, direction, {"projection_row_fraction": False}, "finite number"), + (weight, direction, {"projection_row_fraction": 0.0}, "in (0, 1]"), + (weight, direction, {"max_norm_ratio": True}, "positive and finite"), + ) + messages = [] + for candidate_weight, candidate_direction, kwargs, expected in cases: + try: + distributed_project_weight( + candidate_weight, + candidate_direction, # type: ignore[arg-type] + placement, + **kwargs, + ) + except ContractError as exc: + assert expected in str(exc) + messages.append(str(exc)) + else: + raise AssertionError(f"validation case unexpectedly passed: {expected}") + return messages + + +def _worker(rank: int, init_file: str, mode: str, queue: mp.Queue) -> None: + try: + dist.init_process_group( + "gloo", + init_method=f"file://{init_file}", + rank=rank, + world_size=WORLD_SIZE, + timeout=timedelta(seconds=5), + ) + if mode == "success_matrix": + records = gloo_all_gather_records({"rank": rank}, capacity=64) + digest = hashlib.sha256(b"same").hexdigest() + agreed = require_consensus_digest(digest) + vote = unanimous_vote(3, True) + payload: dict[str, Any] = { + "records": ([item.decode() for item in records], agreed, vote), + "vote_no": unanimous_vote(4, rank == 0), + } + for projection_mode in ( + "column", + "row", + "replicated", + "transposed_column", + "transposed_row", + "nonfinite", + "zero_direction", + "zero_weight", + "column_no_norm", + ): + payload[projection_mode] = _projection_payload(rank, projection_mode) + payload["validation"] = _validation_messages(rank) + queue.put((rank, "ok", payload)) + elif mode == "digest_mismatch": + digest = hashlib.sha256(f"rank-{rank}".encode()).hexdigest() + require_consensus_digest(digest) + queue.put((rank, "unexpected", None)) + elif mode == "identity_mismatch": + identity = RunIdentity( + run_id="1" * 32, + config_digest=str(rank + 1) * 64, + source_digest="a" * 64, + model_digest="b" * 64, + tokenizer_digest="c" * 64, + commit_sha="d" * 40, + world_size=WORLD_SIZE, + ) + require_record_consensus(identity) + queue.put((rank, "unexpected", None)) + elif mode == "stale_sequence": + unanimous_vote(rank, True) + queue.put((rank, "unexpected", None)) + elif mode == "placement_disagreement": + if rank == 0: + placement = _placement(PlacementKind.COLUMN_WISE, rank) + weight = _full_inputs()[0][:2] + else: + placement = _placement(PlacementKind.ROW_WISE, rank) + weight = _full_inputs()[0][:, 2:] + distributed_project_weight(weight, _full_inputs()[1], placement) + queue.put((rank, "unexpected", None)) + elif mode == "placement_name_disagreement": + placement = _placement(PlacementKind.COLUMN_WISE, rank) + if rank == 1: + placement = replace(placement, logical_name="model.layers.1.proj.weight") + distributed_project_weight( + _full_inputs()[0][rank * 2 : (rank + 1) * 2], _full_inputs()[1], placement + ) + queue.put((rank, "unexpected", None)) + elif mode == "one_rank_error": + if rank == 1: + raise RuntimeError("injected rank error") + gloo_all_gather_records({"rank": rank}, capacity=64) + queue.put((rank, "unexpected", None)) + elif mode == "early_exit": + if rank == 1: + queue.put((rank, "exited", None)) + return + gloo_all_gather_records({"rank": rank}, capacity=64) + queue.put((rank, "unexpected", None)) + elif mode == "hang": + if rank == 1: + time.sleep(7) + gloo_all_gather_records({"rank": rank}, capacity=64) + queue.put((rank, "unexpected", None)) + elif mode == "preflight_success": + config = _preflight_config(Path(init_file).parent) + launch = TorchrunEnvironment( + rank=rank, + local_rank=0, + world_size=2, + local_world_size=1, + group_rank=rank, + role_rank=rank, + role_world_size=2, + master_endpoint_digest="8" * 64, + run_id=config.run_id, + rendezvous_id=config.rendezvous_id, + network_interface=config.network_interface, + ) + result = run_preflight( + config, + launch, + probes=_FixedProbes(rank, config.software_versions), + ) + queue.put( + ( + rank, + "ok", + { + "accepted": len(result.attestations), + "identity_digest": result.identity_digest, + "backend": result.topology.backend, + }, + ) + ) + else: + raise AssertionError(f"unknown worker mode {mode}") + except Exception as exc: + queue.put((rank, "error", (type(exc).__name__, str(exc)))) + finally: + if dist.is_available() and dist.is_initialized(): + dist.destroy_process_group() + + +def _run_workers(tmp_path: Path, mode: str, *, timeout: float = 30.0): + context = mp.get_context("spawn") + queue = context.Queue() + init_file = tmp_path / f"{mode}.rendezvous" + processes = [ + context.Process(target=_worker, args=(rank, str(init_file), mode, queue)) + for rank in range(WORLD_SIZE) + ] + for process in processes: + process.start() + for process in processes: + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join(5) + pytest.fail(f"{mode} worker {process.pid} did not terminate") + assert process.exitcode == 0 + + results = [] + for _ in range(WORLD_SIZE): + try: + results.append(queue.get(timeout=2)) + except Empty: + pytest.fail(f"{mode} did not report one result per rank") + queue.close() + queue.join_thread() + return sorted(results) + + +@pytest.fixture(scope="module") +def success_results(tmp_path_factory): + return _run_workers( + tmp_path_factory.mktemp("distributed-success"), "success_matrix", timeout=20.0 + ) + + +def _case_results(success_results, key: str): + return [(rank, status, payload[key]) for rank, status, payload in success_results] + + +def test_bounded_records_digest_and_unanimous_vote_succeed(success_results): + results = _case_results(success_results, "records") + assert [status for _rank, status, _payload in results] == ["ok", "ok"] + expected_records = ['{"rank":0}', '{"rank":1}'] + for _rank, _status, payload in results: + records, digest, vote = payload + assert records == expected_records + assert digest == hashlib.sha256(b"same").hexdigest() + assert vote is True + + +@pytest.mark.parametrize( + ("mode", "message"), + [ + ("digest_mismatch", "rank digests disagree"), + ("identity_mismatch", "rank digests disagree"), + ("stale_sequence", "rank vote sequences disagree"), + ("placement_disagreement", "rank placement metadata disagrees"), + ("placement_name_disagreement", "rank digests disagree"), + ], +) +def test_rank_disagreement_fails_on_every_participant(tmp_path, mode, message): + results = _run_workers(tmp_path, mode) + assert [status for _rank, status, _payload in results] == ["error", "error"] + assert all(message in payload[1] for _rank, _status, payload in results) + + +def test_one_negative_vote_aborts_unanimity_for_every_rank(success_results): + results = _case_results(success_results, "vote_no") + assert [payload for _rank, status, payload in results if status == "ok"] == [False, False] + + +@pytest.mark.parametrize("mode", ["one_rank_error", "early_exit", "hang"]) +def test_rank_exit_or_timeout_terminates_and_reaps_the_worker_group(tmp_path, mode): + results = _run_workers(tmp_path, mode) + statuses = {rank: status for rank, status, _payload in results} + assert statuses[0] == "error" + assert statuses[1] in {"error", "exited"} + + +@pytest.mark.parametrize( + ("mode", "concat_dim", "transposed"), + [ + ("column", 0, False), + ("row", 1, False), + ("transposed_column", 0, True), + ("transposed_row", 1, True), + ], +) +def test_distributed_shards_match_complete_tensor_projection( + success_results, + mode, + concat_dim, + transposed, +): + results = _case_results(success_results, mode) + assert all(status == "ok" for _rank, status, _payload in results) + shards = [ + torch.tensor(payload["weight"], dtype=torch.float64) for _rank, _status, payload in results + ] + actual = torch.cat(shards, dim=concat_dim) + full_weight, direction = _full_inputs(transposed) + expected = project_weight_against_direction( + full_weight, + direction, + norm_preserve=True, + regularization=0.2, + projection_row_fraction=0.5, + ) + torch.testing.assert_close(actual, expected.weight, rtol=1e-10, atol=1e-10) + assert all(payload["projected"] is True for _rank, _status, payload in results) + assert all(payload["layout"] == expected.layout for _rank, _status, payload in results) + assert all( + payload["coefficient_norm_sq"] == pytest.approx(expected.coefficient_norm_sq) + for _rank, _status, payload in results + ) + + +def test_replicated_projection_is_identical_on_every_rank(success_results): + results = _case_results(success_results, "replicated") + weights = [ + torch.tensor(payload["weight"], dtype=torch.float64) + for _rank, status, payload in results + if status == "ok" + ] + assert len(weights) == WORLD_SIZE + torch.testing.assert_close(weights[0], weights[1], rtol=0, atol=0) + full_weight, direction = _full_inputs() + expected = project_weight_against_direction( + full_weight, + direction, + norm_preserve=True, + regularization=0.2, + projection_row_fraction=0.5, + ) + torch.testing.assert_close(weights[0], expected.weight, rtol=1e-10, atol=1e-10) + + +def test_nonfinite_value_on_one_rank_prevents_mutation_everywhere(success_results): + results = _case_results(success_results, "nonfinite") + assert all(status == "ok" for _rank, status, _payload in results) + assert all(payload["projected"] is False for _rank, _status, payload in results) + + +@pytest.mark.parametrize("mode", ["zero_direction", "zero_weight"]) +def test_degenerate_global_inputs_are_deterministic(success_results, mode): + results = _case_results(success_results, mode) + assert all(status == "ok" for _rank, status, _payload in results) + expected_projected = mode == "zero_weight" + assert all(payload["projected"] is expected_projected for _rank, _status, payload in results) + + +def test_projection_without_norm_preservation_reports_no_global_norm(success_results): + results = _case_results(success_results, "column_no_norm") + assert all(status == "ok" for _rank, status, _payload in results) + actual = torch.cat( + [ + torch.tensor(payload["weight"], dtype=torch.float64) + for _rank, _status, payload in results + ], + dim=0, + ) + full_weight, direction = _full_inputs() + expected = project_weight_against_direction( + full_weight, + direction, + norm_preserve=False, + regularization=0.2, + projection_row_fraction=0.5, + ) + torch.testing.assert_close(actual, expected.weight, rtol=1e-10, atol=1e-10) + assert all(payload["coefficient_norm_sq"] == 0.0 for _rank, _status, payload in results) + + +def test_invalid_numerical_requests_fail_closed_on_both_ranks(success_results): + results = _case_results(success_results, "validation") + assert [status for _rank, status, _payload in results] == ["ok", "ok"] + assert all(len(payload) == 11 for _rank, _status, payload in results) + + +def test_real_gloo_preflight_admits_the_complete_fixed_world(tmp_path): + results = _run_workers(tmp_path, "preflight_success") + assert [status for _rank, status, _payload in results] == ["ok", "ok"] + assert {payload["accepted"] for _rank, _status, payload in results} == {2} + assert len({payload["identity_digest"] for _rank, _status, payload in results}) == 1 + assert {payload["backend"] for _rank, _status, payload in results} == {"gloo"} + + +def test_execute_preflight_publishes_success_only_after_all_teardown_acknowledgements( + tmp_path, +): + results = _run_execute_workers(tmp_path, "success") + assert [status for _rank, status, _payload in results] == ["ok", "ok"] + assert {payload for _rank, _status, payload in results} == {("preflighted", None)} + attempt = tmp_path / "staging" / ("1" * 32) + prepared = read_stage_message(attempt / ".preflight.prepared.stage.json") + assert prepared.stage is RuntimeStage.PREFLIGHTED + assert prepared.vote is Vote.PREPARED + assert not (attempt / ".preflight.prepared.json").exists() + acknowledgements = tuple( + read_stage_message(attempt / f".rank-{rank}.teardown.stage.json") + for rank in range(WORLD_SIZE) + ) + assert {item.rank for item in acknowledgements} == {0, 1} + assert {item.vote for item in acknowledgements} == {Vote.COMMITTED} + + +@pytest.mark.parametrize( + ("mode", "code"), + [ + ("identity_failure", "LMS_IDENTITY_MISMATCH"), + ("commit_failure", "LMS_IDENTITY_MISMATCH"), + ("code_failure", "LMS_IDENTITY_MISMATCH"), + ("storage_failure", "LMS_STORAGE_PROFILE_MISMATCH"), + ("resource_failure", "LMS_RESOURCE_ADMISSION_DENIED"), + ("stage_timeout", "LMS_STAGE_TIMEOUT"), + ("cancelled", "LMS_ATTEMPT_CANCELLED"), + ("native_stderr", "LMS_DIAGNOSTIC_REDACTION_FAILED"), + ("evidence_failure", "LMS_EVIDENCE_UNAVAILABLE"), + ("teardown_failure", "LMS_CLEANUP_INCOMPLETE"), + ], +) +def test_execute_preflight_faults_never_report_success_and_reap_workers( + tmp_path, mode, code, capfd +): + results = _run_execute_workers(tmp_path, mode) + assert [status for _rank, status, _payload in results] == ["error", "error"] + assert {payload[0] for _rank, _status, payload in results} == {code} + assert all("secret-bearing" not in payload[1] for _rank, _status, payload in results) + assert "secret-bearing native child" not in capfd.readouterr().err + if mode != "teardown_failure": + attempt = tmp_path / "staging" / ("1" * 32) + aborting = tuple( + read_stage_message(attempt / f".rank-{rank}.aborting.stage.json") + for rank in range(WORLD_SIZE) + ) + terminal = tuple( + read_stage_message(attempt / f".rank-{rank}.terminal.stage.json") + for rank in range(WORLD_SIZE) + ) + assert {item.stage for item in aborting} == {RuntimeStage.ABORTING} + assert {item.stage for item in terminal} == {RuntimeStage.ABORTED} + assert {item.vote for item in (*aborting, *terminal)} == {Vote.ABORT} + + +@pytest.mark.parametrize( + ("mode", "ranks", "exit_codes"), + [ + ("missing_rank", (0,), (0,)), + ("execute_early_exit", (0, 1), (0, 17)), + ], +) +def test_execute_preflight_missing_or_exited_rank_quarantines_and_reaps( + tmp_path, mode, ranks, exit_codes, capfd +): + results, observed_exit_codes = _run_execute_disruption(tmp_path, mode, ranks=ranks) + assert observed_exit_codes == exit_codes + assert len(results) == 1 + assert results[0][1] == "error" + assert results[0][2][0] == "LMS_CLEANUP_INCOMPLETE" + evidence = read_evidence(tmp_path / "staging" / ("1" * 32) / "preflight.json") + assert evidence.result == "quarantined" + assert evidence.error_code == "LMS_CLEANUP_INCOMPLETE" + assert "secret-bearing" not in capfd.readouterr().err + + +@pytest.mark.parametrize("mode", ["rank_exception", "execute_hang"]) +def test_execute_preflight_rank_error_or_hang_is_bounded_and_never_succeeds(tmp_path, mode, capfd): + started = time.monotonic() + results = _run_execute_workers(tmp_path, mode, timeout=20) + elapsed = time.monotonic() - started + assert [status for _rank, status, _payload in results] == ["error", "error"] + evidence = read_evidence(tmp_path / "staging" / ("1" * 32) / "preflight.json") + assert evidence.result != "preflighted" + assert evidence.error_code is not None + assert "secret-bearing" not in capfd.readouterr().err + if mode == "execute_hang": + assert elapsed < 20 diff --git a/tests/test_distributed_launcher.py b/tests/test_distributed_launcher.py new file mode 100644 index 0000000..682411a --- /dev/null +++ b/tests/test_distributed_launcher.py @@ -0,0 +1,392 @@ +"""Fixed-membership launcher and configuration contracts for issue 59.""" + +from __future__ import annotations + +import json +import multiprocessing as mp +import os +import time +from dataclasses import replace + +import pytest + +from obliteratus.distributed.config import DistributedPreflightConfig +from obliteratus.distributed.contracts import ContractError, RuntimeContractError +from obliteratus.distributed.launcher import ( + TorchrunEnvironment, + control_group, +) + + +HEX = "a" * 64 + + +def _teardown_overrun_worker(config, launch, sink_path: str, ready) -> None: + import obliteratus.distributed.launcher as launcher_module + import torch.distributed as child_dist + + descriptor = os.open(sink_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.dup2(descriptor, 2) + os.close(descriptor) + state = {"initialized": False} + launcher_module.validate_network_interface = lambda *args, **kwargs: None + child_dist.is_available = lambda: True # type: ignore[method-assign] + child_dist.is_initialized = lambda: state["initialized"] # type: ignore[method-assign] + child_dist.init_process_group = ( # type: ignore[method-assign] + lambda *args, **kwargs: state.update(initialized=True) + ) + + def delayed_native_write() -> None: + time.sleep(config.teardown_timeout_seconds + 0.25) + os.write(2, b"LATE_NATIVE_MARKER_FROM_TIMED_OUT_TEARDOWN\n") + + child_dist.destroy_process_group = delayed_native_write # type: ignore[method-assign] + ready.set() + with control_group(config, launch): + pass + os._exit(99) + + +def _profile(tmp_path, **overrides): + source = tmp_path / "source" + staging = tmp_path / "staging" + evidence = staging / ("1" * 32) / "preflight.json" + source.mkdir(exist_ok=True) + staging.mkdir(exist_ok=True) + payload = { + "schema_version": 1, + "run": { + "run_id": "1" * 32, + "rendezvous_id": "2" * 32, + "world_size": 2, + "local_world_size": 1, + }, + "identity": { + "source_digest": HEX, + "model_digest": "b" * 64, + "tokenizer_digest": "c" * 64, + "commit_sha": "d" * 40, + "code_digest": "0" * 64, + }, + "topology": { + "tensor_parallel_size": 2, + "coordinator_rank": 0, + "placement_plan_digest": "e" * 64, + "dimension_divisors": [2, 4], + }, + "network": { + "master_addr": "10.10.0.10", + "master_port": 29500, + "interface": "eth0", + "allowed_master_cidrs": ["10.10.0.0/24"], + }, + "source": {"path": str(source)}, + "staging": {"path": str(staging), "storage_digest": "f" * 64}, + "resources": { + "min_free_device_memory_bytes": 1024, + "min_free_host_memory_bytes": 2048, + "min_free_staging_bytes": 4096, + "max_source_files": 1000, + "max_source_bytes": 1099511627776, + "max_source_file_bytes": 1099511627776, + }, + "timeouts": { + "source_seconds": 5, + "init_seconds": 5, + "collective_seconds": 5, + "teardown_seconds": 5, + }, + "software": { + "python": "3.12.11", + "platform": "Linux-test", + "machine": "x86_64", + "torch": "2.13.0", + "transformers": "5.15.0", + "accelerate": "1.10.0", + "safetensors": "0.6.2", + "cuda": "13.0", + "nccl": "2.28.3", + "driver": "580.65", + }, + "execution": { + "device_kind": "cuda", + "device_name": "NVIDIA Test GPU", + "compute_capability": "10.0", + "evidence_tier": "candidate_preflight", + "allowed_environment_keys": [], + "local_files_only": True, + "trust_remote_code": False, + "allow_runtime_install": False, + "allow_plugins": False, + "allow_compilation": False, + "allow_adapters": False, + "allow_quantization": False, + }, + "evidence": {"path": str(evidence)}, + } + payload.update(overrides) + path = tmp_path / "profile.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return DistributedPreflightConfig.from_file(path) + + +def _environment(**overrides): + values = { + "RANK": "0", + "LOCAL_RANK": "0", + "WORLD_SIZE": "2", + "LOCAL_WORLD_SIZE": "1", + "GROUP_RANK": "0", + "ROLE_RANK": "0", + "ROLE_WORLD_SIZE": "2", + "MASTER_ADDR": "10.10.0.10", + "MASTER_PORT": "29500", + "TORCHELASTIC_RUN_ID": "2" * 32, + "TORCHELASTIC_RESTART_COUNT": "0", + "TORCHELASTIC_MAX_RESTARTS": "0", + "OBLITERATUS_RUN_ID": "1" * 32, + "GLOO_SOCKET_IFNAME": "eth0", + "NCCL_SOCKET_IFNAME": "eth0", + } + values.update(overrides) + return values + + +def test_profile_is_strict_bounded_and_order_independent(tmp_path): + config = _profile(tmp_path) + reordered = json.loads((tmp_path / "profile.json").read_text(encoding="utf-8")) + (tmp_path / "profile.json").write_text( + json.dumps(dict(reversed(list(reordered.items())))), encoding="utf-8" + ) + assert DistributedPreflightConfig.from_file(tmp_path / "profile.json").digest == config.digest + + reordered["password"] = "do-not-echo" + (tmp_path / "profile.json").write_text(json.dumps(reordered), encoding="utf-8") + with pytest.raises(ContractError, match="unknown profile field"): + DistributedPreflightConfig.from_file(tmp_path / "profile.json") + + +def test_profile_byte_bound_is_checked_before_json_parsing(tmp_path): + profile = tmp_path / "profile.json" + profile.write_bytes(b" " * (64 * 1024 + 1)) + with pytest.raises(ContractError, match="regular file"): + DistributedPreflightConfig.from_file(profile) + + +@pytest.mark.parametrize( + "field", + [ + "RANK", + "LOCAL_RANK", + "WORLD_SIZE", + "LOCAL_WORLD_SIZE", + "GROUP_RANK", + "ROLE_RANK", + "ROLE_WORLD_SIZE", + "MASTER_ADDR", + "MASTER_PORT", + "TORCHELASTIC_RUN_ID", + "TORCHELASTIC_RESTART_COUNT", + "TORCHELASTIC_MAX_RESTARTS", + "OBLITERATUS_RUN_ID", + "GLOO_SOCKET_IFNAME", + "NCCL_SOCKET_IFNAME", + ], +) +def test_torchrun_environment_requires_every_fixed_field(tmp_path, field): + config = _profile(tmp_path) + environ = _environment() + del environ[field] + with pytest.raises(ContractError, match="required torchrun environment is incomplete"): + TorchrunEnvironment.from_environ(environ, config) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"RANK": "2"}, "rank must be smaller"), + ({"LOCAL_RANK": "1"}, "local_rank must be smaller"), + ({"WORLD_SIZE": "3"}, "world_size disagrees"), + ({"ROLE_WORLD_SIZE": "1"}, "role_world_size disagrees"), + ({"MASTER_PORT": "0"}, "master_port"), + ({"MASTER_ADDR": "8.8.8.8"}, "master endpoint disagrees"), + ({"MASTER_ADDR": "0.0.0.0"}, "master endpoint disagrees"), + ({"TORCHELASTIC_RESTART_COUNT": "1"}, "restarts are forbidden"), + ({"TORCHELASTIC_MAX_RESTARTS": "1"}, "restarts are forbidden"), + ({"TORCHELASTIC_RUN_ID": "3" * 32}, "rendezvous_id disagrees"), + ({"OBLITERATUS_RUN_ID": "3" * 32}, "run_id disagrees"), + ({"GLOO_SOCKET_IFNAME": "eth1"}, "interface disagrees"), + ({"RANK": "9" * 10000}, "unsigned decimal integer"), + ], +) +def test_torchrun_environment_rejects_dynamic_or_unapproved_membership( + tmp_path, overrides, message +): + with pytest.raises(ContractError, match=message): + TorchrunEnvironment.from_environ(_environment(**overrides), _profile(tmp_path)) + + +def test_torchrun_environment_parses_immutable_identity(tmp_path): + launch = TorchrunEnvironment.from_environ(_environment(), _profile(tmp_path)) + assert (launch.rank, launch.local_rank, launch.world_size) == (0, 0, 2) + assert launch.master_endpoint_digest != "10.10.0.10" + with pytest.raises(Exception): + launch.rank = 1 # type: ignore[misc] + + +def test_control_group_always_destroys_and_uses_explicit_timeout(tmp_path, monkeypatch): + config = _profile(tmp_path) + launch = TorchrunEnvironment.from_environ(_environment(), config) + calls = [] + monkeypatch.setattr( + "obliteratus.distributed.launcher.validate_network_interface", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr("torch.distributed.is_available", lambda: True) + monkeypatch.setattr("torch.distributed.is_initialized", lambda: bool(calls)) + monkeypatch.setattr("torch.distributed.init_process_group", lambda *a, **kw: calls.append(kw)) + monkeypatch.setattr("torch.distributed.destroy_process_group", lambda: calls.append("destroy")) + with pytest.raises(RuntimeError, match="injected"): + with control_group(config, launch): + raise RuntimeError("injected") + assert calls[0]["backend"] == "gloo" + assert calls[0]["rank"] == 0 + assert calls[0]["world_size"] == 2 + assert calls[0]["timeout"].total_seconds() == 5 + assert calls[-1] == "destroy" + + +def test_control_group_teardown_failure_replaces_success(tmp_path, monkeypatch): + config = _profile(tmp_path) + launch = TorchrunEnvironment.from_environ(_environment(), config) + state = {"initialized": False} + monkeypatch.setattr( + "obliteratus.distributed.launcher.validate_network_interface", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr("torch.distributed.is_available", lambda: True) + monkeypatch.setattr("torch.distributed.is_initialized", lambda: state["initialized"]) + monkeypatch.setattr( + "torch.distributed.init_process_group", + lambda *args, **kwargs: state.update(initialized=True), + ) + + def fail_destroy(): + raise RuntimeError("secret-bearing backend diagnostic") + + monkeypatch.setattr("torch.distributed.destroy_process_group", fail_destroy) + with pytest.raises(ContractError, match="control group teardown failed"): + with control_group(config, launch): + pass + + +def test_control_group_refuses_unbound_interface_before_backend_init(tmp_path, monkeypatch): + config = _profile(tmp_path) + launch = TorchrunEnvironment.from_environ(_environment(), config) + calls = [] + monkeypatch.setattr("torch.distributed.is_available", lambda: True) + monkeypatch.setattr("torch.distributed.is_initialized", lambda: False) + monkeypatch.setattr( + "torch.distributed.init_process_group", lambda *args, **kwargs: calls.append(kwargs) + ) + + def refuse(*args, **kwargs): + raise RuntimeContractError( + "LMS_NETWORK_PROFILE_DENIED", "configured interface is unavailable" + ) + + monkeypatch.setattr("obliteratus.distributed.launcher.validate_network_interface", refuse) + with pytest.raises(RuntimeContractError) as error: + with control_group(config, launch): + pass + assert error.value.code == "LMS_NETWORK_PROFILE_DENIED" + assert calls == [] + + +@pytest.mark.parametrize("phase", ["init", "collective", "timeout", "teardown"]) +def test_control_group_suppresses_native_fd2_and_emits_only_stable_code( + tmp_path, monkeypatch, capfd, phase +): + config = _profile(tmp_path) + launch = TorchrunEnvironment.from_environ(_environment(), config) + state = {"initialized": False} + monkeypatch.setattr( + "obliteratus.distributed.launcher.validate_network_interface", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr("torch.distributed.is_available", lambda: True) + monkeypatch.setattr("torch.distributed.is_initialized", lambda: state["initialized"]) + + def initialize(*args, **kwargs): + if phase in {"init", "timeout"}: + os.write(2, b"secret-bearing native backend diagnostic\n") + if phase == "timeout": + raise TimeoutError("raw private endpoint") + state["initialized"] = True + + def destroy(): + if phase == "teardown": + os.write(2, b"secret-bearing native backend diagnostic\n") + state["initialized"] = False + + monkeypatch.setattr("torch.distributed.init_process_group", initialize) + monkeypatch.setattr("torch.distributed.destroy_process_group", destroy) + with pytest.raises(RuntimeContractError) as error: + with control_group(config, launch): + if phase == "collective": + os.write(2, b"secret-bearing native backend diagnostic\n") + assert error.value.code == "LMS_DIAGNOSTIC_REDACTION_FAILED" + assert "secret-bearing" not in str(error.value) + assert "secret-bearing" not in capfd.readouterr().err + + +def test_teardown_overrun_terminates_worker_before_late_native_write(tmp_path): + config = replace(_profile(tmp_path), teardown_timeout_seconds=1) + launch = TorchrunEnvironment.from_environ(_environment(), config) + sink = tmp_path / "worker-stderr.bin" + context = mp.get_context("spawn") + ready = context.Event() + process = context.Process( + target=_teardown_overrun_worker, + args=(config, launch, str(sink), ready), + ) + process.start() + assert ready.wait(20), "teardown-overrun worker did not finish cold startup" + started = time.monotonic() + process.join(3) + elapsed = time.monotonic() - started + if process.is_alive(): + process.terminate() + process.join(2) + pytest.fail("teardown-overrun worker did not terminate within its bound") + assert process.exitcode == 70 + assert elapsed < 3 + assert sink.read_bytes() == b"" + + +def test_config_rejects_weakened_execution_policy(tmp_path): + config = _profile(tmp_path) + with pytest.raises(ContractError, match="trust_remote_code must remain false"): + replace(config, trust_remote_code=True).validate() + + +@pytest.mark.parametrize("name", ["HF_TOKEN", "AWS_ACCESS_KEY_ID", "HTTPS_PROXY"]) +def test_secret_or_proxy_environment_is_rejected_without_echo(tmp_path, name): + environ = _environment() + environ[name] = "private-value" + with pytest.raises(ContractError, match="secret-bearing") as error: + TorchrunEnvironment.from_environ(environ, _profile(tmp_path)) + assert "private-value" not in str(error.value) + + +@pytest.mark.parametrize("address", ["8.8.8.8", "0.0.0.0", "224.0.0.1"]) +def test_profiled_endpoint_must_still_be_private_and_allowlisted(tmp_path, address): + network = { + "master_addr": address, + "master_port": 29500, + "interface": "eth0", + "allowed_master_cidrs": [f"{address}/32"], + } + with pytest.raises(ContractError, match="private"): + config = _profile(tmp_path, network=network) + TorchrunEnvironment.from_environ(_environment(MASTER_ADDR=address), config) diff --git a/tests/test_distributed_preflight.py b/tests/test_distributed_preflight.py new file mode 100644 index 0000000..6f1c386 --- /dev/null +++ b/tests/test_distributed_preflight.py @@ -0,0 +1,383 @@ +"""Admission and source-safety tests for the distributed preflight.""" + +from __future__ import annotations + +import json +import socket +import time +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from obliteratus.distributed.contracts import ( + ContractError, + RuntimeContractError, + RuntimeStage, + StageMessage, + canonical_record, + contract_digest, +) +from obliteratus.distributed.launcher import validate_network_interface +from obliteratus.distributed.preflight import ( + _validate_stage_messages, + RankAttestation, + checkout_code_digest, + checkout_commit, + inspect_source, + storage_mount_digest, + validate_attestations, +) + + +def _attestation(rank: int = 0, **overrides) -> RankAttestation: + values = { + "rank": rank, + "local_rank": 0, + "local_world_size": 1, + "group_rank": rank, + "world_size": 2, + "host_digest": ("1" if rank == 0 else "2") * 64, + "device_digest": ("3" if rank == 0 else "4") * 64, + "device_profile_digest": "0" * 64, + "device_config_digest": contract_digest( + {"kind": "cuda", "name": "test-device", "compute_capability": "10.0"} + ), + "device_kind": "cuda", + "total_device_memory_bytes": 8192, + "free_device_memory_bytes": 4096, + "total_host_memory_bytes": 16384, + "free_host_memory_bytes": 8192, + "free_staging_bytes": 8192, + "software_digest": "5" * 64, + "storage_digest": "6" * 64, + "source_digest": "7" * 64, + "model_digest": "8" * 64, + "tokenizer_digest": "9" * 64, + "config_digest": "a" * 64, + "commit_sha": "b" * 40, + "code_digest": "f" * 64, + "placement_plan_digest": "c" * 64, + "network_interface_digest": "d" * 64, + } + values.update(overrides) + return RankAttestation(**values) + + +def _validate(records): + validate_attestations( + tuple(records), + world_size=2, + tensor_parallel_size=2, + dimension_divisors=(2, 4), + expected_device_kind="cuda", + expected_device_config_digest=contract_digest( + {"kind": "cuda", "name": "test-device", "compute_capability": "10.0"} + ), + expected_software_digest="5" * 64, + expected_source_digest="7" * 64, + expected_model_digest="8" * 64, + expected_tokenizer_digest="9" * 64, + expected_config_digest="a" * 64, + expected_commit_sha="b" * 40, + expected_code_digest="f" * 64, + expected_placement_plan_digest="c" * 64, + expected_storage_digest="6" * 64, + expected_network_interface_digest="d" * 64, + local_world_size=1, + min_free_device_memory_bytes=4096, + min_free_host_memory_bytes=8192, + min_free_staging_bytes=8192, + ) + + +def test_complete_fixed_inventory_passes_exact_boundaries(): + _validate((_attestation(0), _attestation(1))) + + +def test_rank_attestation_decoder_requires_exact_canonical_bytes(): + record = _attestation() + assert RankAttestation.from_bytes(canonical_record(record)) == record + with pytest.raises(ContractError, match="not canonical"): + RankAttestation.from_bytes(b" " + canonical_record(record)) + duplicate = canonical_record(record).replace(b'{"code_digest":', b'{"rank":0,"code_digest":', 1) + with pytest.raises(ContractError, match="duplicate"): + RankAttestation.from_bytes(duplicate) + + +def test_lifecycle_validation_rejects_a_record_bound_to_another_run(): + identity_digest = "a" * 64 + records = ( + StageMessage("1" * 32, identity_digest, 0, 0, RuntimeStage.CREATED), + StageMessage("2" * 32, identity_digest, 1, 0, RuntimeStage.CREATED), + ) + with pytest.raises(RuntimeContractError) as error: + _validate_stage_messages( + records, + run_id="1" * 32, + world_size=2, + stage=RuntimeStage.CREATED, + sequence=0, + identity_digest=identity_digest, + vote=None, + ) + assert error.value.code == "LMS_LIFECYCLE_INVALID" + + +@pytest.mark.parametrize( + ("records", "message"), + [ + ((_attestation(0),), "exactly one attestation"), + ((_attestation(0), _attestation(0)), "rank order"), + ( + (_attestation(0), replace(_attestation(1), device_digest="3" * 64)), + "device identities", + ), + ( + (_attestation(0), replace(_attestation(1), device_profile_digest="e" * 64)), + "device profiles", + ), + ( + (_attestation(0), replace(_attestation(1), local_rank=0, host_digest="1" * 64)), + "local ranks", + ), + ( + (_attestation(0), replace(_attestation(1), software_digest="e" * 64)), + "software identities", + ), + ( + (_attestation(0), replace(_attestation(1), storage_digest="e" * 64)), + "storage identities", + ), + ( + (_attestation(0), replace(_attestation(1), source_digest="e" * 64)), + "source_digest", + ), + ( + (_attestation(0), replace(_attestation(1), free_device_memory_bytes=4095)), + "device memory headroom", + ), + ( + (_attestation(0), replace(_attestation(1), free_host_memory_bytes=8191)), + "host memory headroom", + ), + ( + (_attestation(0), replace(_attestation(1), free_staging_bytes=8191)), + "staging headroom", + ), + ], +) +def test_inventory_disagreement_fails_closed(records, message): + with pytest.raises(ContractError, match=message): + _validate(records) + + +def test_topology_dimensions_must_be_divisible(): + with pytest.raises(ContractError, match="dimension divisor"): + validate_attestations( + (_attestation(0), _attestation(1)), + world_size=2, + tensor_parallel_size=2, + dimension_divisors=(3,), + expected_device_kind="cuda", + expected_device_config_digest=contract_digest( + {"kind": "cuda", "name": "test-device", "compute_capability": "10.0"} + ), + expected_software_digest="5" * 64, + expected_source_digest="7" * 64, + expected_model_digest="8" * 64, + expected_tokenizer_digest="9" * 64, + expected_config_digest="a" * 64, + expected_commit_sha="b" * 40, + expected_code_digest="f" * 64, + expected_placement_plan_digest="c" * 64, + expected_storage_digest="6" * 64, + expected_network_interface_digest="d" * 64, + local_world_size=1, + min_free_device_memory_bytes=1, + min_free_host_memory_bytes=1, + min_free_staging_bytes=1, + ) + + +def test_source_inspection_accepts_only_immutable_local_safetensors(tmp_path): + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + tokenizer = source / "tokenizer.json" + header = json.dumps( + {"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}, + separators=(",", ":"), + ).encode() + weights.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) + tokenizer.write_text('{"model":"fixture"}', encoding="utf-8") + weights.chmod(0o444) + tokenizer.chmod(0o444) + source.chmod(0o555) + first = inspect_source(source) + second = inspect_source(source) + assert first == second + assert first.file_count == 2 + assert len({first.source_digest, first.model_digest, first.tokenizer_digest}) == 3 + + +def test_source_inspection_rejects_arbitrary_bytes_with_safetensors_suffix(tmp_path): + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + tokenizer = source / "tokenizer.json" + weights.write_bytes(b"not-a-safetensors-file") + tokenizer.write_text("{}", encoding="utf-8") + weights.chmod(0o444) + tokenizer.chmod(0o444) + source.chmod(0o555) + with pytest.raises(ContractError, match="safe-structure inspection"): + inspect_source(source) + + +def test_source_inspection_rejects_executable_serialization(tmp_path): + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + pickle_file = source / "pytorch_model.bin" + weights.write_bytes(b"safe") + pickle_file.write_bytes(b"not-executed") + weights.chmod(0o444) + pickle_file.chmod(0o444) + source.chmod(0o555) + with pytest.raises(ContractError, match="outside the safetensors envelope"): + inspect_source(source) + + +def test_source_inspection_rejects_symlinked_directories(tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "tokenizer.json").write_text("{}", encoding="utf-8") + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + weights.write_bytes(b"safe") + (source / "linked").symlink_to(outside, target_is_directory=True) + weights.chmod(0o444) + source.chmod(0o555) + with pytest.raises(RuntimeContractError, match="symbolic links") as error: + inspect_source(source) + assert error.value.code == "LMS_SOURCE_BOUNDARY_VIOLATION" + + +def test_source_inspection_enforces_byte_bounds_before_hashing(tmp_path): + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + tokenizer = source / "tokenizer.json" + weights.write_bytes(b"12345") + tokenizer.write_bytes(b"{}") + weights.chmod(0o444) + tokenizer.chmod(0o444) + source.chmod(0o555) + with pytest.raises(RuntimeContractError, match="configured byte bound") as error: + inspect_source(source, max_file_bytes=4, max_total_bytes=10) + assert error.value.code == "LMS_RESOURCE_ADMISSION_DENIED" + + +def test_source_inspection_enforces_exact_file_and_total_bounds(tmp_path): + source = tmp_path / "source" + source.mkdir() + header = json.dumps( + {"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}, + separators=(",", ":"), + ).encode() + weights = source / "model.safetensors" + tokenizer = source / "tokenizer.json" + weights.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4) + tokenizer.write_text("{}", encoding="utf-8") + total = weights.stat().st_size + tokenizer.stat().st_size + weights.chmod(0o444) + tokenizer.chmod(0o444) + source.chmod(0o555) + assert ( + inspect_source(source, max_files=2, max_total_bytes=total, max_file_bytes=total).file_count + == 2 + ) + with pytest.raises(ContractError, match="file count"): + inspect_source(source, max_files=1, max_total_bytes=total, max_file_bytes=total) + with pytest.raises(ContractError, match="total-byte"): + inspect_source( + source, + max_files=2, + max_total_bytes=total - 1, + max_file_bytes=total - 1, + ) + + +def test_source_inspection_timeout_is_deterministic_before_io(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + source.chmod(0o555) + moments = iter((0.0, 0.0, 2.0)) + monkeypatch.setattr("obliteratus.distributed.preflight.time.monotonic", lambda: next(moments)) + with pytest.raises(RuntimeContractError, match="explicit timeout") as error: + inspect_source(source, timeout_seconds=1) + assert error.value.code == "LMS_STAGE_TIMEOUT" + + +def test_source_inspection_hard_deadline_interrupts_the_structural_inspector(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + weights = source / "model.safetensors" + tokenizer = source / "tokenizer.json" + weights.write_bytes(b"bounded") + tokenizer.write_text("{}", encoding="utf-8") + weights.chmod(0o444) + tokenizer.chmod(0o444) + source.chmod(0o555) + + def block(*args, **kwargs): + time.sleep(5) + raise AssertionError("deadline did not interrupt the inspector") + + monkeypatch.setattr("obliteratus.distributed.preflight.inspect_checkpoint", block) + started = time.monotonic() + with pytest.raises(RuntimeContractError) as error: + inspect_source(source, timeout_seconds=1) + assert error.value.code == "LMS_STAGE_TIMEOUT" + assert time.monotonic() - started < 2 + + +def test_checkout_commit_resolves_a_worktree_reference_without_a_child_process(tmp_path): + checkout = tmp_path / "checkout" + git_dir = tmp_path / "common" / "worktrees" / "candidate" + common = tmp_path / "common" + reference = common / "refs" / "heads" / "candidate" + checkout.mkdir() + git_dir.mkdir(parents=True) + reference.parent.mkdir(parents=True) + (checkout / ".git").write_text(f"gitdir: {git_dir}\n", encoding="utf-8") + (git_dir / "HEAD").write_text("ref: refs/heads/candidate\n", encoding="utf-8") + (git_dir / "commondir").write_text("../..\n", encoding="utf-8") + reference.write_text("a" * 40 + "\n", encoding="utf-8") + assert checkout_commit(checkout) == "a" * 40 + + +def test_checkout_code_digest_changes_with_executable_source(tmp_path): + package = tmp_path / "obliteratus" + package.mkdir() + module = package / "module.py" + module.write_text("VALUE = 1\n", encoding="utf-8") + first = checkout_code_digest(tmp_path) + module.write_text("VALUE = 2\n", encoding="utf-8") + assert checkout_code_digest(tmp_path) != first + + +def test_storage_mount_digest_is_measured_and_stable(tmp_path): + assert storage_mount_digest(tmp_path) == storage_mount_digest(tmp_path) + + +def test_network_interface_requires_allowlisted_address_and_coordinator_binding( + monkeypatch, +): + addresses = {"eth0": [SimpleNamespace(family=socket.AF_INET, address="10.10.0.10")]} + monkeypatch.setattr("psutil.net_if_addrs", lambda: addresses) + validate_network_interface("eth0", ("10.10.0.0/24",), "10.10.0.10", coordinator=True) + with pytest.raises(ContractError, match="not bound"): + validate_network_interface("eth0", ("10.10.0.0/24",), "10.10.0.11", coordinator=True) diff --git a/tests/test_package_export_contracts.py b/tests/test_package_export_contracts.py index 6727c7e..14000ea 100644 --- a/tests/test_package_export_contracts.py +++ b/tests/test_package_export_contracts.py @@ -28,6 +28,7 @@ import obliteratus.analysis as analysis "Watchtower", "get_watchtower", "AutoObliterator", + "CheckpointService", ], ) def test_documented_lazy_export_resolves(name): diff --git a/tests/test_peft_artifacts.py b/tests/test_peft_artifacts.py new file mode 100644 index 0000000..9c85333 --- /dev/null +++ b/tests/test_peft_artifacts.py @@ -0,0 +1,374 @@ +"""Canonical PEFT LoRA export and truthful legacy-format contracts.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from hashlib import sha256 +from pathlib import Path + +import pytest +import torch +from jsonschema import Draft202012Validator +from safetensors.torch import load_file + +from obliteratus.checkpoint_provenance import LineageEvent, ToolIdentity, build_provenance +from obliteratus.lora_ablation import ( + BaseModelIdentity, + load_lora_adapters, + save_legacy_pickle_adapters_trusted, + save_lora_adapters, + save_unsupported_obliteratus_adapters, + validate_adapter_base, +) + + +DIGEST_A = "sha256:" + "a" * 64 +DIGEST_B = "sha256:" + "b" * 64 +COMMIT = "c" * 40 +MANIFEST_SCHEMA = json.loads( + ( + Path(__file__).resolve().parents[1] + / "docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json" + ).read_text() +) + + +def _base() -> BaseModelIdentity: + return BaseModelIdentity( + repo_id="org/exact-base", + revision="d" * 40, + weights_digest=DIGEST_A, + tokenizer_digest=DIGEST_B, + vocab_size=32000, + architecture="TinyForCausalLM", + tied_embeddings=True, + ) + + +def _adapters(): + return { + "model.layers.0.self_attn.q_proj": ( + torch.arange(8, dtype=torch.float32).reshape(4, 2), + torch.arange(6, dtype=torch.float32).reshape(2, 3), + ), + "model.layers.1.mlp.down_proj": ( + torch.arange(10, dtype=torch.float32).reshape(5, 2), + torch.arange(8, dtype=torch.float32).reshape(2, 4), + ), + } + + +def _factory(base: BaseModelIdentity): + def create(output_digests, adapter_identity): + return build_provenance( + sources=(base.to_artifact_identity(),), + converter=ToolIdentity("obliteratus-peft-export", "1.0.0", COMMIT), + obliteratus_commit=COMMIT, + configuration_digest=adapter_identity.config_digest, + tokenizer=base.tokenizer_artifact_identity(), + base_model=base.to_artifact_identity(), + command=("adapter", "export", base.repo_id), + environment={"python": "test", "platform": "cpu", "packages": {}}, + source_topology={"world_size": 1}, + lineage=( + LineageEvent( + "event-surgery", + "surgery", + (), + "obliteratus-peft-export@1.0.0", + ("refusal_direction_ablation",), + ), + ), + input_digests=(base.weights_digest,), + output_digests=output_digests, + transformations=("lora_adapter_export", "surgery"), + observed_scopes=("adapter_weights",), + lost_state=("optimizer_state", "scheduler_state"), + adapter=adapter_identity, + training=None, + unknowns=("training_dataset",), + ) + + return create + + +def _tree(root): + return { + path.name: sha256(path.read_bytes()).hexdigest() + for path in sorted(item for item in root.iterdir() if item.is_file()) + } + + +def test_canonical_export_is_deterministic_standard_named_and_fully_identified(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + base = _base() + + first_artifact = save_lora_adapters( + _adapters(), + first, + base_model=base, + provenance_factory=_factory(base), + lora_alpha=4, + ) + second_artifact = save_lora_adapters( + dict(reversed(list(_adapters().items()))), + second, + base_model=base, + provenance_factory=_factory(base), + lora_alpha=4, + ) + + assert _tree(first) == _tree(second) + assert first_artifact.artifact_id == second_artifact.artifact_id + assert set(_tree(first)) == { + "README.md", + "adapter_config.json", + "adapter_manifest.json", + "adapter_model.safetensors", + "checkpoint-provenance.json", + } + assert not list(first.glob("*.pt")) + config = json.loads((first / "adapter_config.json").read_text()) + assert config["peft_type"] == "LORA" + assert config["base_model_name_or_path"] == base.repo_id + assert config["revision"] == base.revision + assert config["r"] == 2 + assert config["lora_alpha"] == 4 + assert config["target_modules"] == [ + "model.layers.0.self_attn.q_proj", + "model.layers.1.mlp.down_proj", + ] + manifest = json.loads((first / "adapter_manifest.json").read_text()) + Draft202012Validator(MANIFEST_SCHEMA).validate(manifest) + assert manifest["base_model"] == base.to_dict() + assert manifest["scaling"] == 2.0 + assert manifest["merged"] is False + assert manifest["bias"] == "none" + assert manifest["modules_to_save"] == [] + assert manifest["tie_policy"] == "base_model_declared" + assert "exact-base" in (first / "README.md").read_text() + assert first_artifact.weights_path == first / "adapter_model.safetensors" + + +def test_saved_peft_scaling_reproduces_each_exact_internal_delta(tmp_path): + base = _base() + adapters = _adapters() + save_lora_adapters( + adapters, + tmp_path, + base_model=base, + provenance_factory=_factory(base), + lora_alpha=4, + ) + state = load_file(tmp_path / "adapter_model.safetensors", device="cpu") + manifest = json.loads((tmp_path / "adapter_manifest.json").read_text()) + + for mapping in manifest["key_map"]: + original_b, original_a = adapters[mapping["module_name"]] + saved_a = state[mapping["lora_A_key"]] + saved_b = state[mapping["lora_B_key"]] + actual_delta = saved_b @ saved_a * manifest["scaling"] + assert torch.equal(actual_delta, original_b @ original_a) + + loaded = load_lora_adapters(tmp_path, base_model=base) + assert set(loaded) == set(adapters) + for key in adapters: + loaded_b, loaded_a = loaded[key] + original_b, original_a = adapters[key] + assert torch.equal(loaded_b @ loaded_a, original_b @ original_a) + + +@pytest.mark.parametrize( + ("field", "value", "detail"), + [ + ("repo_id", "other/base", "base_model_identity_mismatch"), + ("revision", "e" * 40, "base_model_revision_mismatch"), + ("weights_digest", "sha256:" + "f" * 64, "base_model_digest_mismatch"), + ("tokenizer_digest", "sha256:" + "f" * 64, "tokenizer_digest_mismatch"), + ("vocab_size", 32001, "vocab_size_mismatch"), + ("architecture", "OtherModel", "architecture_mismatch"), + ], +) +def test_wrong_base_or_tokenizer_is_rejected_before_adapter_loading( + tmp_path, + field, + value, + detail, +): + base = _base() + save_lora_adapters( + _adapters(), + tmp_path, + base_model=base, + provenance_factory=_factory(base), + ) + + with pytest.raises(ValueError, match=detail): + validate_adapter_base(tmp_path, replace(base, **{field: value})) + + +def test_missing_or_malformed_canonical_artifact_fails_before_loading(tmp_path): + base = _base() + save_lora_adapters( + _adapters(), + tmp_path, + base_model=base, + provenance_factory=_factory(base), + ) + (tmp_path / "adapter_config.json").write_text("{", encoding="utf-8") + + with pytest.raises(ValueError, match="adapter_config_invalid"): + load_lora_adapters(tmp_path, base_model=base) + + +@pytest.mark.parametrize( + "relative_path", + [ + "README.md", + "adapter_model.safetensors", + "adapter_config.json", + "adapter_manifest.json", + ], +) +def test_tampered_peft_artifact_fails_digest_check_before_weight_loading( + tmp_path, + monkeypatch, + relative_path, +): + import obliteratus.lora_ablation as lora_ablation + + base = _base() + save_lora_adapters( + _adapters(), + tmp_path, + base_model=base, + provenance_factory=_factory(base), + ) + artifact = tmp_path / relative_path + if relative_path.endswith(".json"): + record = json.loads(artifact.read_text(encoding="utf-8")) + record["tampered"] = True + artifact.write_text(json.dumps(record), encoding="utf-8") + else: + artifact.write_bytes(artifact.read_bytes() + b"tampered") + monkeypatch.setattr( + lora_ablation, + "load_file", + lambda *_args, **_kwargs: pytest.fail("weights loaded before integrity check"), + ) + + with pytest.raises(ValueError, match="adapter_artifact_digest_mismatch"): + validate_adapter_base(tmp_path, base) + + +def test_tampered_provenance_digest_fails_before_weight_loading(tmp_path, monkeypatch): + import obliteratus.lora_ablation as lora_ablation + + base = _base() + save_lora_adapters( + _adapters(), + tmp_path, + base_model=base, + provenance_factory=_factory(base), + ) + provenance_path = tmp_path / "checkpoint-provenance.json" + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + provenance["record_digest"] = DIGEST_A + provenance_path.write_text(json.dumps(provenance), encoding="utf-8") + monkeypatch.setattr( + lora_ablation, + "load_file", + lambda *_args, **_kwargs: pytest.fail("weights loaded before integrity check"), + ) + + with pytest.raises(ValueError, match="adapter_provenance_digest_mismatch"): + validate_adapter_base(tmp_path, base) + + +def test_provenance_failure_preserves_prior_destination_and_leaks_no_staging(tmp_path): + destination = tmp_path / "adapter" + destination.mkdir() + sentinel = destination / "prior.txt" + sentinel.write_text("prior", encoding="utf-8") + + def fail_provenance(_digests, _adapter_identity): + raise RuntimeError("injected provenance failure") + + with pytest.raises(FileExistsError, match="absent or empty"): + save_lora_adapters( + _adapters(), + destination, + base_model=_base(), + provenance_factory=fail_provenance, + ) + + assert sentinel.read_text(encoding="utf-8") == "prior" + assert not list(tmp_path.glob(".adapter.staging-*")) + + empty_destination = tmp_path / "empty" + empty_destination.mkdir() + with pytest.raises(RuntimeError, match="injected provenance failure"): + save_lora_adapters( + _adapters(), + empty_destination, + base_model=_base(), + provenance_factory=fail_provenance, + ) + + assert empty_destination.is_dir() + assert not list(empty_destination.iterdir()) + assert not list(tmp_path.glob(".empty.staging-*")) + + +def test_safe_loader_never_calls_torch_load(tmp_path, monkeypatch): + base = _base() + save_lora_adapters( + _adapters(), + tmp_path, + base_model=base, + provenance_factory=_factory(base), + ) + monkeypatch.setattr(torch, "load", lambda *_args, **_kwargs: pytest.fail("pickle loaded")) + + loaded = load_lora_adapters(tmp_path, base_model=base) + + assert loaded + + +def test_unknown_base_uses_truthful_safe_legacy_format_not_peft_or_pickle(tmp_path): + artifact = save_unsupported_obliteratus_adapters( + _adapters(), + tmp_path, + reason="exact base digest unavailable", + ) + + assert artifact.name == "obliteratus_unsupported_adapter.safetensors" + assert not (tmp_path / "adapter_config.json").exists() + assert not list(tmp_path.glob("*.pt")) + record = json.loads((tmp_path / "obliteratus_unsupported_adapter.json").read_text()) + assert record["support_status"] == "unsupported_legacy" + assert record["safe_serialization"] is True + assert record["peft_compatible"] is False + assert record["reason"] == "exact base digest unavailable" + + for unsafe_reason in ("Bearer abcdefghijklmnop", "/private/model/path"): + with pytest.raises(ValueError, match="reason is invalid"): + save_unsupported_obliteratus_adapters( + _adapters(), + tmp_path / "unsafe", + reason=unsafe_reason, + ) + + +def test_pickle_legacy_export_requires_an_explicit_trust_gate(tmp_path, monkeypatch): + with pytest.raises(PermissionError, match="allow_pickle"): + save_legacy_pickle_adapters_trusted(_adapters(), tmp_path, allow_pickle=False) + + observed = [] + monkeypatch.setattr(torch, "save", lambda state, path: observed.append((state, path))) + path = save_legacy_pickle_adapters_trusted(_adapters(), tmp_path, allow_pickle=True) + + assert path.name == "obliteratus_legacy_adapter_unsafe.pt" + assert observed and observed[0][1] == path diff --git a/tests/test_persistence_pipeline.py b/tests/test_persistence_pipeline.py index b0ef1c2..033a384 100644 --- a/tests/test_persistence_pipeline.py +++ b/tests/test_persistence_pipeline.py @@ -263,7 +263,11 @@ def test_write_local_checkpoint_strips_runtime_only_state_and_writes_metadata( checkpoint_dir = tmp_path / "staging" adapter_path = checkpoint_dir / "lora" save_adapters = MagicMock(return_value=adapter_path) - monkeypatch.setattr(lora_ablation, "save_lora_adapters", save_adapters) + monkeypatch.setattr( + lora_ablation, + "save_unsupported_obliteratus_adapters", + save_adapters, + ) state_dict = {"weight": torch.ones(1)} metadata_json = '{"schema": 1}' checkpoint_dir.mkdir() @@ -281,7 +285,14 @@ def test_write_local_checkpoint_strips_runtime_only_state_and_writes_metadata( }, ) tokenizer.save_pretrained.assert_called_once_with(checkpoint_dir) - save_adapters.assert_called_once_with(pipeline._lora_adapters, checkpoint_dir) + save_adapters.assert_called_once_with( + pipeline._lora_adapters, + checkpoint_dir, + reason=( + "This run did not retain an exact base-model commit, weights digest, " + "tokenizer digest, vocabulary, and architecture identity." + ), + ) assert (checkpoint_dir / "abliteration_metadata.json").read_text( encoding="utf-8", ) == metadata_json diff --git a/tests/test_quant_dequant.py b/tests/test_quant_dequant.py index a7792b1..dfbd460 100644 --- a/tests/test_quant_dequant.py +++ b/tests/test_quant_dequant.py @@ -215,6 +215,20 @@ def test_load_json_local_missing_returns_none(tmp_path): assert qd._load_json_from_checkpoint(str(tmp_path), "missing.json") is None +def test_load_json_remote_unexpected_download_failure_returns_none(monkeypatch): + def fail_download(*args, **kwargs): + raise OSError("offline") + + monkeypatch.setattr("huggingface_hub.hf_hub_download", fail_download) + + assert qd._load_json_from_checkpoint( + "org/model", + "config.json", + revision="immutable-sha", + local_files_only=True, + ) is None + + def test_safetensors_key_names_reads_single_file_and_index(tmp_path): from safetensors.torch import save_file diff --git a/tests/test_run_archive.py b/tests/test_run_archive.py index 1287d2e..efac7fb 100644 --- a/tests/test_run_archive.py +++ b/tests/test_run_archive.py @@ -4,9 +4,26 @@ import hashlib import json import os from pathlib import Path +from types import SimpleNamespace import pytest -from obliteratus.run_archive import RunArchive, _worker +from obliteratus.checkpoint_provenance import ( + ArtifactIdentity, + ToolIdentity, + build_provenance, +) +from obliteratus.run_archive import ( + RunArchive, + _atomic_json, + _checkpoint_metrics, + _dataset_inputs, + _option_value, + _process_start_ticks, + _redact_arguments, + _worker, + default_archive_root, + main, +) class FakeProcess: @@ -84,7 +101,9 @@ def test_cancel_validates_identity_then_signals_only_worker_group(tmp_path, monk run_id = archive.launch(["org/model"], popen=lambda *_args, **_kwargs: FakeProcess(5252)) monkeypatch.setattr(archive, "_worker_matches", lambda _manifest: True) signals = [] - monkeypatch.setattr("obliteratus.run_archive.os.killpg", lambda pid, sig: signals.append((pid, sig))) + monkeypatch.setattr( + "obliteratus.run_archive.os.killpg", lambda pid, sig: signals.append((pid, sig)) + ) status = archive.cancel(run_id) @@ -314,8 +333,10 @@ def test_worker_success_writes_metrics_inventory_and_atomic_marker(tmp_path, mon assert result["result"]["metrics"] == {"refusal_rate": 0.2, "coherence": 0.8} assert (tmp_path / run_id / "COMPLETE").is_file() assert lifecycle.events == [ - ("loading", "org/model"), ("resize", 0), ("ready", None), - ("release", "complete") + ("loading", "org/model"), + ("resize", 0), + ("ready", None), + ("release", "complete"), ] @@ -397,3 +418,220 @@ def test_restart_recovery_preserves_partial_save_and_logs(tmp_path, monkeypatch) assert recovered["failure"]["type"] == "WorkerLost" assert (run_dir / "checkpoint" / "partial.bin").is_file() assert (run_dir / "run.log").read_text() == "last durable phase\n" + + +def test_archive_helpers_cover_fallbacks_redaction_and_atomic_directory_sync( + tmp_path, + monkeypatch, +): + monkeypatch.delenv("OBLITERATUS_RUN_ARCHIVE", raising=False) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + assert default_archive_root() == tmp_path / "state" / "obliteratus" / "runs" + + assert _redact_arguments(["model", "--api-key=value", "--revision", "safe"]) == [ + "model", + "--api-key=[REDACTED]", + "--revision", + "safe", + ] + assert _process_start_ticks(2**31 - 1) is None + + target = tmp_path / "record.json" + monkeypatch.setattr( + "obliteratus.run_archive.os.open", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("unsupported")), + ) + _atomic_json(target, {"ok": True}) + assert json.loads(target.read_text(encoding="utf-8")) == {"ok": True} + + +def test_manifest_operations_record_running_log_and_revisions(tmp_path): + archive = RunArchive(tmp_path) + run_id = archive.begin(["org/model"]) + + running = archive.mark_running(run_id, phase="loading") + archive.append_log(run_id, "first line\n") + revised = archive.record_revisions( + run_id, + model_revision="model-sha", + tokenizer_revision="tokenizer-sha", + ) + + assert running["status"] == "running" + assert revised["model_revision"] == "model-sha" + assert revised["tokenizer_revision"] == "tokenizer-sha" + assert (archive._run_dir(run_id) / "run.log").read_text() == "first line\n" + + +def test_checkpoint_provenance_attachment_rejects_invalid_and_conflicting_records( + tmp_path, +): + archive = RunArchive(tmp_path) + run_id = archive.begin(["org/model"]) + + with pytest.raises(ValueError, match="canonical provenance"): + archive.attach_checkpoint_provenance(run_id, object()) + + invalid = SimpleNamespace(to_json=lambda: json.dumps({"artifact_id": "artifact-a"})) + with pytest.raises(ValueError, match="canonical provenance"): + archive.attach_checkpoint_provenance(run_id, invalid) + + def provenance(output_digest: str): + return build_provenance( + sources=( + ArtifactIdentity( + "generated", + "run-archive-test", + "v1", + "sha256:" + "a" * 64, + ), + ), + converter=ToolIdentity("test", "1", "c" * 40), + obliteratus_commit="c" * 40, + configuration_digest=None, + tokenizer=None, + base_model=None, + command=("checkpoint", "attach"), + environment={"python": "test", "platform": "cpu", "packages": {}}, + source_topology={}, + lineage=(), + input_digests=("sha256:" + "a" * 64,), + output_digests=(output_digest,), + transformations=(), + observed_scopes=("model_weights",), + lost_state=(), + ) + + first = provenance("sha256:" + "b" * 64) + attached = archive.attach_checkpoint_provenance(run_id, first) + assert attached["artifact_id"] == first.artifact_id + assert attached["checkpoint_provenance"]["sha256"].startswith("sha256:") + + second = provenance("sha256:" + "d" * 64) + with pytest.raises(ValueError, match="different artifact identity"): + archive.attach_checkpoint_provenance(run_id, second) + + +def test_evaluation_validation_and_failed_terminal_record(tmp_path): + archive = RunArchive(tmp_path) + run_id = archive.begin(["org/model"]) + + with pytest.raises(ValueError, match="unsupported evaluation partition"): + archive.begin_evaluation(run_id, partition="training", evaluator="test-v1") + + checkpoint = archive._run_dir(run_id) / "checkpoint" + checkpoint.mkdir() + (checkpoint / "weights.bin").write_bytes(b"weights") + archive.complete(run_id, checkpoint=checkpoint, metrics={}) + with pytest.raises(ValueError, match="requires qwen38-v1"): + archive.begin_evaluation(run_id, partition="optimizer_tune", evaluator="test-v1") + + archive.record_experiment_protocol( + run_id, + {"protocol": "qwen38-v1", "manifest_sha256": "a" * 64, "counts": {}}, + ) + reservation = archive.begin_evaluation( + run_id, + partition="optimizer_tune", + evaluator="test-v1", + ) + failed = archive.finish_evaluation( + run_id, + reservation["evaluation_id"], + failure=RuntimeError("Bearer abcdefghijklmnop"), + log=["failed safely"], + ) + assert failed["status"] == "failed" + assert failed["failure"] == {"type": "RuntimeError", "message": "[REDACTED]"} + + with pytest.raises(ValueError, match="invalid evaluation ID"): + archive.finish_evaluation(run_id, "invalid") + with pytest.raises(KeyError, match="unknown evaluation ID"): + archive.finish_evaluation(run_id, "eval-" + "e" * 32) + with pytest.raises(ValueError, match="already terminal"): + archive.finish_evaluation(run_id, reservation["evaluation_id"]) + + +def test_complete_cancel_prune_and_worker_identity_fail_closed(tmp_path, monkeypatch): + archive = RunArchive(tmp_path) + run_id = archive.begin(["org/model"]) + + with pytest.raises(ValueError, match="managed run checkpoint"): + archive.complete(run_id, checkpoint=tmp_path) + with pytest.raises(ValueError, match="operator reason"): + archive.prune_checkpoint(run_id, reason=" ") + + queued = archive._load(run_id) + assert archive._worker_matches({"worker": {"pid": 1}}) is False + queued["worker"] = {"pid": 9001, "uid": -1, "start_ticks": 10} + assert archive._worker_matches(queued) is False + queued["worker"] = {"pid": 9001, "uid": os.getuid(), "start_ticks": 10} + monkeypatch.setattr("obliteratus.run_archive._process_start_ticks", lambda _pid: 10) + assert archive._worker_matches(queued) is True + + archive.fail(run_id, RuntimeError("failed"), phase="pipeline") + assert archive.cancel(run_id)["status"] == "failed" + + live_run = archive.begin(["org/model"]) + live = archive._load(live_run) + live["worker"] = {"pid": 9002, "uid": os.getuid(), "start_ticks": 11} + archive._save(live) + monkeypatch.setattr("obliteratus.run_archive._process_start_ticks", lambda _pid: 11) + monkeypatch.setattr( + "obliteratus.run_archive.os.killpg", + lambda *_args: (_ for _ in ()).throw(PermissionError("denied")), + ) + with pytest.raises(RuntimeError, match="cannot cancel worker"): + archive.cancel(live_run) + + +def test_option_dataset_metric_and_main_edge_contracts(tmp_path, monkeypatch): + payload = tmp_path / "pairs.json" + payload.write_bytes(b"pairs") + arguments = [ + "model", + "--dataset=custom", + "--prompt-pairs-file", + str(payload), + "--residue-file=missing.json", + ] + assert _option_value(arguments, "--dataset") == "custom" + assert _option_value(arguments, "--absent") is None + inputs = _dataset_inputs(arguments) + assert inputs[0]["identifier"] == "custom" + assert inputs[1]["sha256"] == hashlib.sha256(b"pairs").hexdigest() + assert inputs[2]["sha256"] is None + + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + assert _checkpoint_metrics(checkpoint) == {} + (checkpoint / "abliteration_metadata.json").write_text( + json.dumps({"metrics": ["invalid"]}), + encoding="utf-8", + ) + assert _checkpoint_metrics(checkpoint) == {} + + with pytest.raises(SystemExit, match="requires obliteration arguments"): + main(["worker", "--archive-root", str(tmp_path), "--run-id", "run-" + "f" * 32]) + + observed = [] + monkeypatch.setattr( + "obliteratus.run_archive._worker", + lambda archive, run_id, args: observed.append((archive.root, run_id, args)) or 7, + ) + run_id = "run-" + "f" * 32 + assert ( + main( + [ + "worker", + "--archive-root", + str(tmp_path), + "--run-id", + run_id, + "--", + "org/model", + ] + ) + == 7 + ) + assert observed == [(tmp_path.resolve(), run_id, ["org/model"])] diff --git a/uv.lock b/uv.lock index df1df41..6b81ec8 100644 --- a/uv.lock +++ b/uv.lock @@ -1592,6 +1592,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, ] +[[package]] +name = "jsonschema" +version = "4.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/69/f7185de793a29082a9f3c7728268ffb31cb5095131a9c139a74078e27336/jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85", size = 357342, upload-time = "2025-08-18T17:03:50.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/9c/8c95d856233c1f82500c2450b8c68576b4cf1c871db3afac5c34ff84e6fd/jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63", size = 90040, upload-time = "2025-08-18T17:03:48.373Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "keyring" version = "25.7.0" @@ -2697,6 +2725,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "hypothesis" }, + { name = "jsonschema" }, { name = "mypy" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -2738,6 +2767,7 @@ requires-dist = [ { name = "flash-linear-attention", marker = "extra == 'qwen-hybrid'", specifier = ">=0.5.2" }, { name = "gradio", marker = "extra == 'spaces'", specifier = ">=6.7,<7.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = "==6.165.3" }, + { name = "jsonschema", marker = "extra == 'dev'", specifier = "==4.25.1" }, { name = "matplotlib", specifier = ">=3.7" }, { name = "mypy", marker = "extra == 'dev'", specifier = "==1.13.0" }, { name = "numpy", specifier = ">=1.24" }, @@ -3625,6 +3655,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/1b/295bf2fa3e740131778065e5ffa2c481f0e7210182d408e9a2c244ff5b0c/readme_renderer-45.0-py3-none-any.whl", hash = "sha256:3385ed220117104a2bceb4a9dac8c5fdf6d1f96890d7ea2a9c7174fd5c84091f", size = 14134, upload-time = "2026-06-09T21:05:15.85Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "regex" version = "2026.7.19" @@ -3795,6 +3840,265 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')", + "python_full_version < '3.11' and sys_platform != 'linux' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/4ba3d0fb7297ebae71171822554abe48d7cab29c28b8f9f2c04b79988c05/rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00", size = 359751, upload-time = "2025-11-30T20:21:34.591Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7c/e4933565ef7f7a0818985d87c15d9d273f1a649afa6a52ea35ad011195ea/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6", size = 389696, upload-time = "2025-11-30T20:21:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/01/6271a2511ad0815f00f7ed4390cf2567bec1d4b1da39e2c27a41e6e3b4de/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7", size = 403136, upload-time = "2025-11-30T20:21:37.728Z" }, + { url = "https://files.pythonhosted.org/packages/55/64/c857eb7cd7541e9b4eee9d49c196e833128a55b89a9850a9c9ac33ccf897/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324", size = 524699, upload-time = "2025-11-30T20:21:38.92Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ed/94816543404078af9ab26159c44f9e98e20fe47e2126d5d32c9d9948d10a/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df", size = 412022, upload-time = "2025-11-30T20:21:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/707f6cf0066a6412aacc11d17920ea2e19e5b2f04081c64526eb35b5c6e7/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3", size = 390522, upload-time = "2025-11-30T20:21:42.17Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/57a85fda37a229ff4226f8cbcf09f2a455d1ed20e802ce5b2b4a7f5ed053/rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221", size = 404579, upload-time = "2025-11-30T20:21:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/c9339293513ec680a721e0e16bf2bac3db6e5d7e922488de471308349bba/rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7", size = 421305, upload-time = "2025-11-30T20:21:44.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/be/522cb84751114f4ad9d822ff5a1aa3c98006341895d5f084779b99596e5c/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff", size = 572503, upload-time = "2025-11-30T20:21:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/de879f7e7ceddc973ea6e4629e9b380213a6938a249e94b0cdbcc325bb66/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7", size = 598322, upload-time = "2025-11-30T20:21:48.709Z" }, + { url = "https://files.pythonhosted.org/packages/48/ac/f01fc22efec3f37d8a914fc1b2fb9bcafd56a299edbe96406f3053edea5a/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139", size = 560792, upload-time = "2025-11-30T20:21:50.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/da/4e2b19d0f131f35b6146425f846563d0ce036763e38913d917187307a671/rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464", size = 221901, upload-time = "2025-11-30T20:21:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/96/cb/156d7a5cf4f78a7cc571465d8aec7a3c447c94f6749c5123f08438bcf7bc/rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169", size = 235823, upload-time = "2025-11-30T20:21:52.505Z" }, + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(python_full_version >= '3.14' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform == 'win32')", + "(python_full_version == '3.13.*' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform == 'win32')", + "python_full_version >= '3.14' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'linux' and sys_platform != 'win32'", + "(python_full_version == '3.12.*' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform == 'win32')", + "python_full_version == '3.12.*' and sys_platform != 'linux' and sys_platform != 'win32'", + "(python_full_version == '3.11.*' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform == 'win32')", + "python_full_version == '3.11.*' and sys_platform != 'linux' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + [[package]] name = "ruff" version = "0.16.2"