CalvinBackup/OBLITERATUS
1
0
Fork 0
mirror of https://github.com/elder-plinius/OBLITERATUS.git synced 2026-09-22 01:10:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add safe distributed checkpoint intake and preflight

Browse Source
This commit is contained in:
Joseph Magly
2026-09-04 19:43:53 -04:00
parent 5cc43c6e52
commit 985c9e9363
108 changed files with 21726 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/case.json
Vendored
+161
View File
@@ -0,0 +1,161 @@
{
"case_id": "mixed-model-peft",
"components": [
"full_model",
"peft_adapter"
],
"expected_manifest_digest": "sha256:a37bcf41c60410c78430f767fcd0ab4c413ceca7fb9ef9da9bbb35f542b0043f",
"features": [
"mixed_full_model_peft"
],
"fragments": [
{
"component_id": "peft-adapter",
"dtype": "float32",
"element_extent": [
1,
2
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:mixed-adapter"
],
"fqn": "adapter.lora_A.weight",
"fragment_digest": "sha256:deea3b24add66f9c401d38a758eb5cb664db0596a3113b5ceaf8c5e774faa321",
"fragment_id": "mixed-adapter",
"global_shape": [
1,
2
],
"local_shape": [
1,
2
],
"logical_tensor_id": "adapter.lora_A.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00001.safetensors",
"payload_key": "mixed-adapter",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00001.safetensors",
"shard_file_id": "rank-00001.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
1
]
]
},
{
"component_id": "full-model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:mixed-model"
],
"fqn": "model.weight",
"fragment_digest": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe",
"fragment_id": "mixed-model",
"global_shape": [
2,
2
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "mixed-model",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "float32",
"logical_tensor_id": "adapter.lora_A.weight",
"payload_key": "tensor_000",
"sha256": "sha256:deea3b24add66f9c401d38a758eb5cb664db0596a3113b5ceaf8c5e774faa321",
"shape": [
1,
2
]
},
{
"dtype": "float32",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_001",
"sha256": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe",
"shape": [
2,
2
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"world_size": 2
},
"target": {
"world_size": 1
}
},
"world_size": 2
}
tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/mixed-model-peft/rank-00001.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/case.json
Vendored
+310
View File
@@ -0,0 +1,310 @@
{
"case_id": "tp2-pp2-to-single",
"components": [
"full_model"
],
"expected_manifest_digest": "sha256:03da2d1dafe858e225174f12394d243dcaf70516c2649bca85c57dce91dff323",
"features": [
"pipeline_parallel",
"topology_a_to_b"
],
"fragments": [
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:tp-pp-r0"
],
"fqn": "model.weight",
"fragment_digest": "sha256:36f52612ab7fad5ef2adca0d34f41471c0c38f50cc92d9a47fa662631d3e1268",
"fragment_id": "tp-pp-r0",
"global_shape": [
4,
4
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00000.safetensors",
"payload_key": "tp-pp-r0",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"pp",
0
],
[
"rank",
0
],
[
"tp",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
0,
2
],
"evidence_refs": [
"synthetic:tp-pp-r1"
],
"fqn": "model.weight",
"fragment_digest": "sha256:762921576e058fd1360006105a48a8225d6baa27085f7d30abed455c288f9308",
"fragment_id": "tp-pp-r1",
"global_shape": [
4,
4
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00001.safetensors",
"payload_key": "tp-pp-r1",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00001.safetensors",
"shard_file_id": "rank-00001.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"pp",
0
],
[
"rank",
1
],
[
"tp",
1
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
2,
0
],
"evidence_refs": [
"synthetic:tp-pp-r2"
],
"fqn": "model.weight",
"fragment_digest": "sha256:5cca39bfe5a82135bad95ae98f2df321fe08ea75b68ea8f939ffa90845138146",
"fragment_id": "tp-pp-r2",
"global_shape": [
4,
4
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00002.safetensors",
"payload_key": "tp-pp-r2",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00002.safetensors",
"shard_file_id": "rank-00002.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"pp",
1
],
[
"rank",
2
],
[
"tp",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
2,
2
],
"evidence_refs": [
"synthetic:tp-pp-r3"
],
"fqn": "model.weight",
"fragment_digest": "sha256:dbb7087f7cd13a205e0dcec5093a66dcfac4f6c237afd8dcafd64d166227083a",
"fragment_id": "tp-pp-r3",
"global_shape": [
4,
4
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00003.safetensors",
"payload_key": "tp-pp-r3",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00003.safetensors",
"shard_file_id": "rank-00003.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"pp",
1
],
[
"rank",
3
],
[
"tp",
1
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "float32",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_000",
"sha256": "sha256:58dda328598e2f7fe472621bfc54935aaa354d1a6ebcaf9562cd743fd575eb19",
"shape": [
4,
4
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"pp": 2,
"tp": 2
},
"target": {
"world_size": 1
}
},
"world_size": 4
}
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00001.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00002.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/tp2-pp2-to-single/rank-00003.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/case.json
Vendored
+393
View File
@@ -0,0 +1,393 @@
{
"case_id": "world1-complete",
"components": [
"full_model"
],
"expected_manifest_digest": "sha256:a6f4338c200816ce0a71bc9034e7c4792528f5520273c94718d332e995649065",
"features": [
"buffer",
"expert",
"scalar",
"tied_weight"
],
"fragments": [
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:w1-buffer"
],
"fqn": "model.running_mean",
"fragment_digest": "sha256:ab2a21ab5e1262d555eea5678035e8fe3e76542c6b474cc04a16a39d4aa02645",
"fragment_id": "w1-buffer",
"global_shape": [
2
],
"local_shape": [
2
],
"logical_tensor_id": "model.running_mean",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-buffer",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "persistent_buffer",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
4
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:w1-embed"
],
"fqn": "model.embed.weight",
"fragment_digest": "sha256:ad73b9acd6e4a74b2f5bb5386658ce3bb146cd040a1867646ab3b973fb6632b1",
"fragment_id": "w1-embed",
"global_shape": [
4
],
"local_shape": [
4
],
"logical_tensor_id": "model.embed.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-embed",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": "tie-embedding-head",
"tie_group_id": "tie-embedding-head",
"topology_coordinates": [
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
2
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:w1-expert"
],
"fqn": "model.experts.0.weight",
"fragment_digest": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe",
"fragment_id": "w1-expert",
"global_shape": [
2,
2
],
"local_shape": [
2,
2
],
"logical_tensor_id": "model.experts.0.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-expert",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"ep",
0
],
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
4
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:w1-head"
],
"fqn": "lm_head.weight",
"fragment_digest": "sha256:ad73b9acd6e4a74b2f5bb5386658ce3bb146cd040a1867646ab3b973fb6632b1",
"fragment_id": "w1-head",
"global_shape": [
4
],
"local_shape": [
4
],
"logical_tensor_id": "lm_head.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-head",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": "tie-embedding-head",
"tie_group_id": "tie-embedding-head",
"topology_coordinates": [
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "int64",
"element_extent": [],
"element_offset": [],
"evidence_refs": [
"synthetic:w1-scalar"
],
"fqn": "model.step",
"fragment_digest": "sha256:35be322d094f9d154a8aba4733b8497f180353bd7ae7b0a15f90b586b549f28b",
"fragment_id": "w1-scalar",
"global_shape": [],
"local_shape": [],
"logical_tensor_id": "model.step",
"padding": {
"after": [],
"before": [],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-scalar",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "persistent_buffer",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
3
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:w1-weight"
],
"fqn": "model.weight",
"fragment_digest": "sha256:e2c0a71510b5394df7773b63fb5f54372b84c3564e67811bde7d665be227976d",
"fragment_id": "w1-weight",
"global_shape": [
2,
3
],
"local_shape": [
2,
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "w1-weight",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "float32",
"logical_tensor_id": "lm_head.weight",
"payload_key": "tensor_000",
"sha256": "sha256:ad73b9acd6e4a74b2f5bb5386658ce3bb146cd040a1867646ab3b973fb6632b1",
"shape": [
4
]
},
{
"dtype": "float32",
"logical_tensor_id": "model.embed.weight",
"payload_key": "tensor_001",
"sha256": "sha256:ad73b9acd6e4a74b2f5bb5386658ce3bb146cd040a1867646ab3b973fb6632b1",
"shape": [
4
]
},
{
"dtype": "float32",
"logical_tensor_id": "model.experts.0.weight",
"payload_key": "tensor_002",
"sha256": "sha256:4c9c4f354e74153db012329d71c8562ec23e498148174b2c49de58f45d47cdbe",
"shape": [
2,
2
]
},
{
"dtype": "float32",
"logical_tensor_id": "model.running_mean",
"payload_key": "tensor_003",
"sha256": "sha256:ab2a21ab5e1262d555eea5678035e8fe3e76542c6b474cc04a16a39d4aa02645",
"shape": [
2
]
},
{
"dtype": "int64",
"logical_tensor_id": "model.step",
"payload_key": "tensor_004",
"sha256": "sha256:35be322d094f9d154a8aba4733b8497f180353bd7ae7b0a15f90b586b549f28b",
"shape": []
},
{
"dtype": "float32",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_005",
"sha256": "sha256:e2c0a71510b5394df7773b63fb5f54372b84c3564e67811bde7d665be227976d",
"shape": [
2,
3
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"world_size": 1
},
"target": {
"world_size": 1
}
},
"world_size": 1
}
tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world1-complete/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/case.json
Vendored
+150
View File
@@ -0,0 +1,150 @@
{
"case_id": "world2-uneven-1d",
"components": [
"full_model"
],
"expected_manifest_digest": "sha256:d0e2e96c6bdfb6ebd29470ef1f789b1a6a8f1ebc89624538cf56600ad3f4cb88",
"features": [
"padding",
"uneven_1d"
],
"fragments": [
{
"component_id": "model",
"dtype": "int64",
"element_extent": [
3
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:w2-r0"
],
"fqn": "model.weight",
"fragment_digest": "sha256:ab25350e3e65efebe24584461683ecda68725576e825e550038b90e7b1479946",
"fragment_id": "w2-r0",
"global_shape": [
7
],
"local_shape": [
4
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
1
],
"before": [
0
],
"semantic": "producer_declared"
},
"partition_axes": [
0
],
"payload_file": "rank-00000.safetensors",
"payload_key": "w2-r0",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
],
[
"tp",
0
]
]
},
{
"component_id": "model",
"dtype": "int64",
"element_extent": [
4
],
"element_offset": [
3
],
"evidence_refs": [
"synthetic:w2-r1"
],
"fqn": "model.weight",
"fragment_digest": "sha256:6df0e128a82e6d8c332ed546fca7d48406970ea60531b4d64ee41bdffc3d5da4",
"fragment_id": "w2-r1",
"global_shape": [
7
],
"local_shape": [
4
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [
0
],
"payload_file": "rank-00001.safetensors",
"payload_key": "w2-r1",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00001.safetensors",
"shard_file_id": "rank-00001.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
1
],
[
"tp",
1
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "int64",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_000",
"sha256": "sha256:81845a01dafa45c9b26e10a7af52a92e8604d5d8ef690f1e3ccdcfe3b5c6ae98",
"shape": [
7
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"tp": 2
},
"target": {
"world_size": 1
}
},
"world_size": 2
}
tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world2-uneven-1d/rank-00001.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/case.json
Vendored
+255
View File
@@ -0,0 +1,255 @@
{
"case_id": "world4-dp-replicas",
"components": [
"full_model"
],
"expected_manifest_digest": "sha256:d45ab4d7db81d4f7bd55e7ccb04bfb7c936b0bd320f1b57e77d7d02d3d488ff3",
"features": [
"dp_replica"
],
"fragments": [
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:dp-r0"
],
"fqn": "model.weight",
"fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655",
"fragment_id": "dp-r0",
"global_shape": [
3
],
"local_shape": [
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00000.safetensors",
"payload_key": "dp-r0",
"replica": {
"group_id": "dp-full",
"member_count": 4,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"dp",
0
],
[
"rank",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:dp-r1"
],
"fqn": "model.weight",
"fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655",
"fragment_id": "dp-r1",
"global_shape": [
3
],
"local_shape": [
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00001.safetensors",
"payload_key": "dp-r1",
"replica": {
"group_id": "dp-full",
"member_count": 4,
"member_index": 1
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00001.safetensors",
"shard_file_id": "rank-00001.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"dp",
1
],
[
"rank",
1
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:dp-r2"
],
"fqn": "model.weight",
"fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655",
"fragment_id": "dp-r2",
"global_shape": [
3
],
"local_shape": [
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00002.safetensors",
"payload_key": "dp-r2",
"replica": {
"group_id": "dp-full",
"member_count": 4,
"member_index": 2
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00002.safetensors",
"shard_file_id": "rank-00002.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"dp",
2
],
[
"rank",
2
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3
],
"element_offset": [
0
],
"evidence_refs": [
"synthetic:dp-r3"
],
"fqn": "model.weight",
"fragment_digest": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655",
"fragment_id": "dp-r3",
"global_shape": [
3
],
"local_shape": [
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0
],
"before": [
0
],
"semantic": "none"
},
"partition_axes": [],
"payload_file": "rank-00003.safetensors",
"payload_key": "dp-r3",
"replica": {
"group_id": "dp-full",
"member_count": 4,
"member_index": 3
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00003.safetensors",
"shard_file_id": "rank-00003.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"dp",
3
],
[
"rank",
3
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "float32",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_000",
"sha256": "sha256:7f8a2918261b8d80cae770a9aae209b29d8b39654c62aee0013a423796ddb655",
"shape": [
3
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"dp": 4
},
"target": {
"world_size": 1
}
},
"world_size": 4
}
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00001.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00002.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-dp-replicas/rank-00003.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/case.json
Vendored
+309
View File
@@ -0,0 +1,309 @@
{
"case_id": "world4-uneven-2d",
"components": [
"full_model"
],
"expected_manifest_digest": "sha256:356bddfa143562828982c2fd80b83d4a2e0c278814d844568b99aeed1f6cb310",
"features": [
"uneven_2d"
],
"fragments": [
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
3
],
"element_offset": [
0,
0
],
"evidence_refs": [
"synthetic:w4-r0"
],
"fqn": "model.weight",
"fragment_digest": "sha256:e902ad65c5c6702e7ca1da427d13ec926431470bc155ff2983b362cb10d2dff5",
"fragment_id": "w4-r0",
"global_shape": [
5,
7
],
"local_shape": [
2,
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00000.safetensors",
"payload_key": "w4-r0",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00000.safetensors",
"shard_file_id": "rank-00000.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
0
],
[
"tp_col",
0
],
[
"tp_row",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
2,
4
],
"element_offset": [
0,
3
],
"evidence_refs": [
"synthetic:w4-r1"
],
"fqn": "model.weight",
"fragment_digest": "sha256:8dc8137a498004d656f20c18b4726bf12aab21d7010b9980480317d6df38e8c8",
"fragment_id": "w4-r1",
"global_shape": [
5,
7
],
"local_shape": [
2,
4
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00001.safetensors",
"payload_key": "w4-r1",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00001.safetensors",
"shard_file_id": "rank-00001.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
1
],
[
"tp_col",
1
],
[
"tp_row",
0
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3,
3
],
"element_offset": [
2,
0
],
"evidence_refs": [
"synthetic:w4-r2"
],
"fqn": "model.weight",
"fragment_digest": "sha256:cd2d91a4484f0cd6236034895436acbee187e02fd96900593a2512d9882f7e0e",
"fragment_id": "w4-r2",
"global_shape": [
5,
7
],
"local_shape": [
3,
3
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00002.safetensors",
"payload_key": "w4-r2",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00002.safetensors",
"shard_file_id": "rank-00002.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
2
],
[
"tp_col",
0
],
[
"tp_row",
1
]
]
},
{
"component_id": "model",
"dtype": "float32",
"element_extent": [
3,
4
],
"element_offset": [
2,
3
],
"evidence_refs": [
"synthetic:w4-r3"
],
"fqn": "model.weight",
"fragment_digest": "sha256:3c4e748352f778507efe297633562c96b36d3e68030f1c230266d63fd2994179",
"fragment_id": "w4-r3",
"global_shape": [
5,
7
],
"local_shape": [
3,
4
],
"logical_tensor_id": "model.weight",
"padding": {
"after": [
0,
0
],
"before": [
0,
0
],
"semantic": "none"
},
"partition_axes": [
0,
1
],
"payload_file": "rank-00003.safetensors",
"payload_key": "w4-r3",
"replica": {
"group_id": null,
"member_count": 1,
"member_index": 0
},
"role": "parameter",
"shard_digest_ref": "source-digest:rank-00003.safetensors",
"shard_file_id": "rank-00003.safetensors",
"shared_storage_id": null,
"tie_group_id": null,
"topology_coordinates": [
[
"rank",
3
],
[
"tp_col",
1
],
[
"tp_row",
1
]
]
}
],
"oracle_file": "oracles.safetensors",
"oracles": [
{
"dtype": "float32",
"logical_tensor_id": "model.weight",
"payload_key": "tensor_000",
"sha256": "sha256:471d327907fc83cb6703d3424393e5caeefd627fa86d8b1b2f07d3045b6e1433",
"shape": [
5,
7
]
}
],
"schema_id": "obliteratus.checkpoint-fixture-case",
"schema_version": "1.0.0",
"topology": {
"source": {
"tp_columns": 2,
"tp_rows": 2
},
"target": {
"world_size": 1
}
},
"world_size": 4
}
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/oracles.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00000.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00001.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00002.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/cases/world4-uneven-2d/rank-00003.safetensors
Vendored
BIN
View File
Binary file not shown.
tests/fixtures/distributed_checkpoints/v1/fixture-corpus.json
Vendored
+233
View File
@@ -0,0 +1,233 @@
{
"cases": [
{
"case_id": "mixed-model-peft",
"features": [
"mixed_full_model_peft"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:aab22d557fdc3de06143a8db3222680fb5db42eacc2a782e79882ccf200417c6",
"size_bytes": 3577
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:74d4b70269f0c5ad034c5291fb63541629de99a4103c8190215d4c30f3387e1a",
"size_bytes": 168
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:a7cac4935027f303e4c4b37ba93726f17c49e451c2f49e9ce9b6db2c7c82721f",
"size_bytes": 96
},
{
"relative_path": "rank-00001.safetensors",
"sha256": "sha256:d9231d15d8fbc7b6f80014d4fc8b9da125c0bfcbff219ef8a7941076871260e3",
"size_bytes": 88
}
],
"relative_path": "cases/mixed-model-peft",
"world_size": 2
},
{
"case_id": "tp2-pp2-to-single",
"features": [
"pipeline_parallel",
"topology_a_to_b"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:7a667928a5f142449b2309aa6c22b8cf1996c4fe2d96558fb36f6acc2c5ab629",
"size_bytes": 6235
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:03a1b5c42ee9de97d2669f6b7d21dbc30f87edd9bd988e1e154fc03f88b72298",
"size_bytes": 144
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:7d721dd88baac995c4070fcc639de7fff9d7ff1922b1f7076dcc3470b41c34ab",
"size_bytes": 88
},
{
"relative_path": "rank-00001.safetensors",
"sha256": "sha256:13b231f41a6f11c160dab5cdd58184f9ba0e8b6e97606ea118d2785c6322f7ec",
"size_bytes": 88
},
{
"relative_path": "rank-00002.safetensors",
"sha256": "sha256:f1aa964072188df5bc93fee44a1a139c4f2a82811aefb946a5a7ea6aa38cbe9d",
"size_bytes": 88
},
{
"relative_path": "rank-00003.safetensors",
"sha256": "sha256:620e59497637e8038383b6e926b51e07a63943ef340c2e1b1d48a55a903459c2",
"size_bytes": 88
}
],
"relative_path": "cases/tp2-pp2-to-single",
"world_size": 4
},
{
"case_id": "world1-complete",
"features": [
"buffer",
"expert",
"scalar",
"tied_weight"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:8bd58508b678460697f7a2315ab4f2e48798278a38111479ee518661839a98b2",
"size_bytes": 9218
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:eb87c4db023b1272325d27e37e71511240f3cdf80df1712d261fde7d0f7b64b6",
"size_bytes": 488
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:e42a2a167781454d9f6df4a9f870cd732ba237fd32960b5a66446bda1ed5c458",
"size_bytes": 472
}
],
"relative_path": "cases/world1-complete",
"world_size": 1
},
{
"case_id": "world2-uneven-1d",
"features": [
"padding",
"uneven_1d"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:ad350b19d728c3d45961a6695d581dfd44158f389d7cf06420c444bc6e6eb358",
"size_bytes": 3205
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:7e922eba8c68cf28a0444625c79a3c5f88274e47653dae885e8a493ea946b179",
"size_bytes": 128
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:6fac58511adb9184da344e3f6c21fc5e90c42d2365c951863cf72dcc3a868c6b",
"size_bytes": 104
},
{
"relative_path": "rank-00001.safetensors",
"sha256": "sha256:fa4fb09a4ef30971540267395c156dc60a31390a936cd754280b8a658b56bb09",
"size_bytes": 104
}
],
"relative_path": "cases/world2-uneven-1d",
"world_size": 2
},
{
"case_id": "world4-dp-replicas",
"features": [
"dp_replica"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:61840aaec0045c1d648b5a4bdd93fe66612f66844ff506d1f9d98b88f5e221d5",
"size_bytes": 5576
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:42ed9ed388b42f5f0813713f449abec59446c406eca9aef376534ee49394be8e",
"size_bytes": 84
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:66354f04b342b9c935a54ec5ebf3ddfa6da49fc1c549bf76ca886dc8a9d6d428",
"size_bytes": 84
},
{
"relative_path": "rank-00001.safetensors",
"sha256": "sha256:a58e05fb72c9e48ecf24444e150869225ccdb9ddaec1107015171d1d687e7b18",
"size_bytes": 84
},
{
"relative_path": "rank-00002.safetensors",
"sha256": "sha256:ed9633e59e38118d3e8b7121da9c8b8893e3ab2b00bfc4ccb3947350bdc615ef",
"size_bytes": 84
},
{
"relative_path": "rank-00003.safetensors",
"sha256": "sha256:07ac85546ac39bfea2047461b45ea99f68e76adf65dec5f8c63b544d9c32bd32",
"size_bytes": 84
}
],
"relative_path": "cases/world4-dp-replicas",
"world_size": 4
},
{
"case_id": "world4-uneven-2d",
"features": [
"uneven_2d"
],
"files": [
{
"relative_path": "case.json",
"sha256": "sha256:4ccce62bc46cd9738b0ea21de982438dde8edca10ae2a565d6d8d1094482af06",
"size_bytes": 6212
},
{
"relative_path": "oracles.safetensors",
"sha256": "sha256:7109b6b4db6043689e5483ff56422079f4778abc5f92e375e5ee8e0b564cb06a",
"size_bytes": 220
},
{
"relative_path": "rank-00000.safetensors",
"sha256": "sha256:a062693caed152b759d04229348c5ef9d27f4276ffed7ad5b21cc9e3d381ffa6",
"size_bytes": 96
},
{
"relative_path": "rank-00001.safetensors",
"sha256": "sha256:3932804176b74fe5573e4f4f9496ab95ca911aa0cc2f84d6d2b5553a3bbead84",
"size_bytes": 104
},
{
"relative_path": "rank-00002.safetensors",
"sha256": "sha256:2dc96b01af5c564018de0dd8f0e94783c9dd4321e307901dab486d2456a6e654",
"size_bytes": 108
},
{
"relative_path": "rank-00003.safetensors",
"sha256": "sha256:523140f0770d0b2145d72d03bda03c702455f6b57a7bf6efb35a479eb118b3df",
"size_bytes": 120
}
],
"relative_path": "cases/world4-uneven-2d",
"world_size": 4
}
],
"generator": {
"path": "scripts/generate_checkpoint_fixtures.py",
"version": "1.0.0"
},
"license": "AGPL-3.0-or-later",
"limits": {
"max_case_bytes": 65536,
"max_cases": 16,
"max_files_per_case": 16,
"max_tensors_per_case": 16
},
"provenance": {
"kind": "deterministic_synthetic",
"seed": 0,
"third_party_data": false,
"third_party_weights": false
},
"schema_id": "obliteratus.checkpoint-fixture-corpus",
"schema_version": "1.0.0"
}
tests/fixtures/distributed_checkpoints/v1/negative-cases.json
Vendored
+108
View File
@@ -0,0 +1,108 @@
{
"cases": [
{
"case_id": "negative-01",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "coverage_gap",
"mutation": "deterministic:coverage_gap"
},
{
"case_id": "negative-02",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "coverage_overlap",
"mutation": "deterministic:coverage_overlap"
},
{
"case_id": "negative-03",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "dimension_mismatch",
"mutation": "deterministic:dimension_mismatch"
},
{
"case_id": "negative-04",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "extra_shard",
"mutation": "deterministic:extra_shard"
},
{
"case_id": "negative-05",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "fragment_out_of_bounds",
"mutation": "deterministic:fragment_out_of_bounds"
},
{
"case_id": "negative-06",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "integer_overflow",
"mutation": "deterministic:integer_overflow"
},
{
"case_id": "negative-07",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "missing_shard",
"mutation": "deterministic:missing_shard"
},
{
"case_id": "negative-08",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "negative_integer",
"mutation": "deterministic:negative_integer"
},
{
"case_id": "negative-09",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "padding_shape_mismatch",
"mutation": "deterministic:padding_shape_mismatch"
},
{
"case_id": "negative-10",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "path_traversal",
"mutation": "deterministic:path_traversal"
},
{
"case_id": "negative-11",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "payload_dtype_mismatch",
"mutation": "deterministic:payload_dtype_mismatch"
},
{
"case_id": "negative-12",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "payload_shape_mismatch",
"mutation": "deterministic:payload_shape_mismatch"
},
{
"case_id": "negative-13",
"expected_code": "DCI_VALIDATION_FAILED",
"failure": "replica_digest_mismatch",
"mutation": "deterministic:replica_digest_mismatch"
},
{
"case_id": "negative-14",
"expected_code": "DCI_RESOURCE_LIMIT",
"failure": "resource_manifest_bomb",
"mutation": "deterministic:resource_manifest_bomb"
},
{
"case_id": "negative-15",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "source_special_file",
"mutation": "deterministic:source_special_file"
},
{
"case_id": "negative-16",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "source_symlink",
"mutation": "deterministic:source_symlink"
},
{
"case_id": "negative-17",
"expected_code": "DCI_SOURCE_BOUNDARY_VIOLATION",
"failure": "truncated_shard",
"mutation": "deterministic:truncated_shard"
}
],
"schema_id": "obliteratus.checkpoint-negative-fixtures",
"schema_version": "1.0.0"
}
tests/test_checkpoint_capabilities.py
+303
View File
@@ -0,0 +1,303 @@
"""Producer-neutral checkpoint capability and dependency diagnostics."""
from __future__ import annotations
import pytest
import obliteratus.checkpoint_capabilities as capability_module
from obliteratus.checkpoint_capabilities import (
AdapterCapability,
AdapterRegistry,
ExactDependency,
registry_from,
)
def _capability(
adapter_id: str = "example-dcp",
*,
checkpoint_format: str = "pytorch_dcp",
distribution: str = "example-producer",
version: str = "1.2.3",
) -> AdapterCapability:
return AdapterCapability(
adapter_id=adapter_id,
adapter_version="0.1.0",
producer=distribution,
producer_version=version,
formats=(checkpoint_format,),
required_extras=("checkpoint-example",),
required_dependencies=(ExactDependency(distribution, version),),
)
def test_empty_registry_does_not_guess_an_adapter_or_dependency():
resolution = AdapterRegistry().resolve(
"pytorch_dcp",
version_provider=lambda _name: pytest.fail("package metadata queried"),
)
assert resolution.to_dict() == {
"status": "missing",
"adapter_id": None,
"adapter_version": None,
"capability_digest": None,
"reason": (
"No approved exact capability is registered; structural recognition "
"does not select an adapter or dependency set."
),
}
@pytest.mark.parametrize("observed", [None, "1.2.2"])
def test_missing_or_incompatible_dependency_has_exact_bounded_diagnostic(observed):
capability = _capability()
resolution = AdapterRegistry((capability,)).resolve(
"pytorch_dcp",
version_provider=lambda name: observed if name == "example-producer" else None,
project_version="9.8.7",
observed_producer="example-producer",
observed_producer_version="1.2.3",
)
assert resolution.status == "missing"
assert resolution.adapter_id == "example-dcp"
assert resolution.adapter_version == "0.1.0"
assert resolution.capability_digest == capability.capability_digest
assert resolution.reason == (
"source_identity=verified; dependency_status=missing_or_incompatible; "
"install_extra=obliteratus[checkpoint-example]==9.8.7; "
"required_versions=example-producer==1.2.3; "
f"observed_versions=example-producer={observed or '<missing>'}"
)
def test_present_exact_dependency_matches_without_granting_trust():
capability = _capability()
resolution = AdapterRegistry((capability,)).resolve(
"pytorch_dcp",
version_provider=lambda _name: "1.2.3",
observed_producer="example-producer",
observed_producer_version="1.2.3",
)
assert resolution.status == "matched"
assert resolution.adapter_id == capability.adapter_id
assert resolution.capability_digest == capability.capability_digest
assert "trust authorization is still required" in resolution.reason
def test_format_only_candidate_never_becomes_an_exact_match():
resolution = AdapterRegistry((_capability(),)).resolve(
"pytorch_dcp",
version_provider=lambda _name: "1.2.3",
)
assert resolution.status == "missing"
assert resolution.adapter_id == "example-dcp"
assert resolution.dependency_unavailable is False
assert resolution.reason.startswith("source_identity=unverified")
def test_observed_source_identity_must_be_complete_and_match_exactly():
registry = AdapterRegistry((_capability(),))
with pytest.raises(ValueError, match="provided together"):
registry.resolve("pytorch_dcp", observed_producer="example-producer")
resolution = registry.resolve(
"pytorch_dcp",
observed_producer="example-producer",
observed_producer_version="1.2.4",
version_provider=lambda _name: pytest.fail("package metadata queried"),
)
assert resolution.status == "missing"
assert resolution.adapter_id is None
assert "No exact capability matches" in resolution.reason
def test_resolution_and_digest_are_deterministic_for_explicit_records():
first = _capability(
"adapter-b",
checkpoint_format="deepspeed_zero",
distribution="deepspeed",
version="0.16.1",
)
second = _capability(
"adapter-a",
checkpoint_format="megatron_torch_dist",
distribution="megatron-core",
version="0.16.1",
)
registry = registry_from([first, second])
assert registry.capabilities == (second, first)
assert first.capability_digest == _capability(
"adapter-b",
checkpoint_format="deepspeed_zero",
distribution="deepspeed",
version="0.16.1",
).capability_digest
def test_multiple_format_matches_fail_closed_without_dependency_queries():
registry = AdapterRegistry((_capability("adapter-a"), _capability("adapter-b")))
resolution = registry.resolve(
"pytorch_dcp",
version_provider=lambda _name: pytest.fail("package metadata queried"),
)
assert resolution.status == "ambiguous"
assert resolution.adapter_id is None
assert resolution.reason == "Multiple exact capabilities match: adapter-a,adapter-b."
@pytest.mark.parametrize(
("factory", "message"),
[
(lambda: ExactDependency("bad name", "1.0.0"), "distribution"),
(lambda: ExactDependency("producer", "unselected"), "version"),
(
lambda: AdapterCapability(
"adapter",
"1.0.0",
"producer",
"1.0.0",
("hf_safetensors",),
("extra",),
(ExactDependency("producer", "1.0.0"),),
),
"trust-required",
),
(
lambda: AdapterRegistry((_capability(), _capability())),
"identifiers must be unique",
),
],
)
def test_invalid_or_non_exact_capabilities_are_rejected(factory, message):
with pytest.raises(ValueError, match=message):
factory()
def test_untrusted_observed_version_is_redacted_from_diagnostic():
resolution = AdapterRegistry((_capability(),)).resolve(
"pytorch_dcp",
version_provider=lambda _name: "bad\nlocal-path=/secret",
)
assert "<invalid>" in resolution.reason
assert "/secret" not in resolution.reason
def test_registry_and_capability_size_limits_are_bounded():
dependency = ExactDependency("producer", "1.0.0")
with pytest.raises(ValueError, match="required_extras exceeds"):
AdapterCapability(
"adapter",
"1.0.0",
"producer",
"1.0.0",
("pytorch_dcp",),
tuple(f"extra-{index}" for index in range(9)),
(dependency,),
)
with pytest.raises(ValueError, match="registry exceeds"):
AdapterRegistry(
tuple(_capability(f"adapter-{index}") for index in range(17))
)
def test_capability_and_registry_require_immutable_typed_records():
dependency = ExactDependency("producer", "1.0.0")
with pytest.raises(TypeError, match="immutable tuples"):
AdapterCapability(
"adapter",
"1.0.0",
"producer",
"1.0.0",
["pytorch_dcp"], # type: ignore[arg-type]
("extra",),
(dependency,),
)
with pytest.raises(TypeError, match="ExactDependency"):
AdapterCapability(
"adapter",
"1.0.0",
"producer",
"1.0.0",
("pytorch_dcp",),
("extra",),
("not-a-record",), # type: ignore[arg-type]
)
with pytest.raises(TypeError, match="immutable tuple"):
AdapterRegistry([_capability()]) # type: ignore[arg-type]
with pytest.raises(TypeError, match="AdapterCapability"):
AdapterRegistry(("not-a-capability",)) # type: ignore[arg-type]
@pytest.mark.parametrize(
("formats", "extras", "dependencies", "message"),
[
((), ("extra",), (ExactDependency("producer", "1.0.0"),), "formats must"),
(("pytorch_dcp",), (), (ExactDependency("producer", "1.0.0"),), "extras must"),
(("pytorch_dcp",), ("extra",), (), "dependencies must"),
(("pytorch_dcp",) * 9, ("extra",), (ExactDependency("producer", "1.0.0"),), "formats exceeds"),
(
("pytorch_dcp",),
("extra",),
tuple(ExactDependency(f"producer-{index}", "1.0.0") for index in range(5)),
"dependencies exceeds",
),
(
("pytorch_dcp", "pytorch_dcp"),
("extra",),
(ExactDependency("producer", "1.0.0"),),
"formats must be unique",
),
(
("pytorch_dcp",),
("extra", "extra"),
(ExactDependency("producer", "1.0.0"),),
"extras must be unique",
),
(
("pytorch_dcp",),
("extra",),
(ExactDependency("producer", "1.0.0"),) * 2,
"distributions must be unique",
),
(
("pytorch_dcp",),
("bad extra",),
(ExactDependency("producer", "1.0.0"),),
"required_extra",
),
],
)
def test_capability_collections_are_strict_and_bounded(
formats,
extras,
dependencies,
message,
):
with pytest.raises(ValueError, match=message):
AdapterCapability(
"adapter",
"1.0.0",
"producer",
"1.0.0",
formats,
extras,
dependencies,
)
def test_installed_version_probe_returns_metadata_or_absence(monkeypatch):
monkeypatch.setattr(capability_module.metadata, "version", lambda _name: "1.2.3")
assert capability_module._installed_version("example") == "1.2.3"
def missing(_name):
raise capability_module.metadata.PackageNotFoundError
monkeypatch.setattr(capability_module.metadata, "version", missing)
assert capability_module._installed_version("example") is None
tests/test_checkpoint_contract_schemas.py
+101
View File
@@ -0,0 +1,101 @@
"""Executable Draft 2020-12 checks for distributed-checkpoint contracts."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from jsonschema import Draft202012Validator, FormatChecker
ROOT = Path(__file__).resolve().parents[1]
CONTRACTS = ROOT / "docs/checkpoints/schemas"
FIXTURES = CONTRACTS / "fixtures/v1"
def _load(path: Path) -> dict:
value = json.loads(path.read_text(encoding="utf-8"))
assert isinstance(value, dict)
return value
def _validators() -> dict[str, Draft202012Validator]:
result = {}
for path in sorted(CONTRACTS.glob("*.schema.json")):
schema = _load(path)
Draft202012Validator.check_schema(schema)
schema_id = schema["properties"]["schema_id"]["const"]
assert schema_id not in result
result[schema_id] = Draft202012Validator(schema, format_checker=FormatChecker())
return result
def test_every_checkpoint_schema_is_valid_and_has_a_unique_contract_id():
validators = _validators()
assert {
"obliteratus.adapter-capability",
"obliteratus.checkpoint-descriptor",
"obliteratus.checkpoint-error-registry",
"obliteratus.checkpoint-support-matrix",
"obliteratus.checkpoint-trust-policy",
"obliteratus.conversion-manifest",
"obliteratus.trusted-worker-message",
} <= set(validators)
@pytest.mark.parametrize("path", sorted((FIXTURES / "valid").glob("*.json")))
def test_declared_valid_contract_fixture_passes(path):
instance = _load(path)
_validators()[instance["schema_id"]].validate(instance)
@pytest.mark.parametrize("path", sorted((FIXTURES / "invalid").glob("*.json")))
def test_declared_invalid_contract_fixture_fails(path):
instance = _load(path)
errors = list(_validators()[instance["schema_id"]].iter_errors(instance))
assert errors
def test_trust_policy_forbids_persisted_environment_or_secret_fields():
policy = _load(FIXTURES / "valid/trusted-metadata-policy.json")
policy["environment"] = {"TOKEN": "must-not-persist"}
errors = list(_validators()[policy["schema_id"]].iter_errors(policy))
assert any("Additional properties are not allowed" in error.message for error in errors)
def test_support_matrix_and_error_registry_validate_against_their_schemas():
validators = _validators()
for path in (
ROOT / "docs/checkpoints/support-matrix-v1.json",
CONTRACTS / "checkpoint-error-codes-v1.json",
):
instance = _load(path)
validators[instance["schema_id"]].validate(instance)
def test_error_registry_covers_every_fail_closed_degraded_mode_once_or_more():
registry = _load(CONTRACTS / "checkpoint-error-codes-v1.json")
entries = registry["entries"]
assert len({entry["code"] for entry in entries}) == len(entries)
covered = {mode for entry in entries for mode in entry["degraded_modes"]}
assert covered == {f"F{number:02d}" for number in range(1, 21)}
def test_descriptor_blockers_accept_the_canonical_error_registry_vocabulary():
descriptor = _load(CONTRACTS / "checkpoint-descriptor-v1.schema.json")
registry = _load(CONTRACTS / "checkpoint-error-codes-v1.json")
blocker = descriptor["$defs"]["blocker"]["properties"]
assert {entry["category"] for entry in registry["entries"]} <= set(
blocker["category"]["enum"]
)
assert {entry["phase"] for entry in registry["entries"]} <= set(blocker["phase"]["enum"])
def test_descriptor_can_name_legacy_hf_pickle_without_treating_it_as_safetensors():
descriptor = _load(CONTRACTS / "checkpoint-descriptor-v1.schema.json")
assert "hf_pytorch_pickle" in descriptor["$defs"]["checkpointFormat"]["enum"]
tests/test_checkpoint_docs_contracts.py
+108
View File
@@ -0,0 +1,108 @@
"""Offline contracts for checkpoint documentation and support claims."""
from __future__ import annotations
import json
from pathlib import Path
from scripts import check_checkpoint_docs
ROOT = Path(__file__).resolve().parents[1]
def _write_json(path: Path, value: object) -> None:
path.write_text(json.dumps(value), encoding="utf-8")
def _matrix() -> dict:
return json.loads(
(ROOT / "docs/checkpoints/support-matrix-v1.json").read_text(encoding="utf-8"),
)
def test_checkpoint_documentation_contract_passes_offline():
assert check_checkpoint_docs.validate_all() == []
def test_support_matrix_rejects_unknown_fields_and_duplicate_rows(tmp_path):
matrix = _matrix()
matrix["unexpected"] = True
matrix["rows"].append(matrix["rows"][0])
path = tmp_path / "matrix.json"
_write_json(path, matrix)
errors = check_checkpoint_docs.validate_matrix(path)
assert "support matrix has unknown fields: unexpected" in errors
assert any("duplicate row id" in error for error in errors)
def test_supported_claim_requires_exact_retained_evidence(tmp_path):
matrix = _matrix()
row = matrix["rows"][0]
row["capabilities"]["detect"]["value"] = "supported"
row["producer_versions"] = ["Transformers compatible; exact version varies"]
row["evidence"].update(
candidate_commit=None,
fixture_digest=None,
retained_result=None,
)
path = tmp_path / "matrix.json"
_write_json(path, matrix)
errors = check_checkpoint_docs.validate_matrix(path)
assert any("exact producer versions" in error for error in errors)
assert any("evidence.candidate_commit" in error for error in errors)
assert any("evidence.fixture_digest" in error for error in errors)
assert any("evidence.retained_result" in error for error in errors)
def test_local_link_validator_rejects_missing_file_and_anchor(tmp_path):
docs = tmp_path / "docs/checkpoints"
docs.mkdir(parents=True)
(docs / "guide.md").write_text(
"# Guide\n\n[missing](missing.md) [anchor](target.md#absent)\n",
encoding="utf-8",
)
(docs / "target.md").write_text("# Present\n", encoding="utf-8")
errors = check_checkpoint_docs.validate_local_links(docs, tmp_path)
assert any("missing local link" in error for error in errors)
assert any("missing anchor" in error for error in errors)
def test_documented_cli_examples_stop_before_dispatch(monkeypatch, tmp_path):
docs = tmp_path / "docs/checkpoints"
docs.mkdir(parents=True)
(docs / "guide.md").write_text(
"Run `python3 -m obliteratus --help` to inspect current syntax.\n",
encoding="utf-8",
)
dispatched = False
def forbidden_dispatch(_args):
nonlocal dispatched
dispatched = True
raise AssertionError("model command dispatch must not run")
monkeypatch.setattr("obliteratus.cli._apply_gpu_selection", forbidden_dispatch)
assert check_checkpoint_docs.validate_cli_examples(docs, tmp_path) == []
assert dispatched is False
def test_invalid_documented_cli_example_fails_closed(tmp_path):
docs = tmp_path / "docs/checkpoints"
docs.mkdir(parents=True)
(docs / "guide.md").write_text(
"Run `obliteratus --not-a-real-option` for diagnostics.\n",
encoding="utf-8",
)
errors = check_checkpoint_docs.validate_cli_examples(docs, tmp_path)
assert len(errors) == 1
assert "invalid CLI example" in errors[0]
tests/test_checkpoint_errors.py
+38
View File
@@ -0,0 +1,38 @@
"""Stable runtime failures remain aligned with the accepted error registry."""
from __future__ import annotations
import json
from pathlib import Path
from obliteratus.checkpoint_errors import CheckpointContractError
ROOT = Path(__file__).resolve().parents[1]
def test_runtime_error_contract_covers_every_registered_code_exactly():
registry = json.loads(
(
ROOT / "docs/checkpoints/schemas/checkpoint-error-codes-v1.json"
).read_text()
)
for entry in registry["entries"]:
error = CheckpointContractError(
entry["code"],
detail="bounded_detail",
affected_refs=("b", "a", "a"),
)
assert error.code == entry["code"]
assert error.category == entry["category"]
assert error.phase == entry["phase"]
assert error.affected_refs == ("a", "b")
assert set(error.to_blocker()) == {
"code",
"category",
"phase",
"affected_refs",
"retryable",
"next_action",
}
tests/test_checkpoint_evaluation.py
+215 -3
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import hashlib
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -43,11 +45,221 @@ def test_checkpoint_inventory_verifies_size_hash_and_managed_path(tmp_path):
),
encoding="utf-8",
)
manifest = {
"result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}
}
manifest = {"result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}}
assert _verify_checkpoint_inventory(run_dir, manifest) == checkpoint.resolve()
weights.write_bytes(b"tampered weights")
with pytest.raises(ValueError, match="hash changed"):
_verify_checkpoint_inventory(run_dir, manifest)
def _inventory_candidate(tmp_path):
run_dir = tmp_path / ("run-" + "b" * 32)
checkpoint = run_dir / "checkpoint"
checkpoint.mkdir(parents=True)
weights = checkpoint / "weights.bin"
weights.write_bytes(b"verified weights")
inventory = run_dir / "artifact-inventory.json"
artifact = {
"path": "checkpoint/weights.bin",
"bytes": weights.stat().st_size,
"sha256": hashlib.sha256(weights.read_bytes()).hexdigest(),
}
inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8")
manifest = {"result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}}
return run_dir, checkpoint, inventory, artifact, manifest
def test_checkpoint_inventory_rejects_unmanaged_empty_and_noncheckpoint_records(tmp_path):
run_dir, checkpoint, inventory, artifact, manifest = _inventory_candidate(tmp_path)
manifest["result"]["checkpoint"] = str(tmp_path / "outside")
with pytest.raises(ValueError, match="outside its managed"):
_verify_checkpoint_inventory(run_dir, manifest)
manifest["result"]["checkpoint"] = str(checkpoint)
for value, message in (([], "empty"), ([{"path": "notes.md"}], "no checkpoint")):
inventory.write_text(json.dumps({"artifacts": value}), encoding="utf-8")
with pytest.raises(ValueError, match=message):
_verify_checkpoint_inventory(run_dir, manifest)
artifact["path"] = "checkpoint/../notes.md"
inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8")
with pytest.raises(ValueError, match="invalid checkpoint artifact"):
_verify_checkpoint_inventory(run_dir, manifest)
def test_checkpoint_inventory_rejects_size_change(tmp_path):
run_dir, _checkpoint, inventory, artifact, manifest = _inventory_candidate(tmp_path)
artifact["bytes"] += 1
inventory.write_text(json.dumps({"artifacts": [artifact]}), encoding="utf-8")
with pytest.raises(ValueError, match="size changed"):
_verify_checkpoint_inventory(run_dir, manifest)
class _FakeLifecycle:
def __init__(self):
self.events = []
def loading(self, checkpoint):
self.events.append(("loading", checkpoint))
def resize(self, memory):
self.events.append(("resize", memory))
def ready(self, memory):
self.events.append(("ready", memory))
def release(self, *, reason):
self.events.append(("release", reason))
class _FakeArchive:
def __init__(self, run_dir: Path, manifest):
self.run_dir = run_dir
self.manifest = manifest
self.finishes = []
def begin_evaluation(self, run_id, *, partition, evaluator):
assert run_id == self.manifest["run_id"]
self.reservation = (partition, evaluator)
return {"evaluation_id": "eval-" + "c" * 32}
def result(self, run_id):
assert run_id == self.manifest["run_id"]
return self.manifest
def _run_dir(self, run_id):
assert run_id == self.manifest["run_id"]
return self.run_dir
def finish_evaluation(self, run_id, evaluation_id, **kwargs):
self.finishes.append((run_id, evaluation_id, kwargs))
@pytest.mark.parametrize(
("refusal_rate", "coherence", "expected"),
[(0.2, 0.9, 0), (0.3, 0.9, 2), (0.2, 0.79, 2)],
)
def test_evaluate_reloads_verifies_and_records_objective(
tmp_path,
monkeypatch,
refusal_rate,
coherence,
expected,
):
import obliteratus.checkpoint_evaluation as checkpoint_evaluation
run_dir, checkpoint, _inventory, _artifact, manifest = _inventory_candidate(tmp_path)
run_id = run_dir.name
manifest["run_id"] = run_id
manifest["result"]["metrics"] = {
"baseline_perplexity": 3.5,
"baseline_coherence": 0.95,
}
archive = _FakeArchive(run_dir, manifest)
lifecycle = _FakeLifecycle()
pipelines = []
class FakePipeline:
def __init__(self, **kwargs):
self.kwargs = kwargs
self._quality_metrics = {
"refusal_rate": refusal_rate,
"coherence": coherence,
}
self.cleaned = False
pipelines.append(self)
def _summon(self):
self.kwargs["on_log"]("loaded")
def _verify(self):
self.kwargs["on_log"]("verified")
def cleanup_failed_run(self):
self.cleaned = True
memory = SimpleNamespace(reserved_bytes=1)
monkeypatch.setattr(checkpoint_evaluation, "RunArchive", lambda _root: archive)
monkeypatch.setattr(checkpoint_evaluation, "from_environment", lambda: lifecycle)
monkeypatch.setattr(checkpoint_evaluation, "AbliterationPipeline", FakePipeline)
monkeypatch.setattr(checkpoint_evaluation, "_partition_pairs", lambda _part: (("h", "s"),))
monkeypatch.setattr(checkpoint_evaluation, "measure_torch_memory", lambda _torch: memory)
assert checkpoint_evaluation.evaluate(run_id, "optimizer_tune", str(tmp_path)) == expected
assert archive.reservation == (
"optimizer_tune",
checkpoint_evaluation.EVALUATOR_VERSION,
)
assert pipelines[0].kwargs["model_name"] == str(checkpoint.resolve())
assert pipelines[0]._stock_baseline == {"perplexity": 3.5, "coherence": 0.95}
assert pipelines[0].cleaned is True
metrics = archive.finishes[0][2]["metrics"]
assert metrics["passes_objective"] is (expected == 0)
assert archive.finishes[0][2]["log"] == ["loaded", "verified"]
assert lifecycle.events[-1] == ("release", "evaluation_optimizer_tune_complete")
def test_evaluate_records_failure_and_releases_lifecycle(tmp_path, monkeypatch):
import obliteratus.checkpoint_evaluation as checkpoint_evaluation
run_dir, _checkpoint, _inventory, _artifact, manifest = _inventory_candidate(tmp_path)
run_id = run_dir.name
manifest["run_id"] = run_id
manifest["result"]["metrics"] = {
"baseline_perplexity": 3.5,
"baseline_coherence": 0.95,
}
archive = _FakeArchive(run_dir, manifest)
lifecycle = _FakeLifecycle()
class FailedPipeline:
_quality_metrics = {}
def __init__(self, **_kwargs):
pass
def _summon(self):
raise RuntimeError("summon failed")
def cleanup_failed_run(self):
self.cleaned = True
monkeypatch.setattr(checkpoint_evaluation, "RunArchive", lambda _root: archive)
monkeypatch.setattr(checkpoint_evaluation, "from_environment", lambda: lifecycle)
monkeypatch.setattr(checkpoint_evaluation, "AbliterationPipeline", FailedPipeline)
monkeypatch.setattr(checkpoint_evaluation, "_partition_pairs", lambda _part: (("h", "s"),))
with pytest.raises(RuntimeError, match="summon failed"):
checkpoint_evaluation.evaluate(run_id, "final_test", str(tmp_path))
assert isinstance(archive.finishes[0][2]["failure"], RuntimeError)
assert lifecycle.events[-1] == ("release", "evaluation_final_test_complete")
def test_main_delegates_parsed_evaluation_arguments(monkeypatch):
import obliteratus.checkpoint_evaluation as checkpoint_evaluation
observed = []
monkeypatch.setattr(
"sys.argv",
[
"checkpoint-evaluation",
"--archive-root",
"/archive",
"--run-id",
"run-" + "d" * 32,
"--partition",
"final_test",
],
)
monkeypatch.setattr(
checkpoint_evaluation,
"evaluate",
lambda *args: observed.append(args) or 2,
)
assert checkpoint_evaluation.main() == 2
assert observed == [("run-" + "d" * 32, "final_test", "/archive")]
tests/test_checkpoint_fixture_corpus.py
+207
View File
@@ -0,0 +1,207 @@
"""Deterministic, project-owned distributed-checkpoint fixture corpus."""
from __future__ import annotations
import json
from hashlib import sha256
from pathlib import Path
import shutil
import pytest
import torch
from obliteratus.checkpoint_fixtures import load_fixture_case
from obliteratus.checkpoint_fragments import reconstruct_logical_tensor, validate_fragments
from scripts.generate_checkpoint_fixtures import generate_corpus
ROOT = Path(__file__).resolve().parents[1]
COMMITTED = ROOT / "tests/fixtures/distributed_checkpoints/v1"
def _tree_digest(root: Path) -> str:
digest = sha256()
for path in sorted(item for item in root.rglob("*") if item.is_file()):
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
digest.update(b"\0")
digest.update(path.read_bytes())
digest.update(b"\0")
return digest.hexdigest()
def _manifest(root: Path = COMMITTED) -> dict:
value = json.loads((root / "fixture-corpus.json").read_text(encoding="utf-8"))
assert isinstance(value, dict)
return value
def test_generator_is_byte_deterministic_and_committed_corpus_is_current(tmp_path):
first = tmp_path / "first"
second = tmp_path / "second"
generate_corpus(first)
generate_corpus(second)
assert _tree_digest(first) == _tree_digest(second)
assert _tree_digest(first) == _tree_digest(COMMITTED)
def test_manifest_is_project_owned_self_hashing_and_bounded():
manifest = _manifest()
assert manifest["schema_id"] == "obliteratus.checkpoint-fixture-corpus"
assert manifest["schema_version"] == "1.0.0"
assert manifest["generator"] == {
"path": "scripts/generate_checkpoint_fixtures.py",
"version": "1.0.0",
}
assert manifest["license"] == "AGPL-3.0-or-later"
assert manifest["provenance"] == {
"kind": "deterministic_synthetic",
"seed": 0,
"third_party_data": False,
"third_party_weights": False,
}
assert manifest["limits"] == {
"max_case_bytes": 65536,
"max_cases": 16,
"max_files_per_case": 16,
"max_tensors_per_case": 16,
}
assert len(manifest["cases"]) <= manifest["limits"]["max_cases"]
for case in manifest["cases"]:
case_root = COMMITTED / case["relative_path"]
assert len(case["files"]) <= manifest["limits"]["max_files_per_case"]
assert sum(item["size_bytes"] for item in case["files"]) <= manifest["limits"][
"max_case_bytes"
]
for item in case["files"]:
payload = (case_root / item["relative_path"]).read_bytes()
assert len(payload) == item["size_bytes"]
assert f"sha256:{sha256(payload).hexdigest()}" == item["sha256"]
def test_valid_cases_cover_wave_two_neutral_topologies_and_features():
cases = _manifest()["cases"]
assert {case["world_size"] for case in cases} == {1, 2, 4}
assert {case["case_id"] for case in cases} == {
"mixed-model-peft",
"tp2-pp2-to-single",
"world1-complete",
"world2-uneven-1d",
"world4-dp-replicas",
"world4-uneven-2d",
}
features = {feature for case in cases for feature in case["features"]}
assert {
"buffer",
"dp_replica",
"expert",
"mixed_full_model_peft",
"padding",
"pipeline_parallel",
"scalar",
"tied_weight",
"topology_a_to_b",
"uneven_1d",
"uneven_2d",
} <= features
def test_every_valid_case_reconstructs_the_independent_value_oracle():
for case_record in _manifest()["cases"]:
case = load_fixture_case(COMMITTED / case_record["relative_path"])
result = validate_fragments(case.fragments)
assert result.manifest_digest == case.expected_manifest_digest
assert len(result.logical_tensors) <= _manifest()["limits"]["max_tensors_per_case"]
for logical_tensor_id, oracle in case.tensor_oracles.items():
tensor = reconstruct_logical_tensor(result, logical_tensor_id)
assert tuple(tensor.shape) == oracle.shape
assert str(tensor.dtype).removeprefix("torch.") == oracle.dtype
assert torch.equal(tensor, oracle.values)
raw = tensor.contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
assert f"sha256:{sha256(raw).hexdigest()}" == oracle.sha256
def test_negative_catalog_covers_each_required_corruption_family():
catalog = json.loads((COMMITTED / "negative-cases.json").read_text(encoding="utf-8"))
assert catalog["schema_id"] == "obliteratus.checkpoint-negative-fixtures"
assert {case["failure"] for case in catalog["cases"]} == {
"dimension_mismatch",
"extra_shard",
"fragment_out_of_bounds",
"integer_overflow",
"missing_shard",
"negative_integer",
"padding_shape_mismatch",
"path_traversal",
"payload_dtype_mismatch",
"payload_shape_mismatch",
"replica_digest_mismatch",
"resource_manifest_bomb",
"source_special_file",
"source_symlink",
"truncated_shard",
"coverage_gap",
"coverage_overlap",
}
assert all(case["expected_code"].startswith("DCI_") for case in catalog["cases"])
def _mutable_case(tmp_path: Path) -> Path:
destination = tmp_path / "case"
shutil.copytree(COMMITTED / "cases/world1-complete", destination)
return destination
@pytest.mark.parametrize(
("mutate", "message"),
[
(lambda record: record.update(schema_id="unknown"), "unsupported fixture case schema"),
(lambda record: record.update(fragments={}), "fragment list is invalid"),
(lambda record: record.update(fragments=[None]), "fragment must be an object"),
(
lambda record: record["fragments"][0].update(payload_file="../escape"),
"unsafe fixture payload file",
),
(
lambda record: record["fragments"][0].update(payload_key="missing"),
"fixture payload key is missing",
),
],
)
def test_fixture_loader_rejects_invalid_case_contracts(tmp_path, mutate, message):
case = _mutable_case(tmp_path)
record = json.loads((case / "case.json").read_text(encoding="utf-8"))
mutate(record)
(case / "case.json").write_text(json.dumps(record), encoding="utf-8")
with pytest.raises(ValueError, match=message):
load_fixture_case(case)
def test_fixture_loader_rejects_non_object_oversized_and_nonregular_json(tmp_path):
case = _mutable_case(tmp_path)
(case / "case.json").write_text("[]", encoding="utf-8")
with pytest.raises(ValueError, match="not an object"):
load_fixture_case(case)
(case / "case.json").write_bytes(b" " * (256 * 1024 + 1))
with pytest.raises(ValueError, match="exceeds"):
load_fixture_case(case)
(case / "case.json").unlink()
(case / "case.json").mkdir()
with pytest.raises(ValueError, match="not a regular file"):
load_fixture_case(case)
def test_fixture_loader_rejects_symlink_root(tmp_path):
link = tmp_path / "linked"
link.symlink_to(COMMITTED / "cases/world1-complete", target_is_directory=True)
with pytest.raises(ValueError, match="non-symlink directory"):
load_fixture_case(link)
tests/test_checkpoint_fragments.py
+536
View File
@@ -0,0 +1,536 @@
"""Producer-neutral tensor-fragment validation and reconstruction oracles."""
from __future__ import annotations
from dataclasses import FrozenInstanceError, replace
from hashlib import sha256
import pytest
import torch
from hypothesis import given, strategies as st
from obliteratus.checkpoint_errors import CheckpointContractError
from obliteratus.checkpoint_fragments import (
FragmentLimits,
Padding,
Replica,
TensorFragment,
reconstruct_logical_tensor,
validate_fragments,
)
def _digest(tensor: torch.Tensor) -> str:
payload = (
tensor.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
)
return f"sha256:{sha256(payload).hexdigest()}"
def _fragment(
fragment_id: str,
payload: torch.Tensor,
*,
global_shape: tuple[int, ...],
offset: tuple[int, ...],
extent: tuple[int, ...] | None = None,
logical_tensor_id: str = "tensor.weight",
padding: Padding | None = None,
replica: Replica | None = None,
tie_group_id: str | None = None,
partition_axes: tuple[int, ...] = (0,),
) -> TensorFragment:
extent = extent if extent is not None else tuple(payload.shape)
padding = padding or Padding.zeros(len(global_shape))
replica = replica or Replica.unique()
return TensorFragment(
fragment_id=fragment_id,
component_id="model",
fqn=logical_tensor_id,
role="parameter",
dtype=str(payload.dtype).removeprefix("torch."),
global_shape=global_shape,
local_shape=tuple(payload.shape),
element_offset=offset,
element_extent=extent,
padding=padding,
shard_file_id=f"shard-{fragment_id}",
shard_digest_ref=f"digest-{fragment_id}",
fragment_digest=_digest(
payload[
tuple(
slice(before, before + size)
for before, size in zip(padding.before, extent, strict=True)
)
]
if global_shape
else payload
),
replica=replica,
partition_axes=partition_axes if global_shape else (),
logical_tensor_id=logical_tensor_id,
tie_group_id=tie_group_id,
shared_storage_id=None,
topology_coordinates=(("tp", 0), ("pp", 1)),
evidence_refs=("evidence-1",),
payload=payload,
)
def _assert_refused(fragments: list[TensorFragment], detail: str) -> None:
with pytest.raises(CheckpointContractError) as caught:
validate_fragments(fragments)
assert caught.value.code == "DCI_VALIDATION_FAILED"
assert caught.value.detail == detail
assert any(reference.startswith("tensor") for reference in caught.value.affected_refs)
def test_records_are_frozen_and_scalar_round_trips():
fragment = _fragment(
"scalar",
torch.tensor(7.0),
global_shape=(),
offset=(),
partition_axes=(),
)
with pytest.raises(FrozenInstanceError):
fragment.dtype = "float16" # type: ignore[misc]
result = validate_fragments([fragment])
actual = reconstruct_logical_tensor(result, "tensor.weight")
assert actual.shape == torch.Size([])
assert actual.item() == 7.0
assert result.logical_tensors[0].topology_coordinates == (("pp", 1), ("tp", 0))
def test_uneven_fragments_reconstruct_independently_of_record_order():
expected = torch.arange(15, dtype=torch.float32).reshape(3, 5)
fragments = [
_fragment(
"right",
expected[:, 2:].clone(),
global_shape=(3, 5),
offset=(0, 2),
partition_axes=(1,),
),
_fragment(
"left",
expected[:, :2].clone(),
global_shape=(3, 5),
offset=(0, 0),
partition_axes=(1,),
),
]
forward = validate_fragments(fragments)
reverse = validate_fragments(list(reversed(fragments)))
assert torch.equal(reconstruct_logical_tensor(forward, "tensor.weight"), expected)
assert torch.equal(reconstruct_logical_tensor(reverse, "tensor.weight"), expected)
assert forward.manifest_digest == reverse.manifest_digest
assert [item.fragment_id for item in forward.logical_tensors[0].fragments] == [
"left",
"right",
]
def test_two_dimensional_tiles_cover_the_logical_tensor_exactly():
expected = torch.arange(24, dtype=torch.int64).reshape(4, 6)
fragments = []
for row, (start, stop) in enumerate(((0, 1), (1, 4))):
for column, (left, right) in enumerate(((0, 2), (2, 6))):
fragments.append(
_fragment(
f"tile-{row}-{column}",
expected[start:stop, left:right].clone(),
global_shape=(4, 6),
offset=(start, left),
partition_axes=(0, 1),
)
)
result = validate_fragments(fragments)
assert torch.equal(reconstruct_logical_tensor(result, "tensor.weight"), expected)
def test_declared_padding_is_removed_before_reconstruction():
payload = torch.tensor([-1, 10, 11, -2], dtype=torch.int32)
fragment = _fragment(
"padded",
payload,
global_shape=(2,),
offset=(0,),
extent=(2,),
padding=Padding(before=(1,), after=(1,), semantic="producer_declared"),
)
result = validate_fragments([fragment])
assert torch.equal(
reconstruct_logical_tensor(result, "tensor.weight"),
torch.tensor([10, 11], dtype=torch.int32),
)
def test_explicit_replicas_are_deduplicated_only_after_digest_agreement():
payload = torch.tensor([1.0, 2.0])
fragments = [
_fragment(
"replica-0",
payload.clone(),
global_shape=(2,),
offset=(0,),
replica=Replica("dp-0", 0, 2),
),
_fragment(
"replica-1",
payload.clone(),
global_shape=(2,),
offset=(0,),
replica=Replica("dp-0", 1, 2),
),
]
result = validate_fragments(fragments)
assert len(result.logical_tensors[0].fragments) == 1
assert result.logical_tensors[0].replica_members == (("replica-0", "replica-1"),)
@pytest.mark.parametrize(
("mutator", "detail"),
[
(lambda item: replace(item, element_offset=(-1,)), "negative_integer"),
(
lambda item: replace(item, element_offset=((1 << 63) - 1,)),
"integer_overflow",
),
(lambda item: replace(item, element_offset=(1,)), "fragment_out_of_bounds"),
(lambda item: replace(item, local_shape=(3,)), "padding_shape_mismatch"),
(lambda item: replace(item, partition_axes=(1,)), "partition_axis_out_of_bounds"),
],
)
def test_invalid_fragment_geometry_fails_closed(mutator, detail):
valid = _fragment(
"fragment",
torch.tensor([1.0, 2.0]),
global_shape=(2,),
offset=(0,),
)
_assert_refused([mutator(valid)], detail)
def test_gap_and_overlap_are_distinct_refusals():
left = _fragment("left", torch.tensor([1.0]), global_shape=(3,), offset=(0,))
right = _fragment("right", torch.tensor([3.0]), global_shape=(3,), offset=(2,))
_assert_refused([left, right], "coverage_gap")
overlap = _fragment("overlap", torch.tensor([2.0, 3.0]), global_shape=(3,), offset=(1,))
_assert_refused([replace(left, payload=torch.tensor([1.0, 2.0]), local_shape=(2,), element_extent=(2,), fragment_digest=None), overlap], "coverage_overlap")
def test_replica_membership_and_content_disagreement_fail_closed():
payload = torch.tensor([1.0, 2.0])
first = _fragment(
"replica-0",
payload,
global_shape=(2,),
offset=(0,),
replica=Replica("dp-0", 0, 2),
)
_assert_refused([first], "replica_members_missing")
disagreeing = _fragment(
"replica-1",
torch.tensor([1.0, 3.0]),
global_shape=(2,),
offset=(0,),
replica=Replica("dp-0", 1, 2),
)
_assert_refused([first, disagreeing], "replica_digest_mismatch")
def test_tied_tensors_require_matching_shape_dtype_and_values():
first = _fragment(
"embedding",
torch.tensor([1.0, 2.0]),
global_shape=(2,),
offset=(0,),
logical_tensor_id="model.embed.weight",
tie_group_id="tie-0",
)
second = _fragment(
"lm-head",
torch.tensor([1.0, 3.0]),
global_shape=(2,),
offset=(0,),
logical_tensor_id="lm_head.weight",
tie_group_id="tie-0",
)
with pytest.raises(CheckpointContractError) as caught:
validate_fragments([first, second])
assert caught.value.detail == "tie_group_content_mismatch"
assert caught.value.affected_refs == ("lm_head.weight", "model.embed.weight")
def test_fragment_and_overlap_limits_refuse_before_expensive_work():
first = _fragment("first", torch.tensor([1.0]), global_shape=(2,), offset=(0,))
second = _fragment("second", torch.tensor([2.0]), global_shape=(2,), offset=(1,))
with pytest.raises(CheckpointContractError) as count_error:
validate_fragments([first, second], limits=FragmentLimits(max_fragments=1))
assert count_error.value.code == "DCI_RESOURCE_LIMIT"
assert count_error.value.detail == "max_fragments"
with pytest.raises(CheckpointContractError) as work_error:
validate_fragments([first, second], limits=FragmentLimits(max_overlap_checks=0))
assert work_error.value.code == "DCI_RESOURCE_LIMIT"
assert work_error.value.detail == "max_overlap_checks"
@given(
size=st.integers(min_value=2, max_value=64),
split=st.integers(min_value=1, max_value=63),
)
def test_one_dimensional_partition_property(size: int, split: int):
split = min(split, size - 1)
expected = torch.arange(size, dtype=torch.int64)
fragments = [
_fragment("a", expected[:split].clone(), global_shape=(size,), offset=(0,)),
_fragment("b", expected[split:].clone(), global_shape=(size,), offset=(split,)),
]
result = validate_fragments(fragments)
assert torch.equal(reconstruct_logical_tensor(result, "tensor.weight"), expected)
def test_limits_and_top_level_fragment_contracts_fail_closed():
with pytest.raises(ValueError, match="non-negative integer"):
FragmentLimits(max_fragments=-1)
with pytest.raises(CheckpointContractError) as empty:
validate_fragments([])
assert empty.value.detail == "fragment_set_empty"
with pytest.raises(CheckpointContractError) as wrong_type:
validate_fragments([object()]) # type: ignore[list-item]
assert wrong_type.value.detail == "fragment_type_invalid"
valid = _fragment("same", torch.ones(1), global_shape=(1,), offset=(0,))
with pytest.raises(CheckpointContractError) as duplicate:
validate_fragments([valid, valid])
assert duplicate.value.detail == "fragment_id_duplicate"
result = validate_fragments([valid])
with pytest.raises(KeyError, match="absent"):
result.get("absent")
@pytest.mark.parametrize(
("mutator", "detail", "code"),
[
(lambda item: replace(item, global_shape=[1]), "shape_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type]
(
lambda item: replace(item, global_shape=(1, 1), local_shape=(1,)),
"dimension_mismatch",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, global_shape=(1, 1)),
"max_dimensions",
"DCI_RESOURCE_LIMIT",
),
(lambda item: replace(item, element_extent=(True,)), "integer_type_invalid", "DCI_VALIDATION_FAILED"),
(
lambda item: replace(item, global_shape=((1 << 63),)),
"integer_overflow",
"DCI_VALIDATION_FAILED",
),
(lambda item: replace(item, component_id=""), "identifier_invalid", "DCI_VALIDATION_FAILED"),
(lambda item: replace(item, fqn="x" * 4097), "identifier_invalid", "DCI_VALIDATION_FAILED"),
(lambda item: replace(item, role="unknown"), "role_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type]
(lambda item: replace(item, dtype="string"), "dtype_unsupported", "DCI_VALIDATION_FAILED"),
(lambda item: replace(item, fragment_digest="bad"), "fragment_digest_invalid", "DCI_VALIDATION_FAILED"),
(lambda item: replace(item, padding=object()), "padding_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type]
(lambda item: replace(item, replica=object()), "replica_type_invalid", "DCI_VALIDATION_FAILED"), # type: ignore[arg-type]
(
lambda item: replace(item, padding=Padding((0,), (0,), "invalid")), # type: ignore[arg-type]
"padding_semantic_invalid",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, padding=Padding((1,), (0,), "none"), local_shape=(2,)),
"undeclared_padding",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, partition_axes=(0, 0)),
"partition_axis_duplicate",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, replica=Replica(None, 1, 1)),
"replica_declaration_invalid",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, replica=Replica("group", 0, 1)),
"replica_declaration_invalid",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, replica=Replica("group", 2, 2)),
"replica_member_out_of_bounds",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, topology_coordinates=(("tp", -1),)),
"topology_coordinate_invalid",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, topology_coordinates=(("tp", 0), ("tp", 1))),
"topology_coordinate_duplicate",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, topology_coordinates=(("tp",),)),
"topology_coordinate_invalid",
"DCI_VALIDATION_FAILED",
),
(
lambda item: replace(item, evidence_refs=(object(),)),
"evidence_ref_invalid",
"DCI_VALIDATION_FAILED",
),
],
)
def test_fragment_metadata_validation_covers_each_fail_closed_family(mutator, detail, code):
valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,))
limits = FragmentLimits(max_dimensions=1) if detail == "max_dimensions" else FragmentLimits()
with pytest.raises(CheckpointContractError) as caught:
validate_fragments([mutator(valid)], limits=limits)
assert caught.value.code == code
assert caught.value.detail == detail
@pytest.mark.parametrize(
("payload", "local_shape", "dtype", "digest", "detail"),
[
(object(), (1,), "float32", None, "payload_type_invalid"),
(torch.ones(2), (1,), "float32", None, "payload_shape_mismatch"),
(torch.ones(1), (1,), "float64", None, "payload_dtype_mismatch"),
(torch.ones(1), (1,), "float32", "sha256:" + "0" * 64, "fragment_digest_mismatch"),
],
)
def test_payload_contract_refuses_wrong_type_shape_dtype_and_digest(
payload,
local_shape,
dtype,
digest,
detail,
):
valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,))
candidate = replace(
valid,
payload=payload,
local_shape=local_shape,
dtype=dtype,
fragment_digest=digest,
)
_assert_refused([candidate], detail)
def test_payload_contract_rejects_non_strided_tensor_layout():
valid = _fragment("fragment", torch.ones(1), global_shape=(1,), offset=(0,))
sparse = torch.sparse_coo_tensor(
torch.tensor([[0]]),
torch.tensor([1.0]),
size=(1,),
check_invariants=True,
)
_assert_refused(
[replace(valid, payload=sparse, fragment_digest=None)],
"payload_layout_unsupported",
)
def test_replica_metadata_count_and_digest_availability_are_mandatory():
payload = torch.ones(1)
first = _fragment(
"a",
payload,
global_shape=(1,),
offset=(0,),
replica=Replica("group", 0, 2),
)
second = _fragment(
"b",
payload,
global_shape=(1,),
offset=(0,),
replica=Replica("group", 1, 3),
)
_assert_refused([first, second], "replica_count_mismatch")
second = replace(second, replica=Replica("group", 1, 2), component_id="other")
_assert_refused([first, second], "replica_metadata_mismatch")
first = replace(first, payload=None, fragment_digest=None)
second = replace(second, payload=None, fragment_digest=None, component_id="model")
_assert_refused([first, second], "replica_digest_unavailable")
def test_logical_metadata_resource_zero_and_payload_absence_boundaries():
first = _fragment("a", torch.ones(1), global_shape=(1,), offset=(0,))
left = _fragment("left", torch.ones(1), global_shape=(2,), offset=(0,))
right = _fragment("right", torch.ones(1), global_shape=(2,), offset=(1,))
_assert_refused([left, replace(right, fqn="other")], "logical_tensor_metadata_mismatch")
with pytest.raises(CheckpointContractError) as limit:
validate_fragments([first], limits=FragmentLimits(max_elements_per_tensor=0))
assert limit.value.code == "DCI_RESOURCE_LIMIT"
assert limit.value.detail == "max_elements_per_tensor"
zero_a = _fragment("zero-a", torch.empty(0), global_shape=(0,), offset=(0,))
zero_b = _fragment("zero-b", torch.empty(0), global_shape=(0,), offset=(0,))
_assert_refused([zero_a, zero_b], "zero_tensor_representation_ambiguous")
absent = replace(first, payload=None, fragment_digest=_digest(torch.ones(1)))
result = validate_fragments([absent])
with pytest.raises(CheckpointContractError) as unavailable:
reconstruct_logical_tensor(result, "tensor.weight")
assert unavailable.value.detail == "payload_unavailable"
def test_tie_groups_require_two_members_and_identical_metadata():
single = _fragment(
"single",
torch.ones(1),
global_shape=(1,),
offset=(0,),
tie_group_id="tie",
)
_assert_refused([single], "tie_group_member_missing")
second = _fragment(
"second",
torch.ones(2),
global_shape=(2,),
offset=(0,),
logical_tensor_id="tensor.other",
tie_group_id="tie",
)
with pytest.raises(CheckpointContractError) as mismatch:
validate_fragments([single, second])
assert mismatch.value.detail == "tie_group_metadata_mismatch"
tests/test_checkpoint_inspection.py
+753
View File
@@ -0,0 +1,753 @@
"""Offline, structure-only checkpoint inspection tests."""
from __future__ import annotations
import json
import os
import socket
import subprocess
from pathlib import Path
import pytest
import torch
from jsonschema import Draft202012Validator, FormatChecker
from safetensors.torch import save_file
import obliteratus.checkpoint_inspection as inspection_module
from obliteratus.checkpoint_capabilities import (
AdapterCapability,
AdapterRegistry,
ExactDependency,
)
from obliteratus.checkpoint_errors import CheckpointContractError
from obliteratus.checkpoint_inspection import InspectionLimits, inspect_checkpoint
ROOT = Path(__file__).resolve().parents[1]
DESCRIPTOR_SCHEMA = json.loads(
(ROOT / "docs/checkpoints/schemas/checkpoint-descriptor-v1.schema.json").read_text()
)
def _save(path: Path, **tensors: torch.Tensor) -> None:
save_file(dict(sorted(tensors.items())), path)
def _assert_contract(report) -> dict:
descriptor = report.to_dict()
Draft202012Validator(
DESCRIPTOR_SCHEMA,
format_checker=FormatChecker(),
).validate(descriptor)
assert json.loads(report.to_json()) == descriptor
return descriptor
def test_direct_hf_safetensors_is_inventory_backed_and_canonical_ready(tmp_path):
_save(tmp_path / "model.safetensors", weight=torch.arange(6).reshape(2, 3))
(tmp_path / "config.json").write_text('{"model_type":"tiny"}\n', encoding="utf-8")
first = inspect_checkpoint(tmp_path)
second = inspect_checkpoint(tmp_path)
descriptor = _assert_contract(first)
assert first.primary_format == "hf_safetensors"
assert first.support_decision == "canonical_hf_ready"
assert first.descriptor_id == second.descriptor_id
assert descriptor["classification_confidence"] == "verified"
assert descriptor["safety"] == {
"inspection_level": "safe_structure",
"trust_required": False,
"inventory_revalidated": True,
"unsafe_serialization_findings": [],
"violations": [],
}
assert descriptor["state"] == {
"observed_scopes": ["model_weights"],
"classification": "weights_only",
}
assert descriptor["resource_estimate"]["tensor_count"] == 1
assert descriptor["resource_estimate"]["logical_bytes"] == 48
assert not descriptor["blockers"]
def test_indexed_safetensors_validates_safe_weight_map_and_shards(tmp_path):
_save(tmp_path / "model-00001-of-00002.safetensors", a=torch.tensor([1.0]))
_save(tmp_path / "model-00002-of-00002.safetensors", b=torch.tensor([2.0, 3.0]))
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{
"metadata": {"total_size": 12},
"weight_map": {
"a": "model-00001-of-00002.safetensors",
"b": "model-00002-of-00002.safetensors",
},
}
),
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "hf_safetensors"
assert descriptor["resource_estimate"]["tensor_count"] == 2
assert descriptor["resource_estimate"]["shard_count"] == 2
assert descriptor["support_decision"] == "canonical_hf_ready"
def test_direct_and_indexed_hf_signatures_are_an_ambiguous_collision(tmp_path):
_save(tmp_path / "model.safetensors", direct=torch.ones(1))
_save(tmp_path / "model-00001-of-00001.safetensors", indexed=torch.ones(1))
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"indexed": "model-00001-of-00001.safetensors",
}
}
),
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "ambiguous"
assert descriptor["support_decision"] == "blocked"
assert "hf_layout_collision" in descriptor["safety"]["violations"]
@pytest.mark.parametrize("mode", ["missing", "extra"])
def test_indexed_safetensors_missing_or_extra_shards_are_blocked(tmp_path, mode):
_save(tmp_path / "model-00001-of-00001.safetensors", a=torch.ones(1))
referenced = (
"missing.safetensors"
if mode == "missing"
else "model-00001-of-00001.safetensors"
)
if mode == "extra":
_save(tmp_path / "model-00002-of-00002.safetensors", extra=torch.ones(1))
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a": referenced}}),
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["support_decision"] == "blocked"
assert "hf_weight_map_shard_mismatch" in descriptor["safety"]["violations"]
assert descriptor["blockers"][0]["code"] == "DCI_VALIDATION_FAILED"
@pytest.mark.parametrize(
("header", "payload", "violation"),
[
({"weight": {"dtype": "F32", "shape": [2], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_range_invalid"),
({"weight": {"dtype": "UNKNOWN", "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"),
({"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 8]}}, b"\0" * 4, "safetensors_range_invalid"),
({"weight": {"dtype": "F32", "shape": [1], "data_offsets": [4, 8]}}, b"\0" * 8, "safetensors_range_gap"),
],
)
def test_safetensors_dtype_shape_and_range_corruption_is_blocked(
tmp_path,
header,
payload,
violation,
):
encoded = json.dumps(header, separators=(",", ":")).encode()
(tmp_path / "model.safetensors").write_bytes(
len(encoded).to_bytes(8, "little") + encoded + payload
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["support_decision"] == "blocked"
assert violation in descriptor["safety"]["violations"]
@pytest.mark.parametrize(
("files", "expected_format"),
[
({"pytorch_model.bin": b"pickle"}, "hf_pytorch_pickle"),
({".metadata": b"opaque-dcp-metadata", "__0_0.distcp": b"payload"}, "pytorch_dcp"),
(
{
".metadata": b"opaque-dcp-metadata",
"fsdp_metadata.json": b'{"state_dict_type":"SHARDED_STATE_DICT"}',
},
"fsdp_state_dict",
),
(
{
"metadata.json": b'{"sharded_backend":"torch_dist","version":"1.0"}',
"common.pt": b"pickle",
},
"megatron_torch_dist",
),
({"zero_pp_rank_0_mp_rank_00_optim_states.pt": b"pickle"}, "deepspeed_zero"),
(
{
"universal_checkpoint_info.json": b'{"type":"universal"}',
"zero_pp_rank_0_mp_rank_00_model_states.pt": b"pickle",
},
"deepspeed_universal",
),
],
)
def test_vendor_and_pickle_layouts_are_classified_without_payload_access(
tmp_path,
files,
expected_format,
):
for name, payload in files.items():
(tmp_path / name).write_bytes(payload)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == expected_format
assert descriptor["support_decision"] == "trusted_inspection_required"
assert descriptor["safety"]["trust_required"] is True
assert descriptor["adapter_resolution"]["status"] == "missing"
assert {item["code"] for item in descriptor["blockers"]} == {
"DCI_TRUST_POLICY_REQUIRED"
}
def test_exact_registered_capability_reports_missing_extra_and_version(tmp_path):
(tmp_path / ".metadata").write_bytes(b"opaque-dcp-metadata")
(tmp_path / "__0_0.distcp").write_bytes(b"payload-never-read-as-a-tensor")
registry = AdapterRegistry(
(
AdapterCapability(
adapter_id="example-dcp",
adapter_version="0.1.0",
producer="example-producer",
producer_version="1.2.3",
formats=("pytorch_dcp",),
required_extras=("checkpoint-example",),
required_dependencies=(
ExactDependency("obliteratus-test-package-that-does-not-exist", "1.2.3"),
),
),
)
)
descriptor = _assert_contract(
inspect_checkpoint(tmp_path, adapter_registry=registry)
)
assert descriptor["primary_format"] == "pytorch_dcp"
assert descriptor["support_decision"] == "trusted_inspection_required"
assert descriptor["adapter_resolution"]["status"] == "missing"
assert descriptor["adapter_resolution"]["adapter_id"] == "example-dcp"
assert "install_extra=obliteratus[checkpoint-example]==0.1.3" in descriptor[
"adapter_resolution"
]["reason"]
assert (
"required_versions=obliteratus-test-package-that-does-not-exist==1.2.3"
in descriptor["adapter_resolution"]["reason"]
)
assert {item["code"] for item in descriptor["blockers"]} == {
"DCI_TRUST_POLICY_REQUIRED",
"DCI_TRUST_RUNTIME_UNAVAILABLE",
}
def test_ambiguous_registered_capabilities_fail_closed(tmp_path):
(tmp_path / ".metadata").write_bytes(b"opaque-dcp-metadata")
dependency = ExactDependency("example-producer", "1.2.3")
registry = AdapterRegistry(
tuple(
AdapterCapability(
adapter_id=f"example-{index}",
adapter_version="0.1.0",
producer="example-producer",
producer_version="1.2.3",
formats=("pytorch_dcp",),
required_extras=("checkpoint-example",),
required_dependencies=(dependency,),
)
for index in range(2)
)
)
descriptor = _assert_contract(
inspect_checkpoint(tmp_path, adapter_registry=registry)
)
assert descriptor["adapter_resolution"]["status"] == "ambiguous"
assert descriptor["support_decision"] == "blocked"
assert [item["code"] for item in descriptor["blockers"]].count(
"DCI_UNSUPPORTED_FORMAT_OR_VERSION"
) == 1
def test_peft_layout_is_safe_safetensors_but_keeps_adapter_identity(tmp_path):
_save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1, 2))
(tmp_path / "adapter_config.json").write_text(
'{"base_model_name_or_path":"local/base","peft_type":"LORA"}\n',
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "peft_safetensors"
assert descriptor["components"][0]["kind"] == "peft_adapter"
assert descriptor["components"][0]["format"] == "peft_safetensors"
assert descriptor["state"]["observed_scopes"] == ["adapter_weights"]
assert descriptor["support_decision"] == "canonical_hf_ready"
def test_mixed_model_and_adapter_components_are_preserved_and_blocked(tmp_path):
_save(tmp_path / "model.safetensors", weight=torch.ones(2, 2))
_save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1, 2))
(tmp_path / "adapter_config.json").write_text('{"peft_type":"LORA"}\n')
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "ambiguous"
assert [(item["kind"], item["format"]) for item in descriptor["components"]] == [
("model", "hf_safetensors"),
("peft_adapter", "peft_safetensors"),
]
assert descriptor["state"]["observed_scopes"] == ["adapter_weights", "model_weights"]
assert descriptor["support_decision"] == "blocked"
assert descriptor["blockers"][0]["code"] == "DCI_UNSUPPORTED_FORMAT_OR_VERSION"
def test_unknown_layout_is_a_stable_blocked_descriptor(tmp_path):
(tmp_path / "notes.txt").write_text("not a checkpoint\n")
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "unknown"
assert descriptor["classification_confidence"] == "unknown"
assert descriptor["components"][0]["kind"] == "unknown"
assert descriptor["support_decision"] == "blocked"
def test_legacy_pickle_payload_is_never_executed(tmp_path):
marker = tmp_path / "payload-executed"
command = f"touch {marker}".encode("utf-8")
malicious = b"cos\nsystem\n(S'" + command + b"'\ntR."
(tmp_path / "pytorch_model.bin").write_bytes(malicious)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "hf_pytorch_pickle"
assert not marker.exists()
def test_remote_code_declaration_is_inert_and_not_imported(tmp_path, monkeypatch):
_save(tmp_path / "model.safetensors", weight=torch.ones(1))
(tmp_path / "config.json").write_text(
json.dumps({"auto_map": {"AutoModel": "must_not_import.Model"}}),
encoding="utf-8",
)
imported: list[str] = []
original_import = __import__
def guarded_import(name, *args, **kwargs):
if name.startswith("must_not_import"):
imported.append(name)
raise AssertionError("remote code import attempted")
return original_import(name, *args, **kwargs)
monkeypatch.setattr("builtins.__import__", guarded_import)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["primary_format"] == "hf_safetensors"
assert imported == []
def test_default_inspection_invokes_no_reader_network_process_group_or_plugin(
tmp_path,
monkeypatch,
):
_save(tmp_path / "model.safetensors", weight=torch.ones(1))
before = {
path.name: path.read_bytes()
for path in tmp_path.iterdir()
if path.is_file()
}
monkeypatch.setattr(torch, "load", lambda *_a, **_k: pytest.fail("torch.load called"))
monkeypatch.setattr(
torch.distributed,
"init_process_group",
lambda *_a, **_k: pytest.fail("process group initialized"),
)
monkeypatch.setattr(
socket,
"create_connection",
lambda *_a, **_k: pytest.fail("network opened"),
)
monkeypatch.setattr(
subprocess,
"run",
lambda *_a, **_k: pytest.fail("subprocess started"),
)
monkeypatch.setattr(
"importlib.metadata.entry_points",
lambda *_a, **_k: pytest.fail("plugin discovery attempted"),
)
monkeypatch.setattr(
"importlib.metadata.version",
lambda *_a, **_k: pytest.fail("package metadata queried without a capability"),
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert descriptor["support_decision"] == "canonical_hf_ready"
assert {
path.name: path.read_bytes()
for path in tmp_path.iterdir()
if path.is_file()
} == before
def test_duplicate_recognized_basenames_fail_closed(tmp_path):
for directory in (tmp_path / "a", tmp_path / "b"):
directory.mkdir()
_save(directory / "model.safetensors", weight=torch.ones(1))
with pytest.raises(CheckpointContractError) as caught:
inspect_checkpoint(tmp_path)
assert caught.value.code == "DCI_VALIDATION_FAILED"
assert caught.value.detail == "duplicate_basename"
def test_symlink_and_special_file_sources_fail_before_classification(tmp_path):
target = tmp_path / "target.safetensors"
target.write_bytes(b"not relevant")
(tmp_path / "model.safetensors").symlink_to(target.name)
with pytest.raises(CheckpointContractError) as symlink_error:
inspect_checkpoint(tmp_path)
assert symlink_error.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION"
assert symlink_error.value.detail == "source_symlink"
(tmp_path / "model.safetensors").unlink()
if hasattr(os, "mkfifo"):
os.mkfifo(tmp_path / "special")
with pytest.raises(CheckpointContractError) as special_error:
inspect_checkpoint(tmp_path)
assert special_error.value.detail == "source_special_file"
def test_resource_limits_apply_before_header_or_json_allocation(tmp_path):
(tmp_path / "a").write_bytes(b"a")
(tmp_path / "b").write_bytes(b"b")
with pytest.raises(CheckpointContractError) as file_error:
inspect_checkpoint(tmp_path, limits=InspectionLimits(max_files=1))
assert file_error.value.code == "DCI_RESOURCE_LIMIT"
assert file_error.value.detail == "max_files"
for path in tmp_path.iterdir():
path.unlink()
(tmp_path / "model.safetensors").write_bytes((1024).to_bytes(8, "little") + b"{}")
with pytest.raises(CheckpointContractError) as header_error:
inspect_checkpoint(
tmp_path,
limits=InspectionLimits(max_safetensors_header_bytes=64),
)
assert header_error.value.detail == "max_safetensors_header_bytes"
def test_inventory_race_is_detected_and_never_returned_as_success(tmp_path, monkeypatch):
weights = tmp_path / "model.safetensors"
_save(weights, weight=torch.ones(1))
original = inspection_module._hash_regular_file
changed = False
def race(path, expected, limits):
nonlocal changed
result = original(path, expected, limits)
if not changed:
changed = True
path.write_bytes(path.read_bytes() + b"changed")
return result
monkeypatch.setattr(inspection_module, "_hash_regular_file", race)
with pytest.raises(CheckpointContractError) as caught:
inspect_checkpoint(tmp_path)
assert caught.value.code == "DCI_SOURCE_CHANGED"
assert caught.value.detail == "source_changed"
def _raw_safetensors(path: Path, header: object, payload: bytes = b"") -> None:
encoded = json.dumps(header, separators=(",", ":")).encode("utf-8")
path.write_bytes(len(encoded).to_bytes(8, "little") + encoded + payload)
def test_inspection_limits_and_source_root_types_fail_closed(tmp_path):
with pytest.raises(ValueError, match="positive integer"):
InspectionLimits(max_files=0)
with pytest.raises(CheckpointContractError) as missing:
inspect_checkpoint(tmp_path / "missing")
assert missing.value.detail == "source_missing"
target = tmp_path / "target"
target.mkdir()
root_link = tmp_path / "root-link"
root_link.symlink_to(target, target_is_directory=True)
with pytest.raises(CheckpointContractError) as symlink:
inspect_checkpoint(root_link)
assert symlink.value.detail == "source_symlink"
if hasattr(os, "mkfifo"):
fifo = tmp_path / "root-fifo"
os.mkfifo(fifo)
with pytest.raises(CheckpointContractError) as special:
inspect_checkpoint(fifo)
assert special.value.detail == "source_special_file"
def test_ancestor_symlink_and_open_race_fail_with_stable_source_errors(
tmp_path,
monkeypatch,
):
actual = tmp_path / "actual"
source = actual / "checkpoint"
source.mkdir(parents=True)
_save(source / "model.safetensors", weight=torch.ones(1))
alias = tmp_path / "alias"
alias.symlink_to(actual, target_is_directory=True)
with pytest.raises(CheckpointContractError) as symlink:
inspect_checkpoint(alias / "checkpoint")
assert symlink.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION"
assert symlink.value.detail == "source_symlink"
weights = source / "model.safetensors"
real_open = inspection_module.os.open
def fail_observed_open(path, flags):
if Path(path) == weights:
raise OSError("injected source replacement")
return real_open(path, flags)
monkeypatch.setattr(inspection_module.os, "open", fail_observed_open)
with pytest.raises(CheckpointContractError) as changed:
inspect_checkpoint(source)
assert changed.value.code == "DCI_SOURCE_CHANGED"
assert changed.value.detail == "source_changed"
def test_single_file_and_directory_byte_limits_are_enforced(tmp_path):
weights = tmp_path / "single.safetensors"
_save(weights, weight=torch.ones(1))
descriptor = _assert_contract(inspect_checkpoint(weights))
assert descriptor["source_inventory"]["files"][0]["relative_path"] == weights.name
nested = tmp_path / "nested"
nested.mkdir()
(nested / "file").write_text("x", encoding="utf-8")
with pytest.raises(CheckpointContractError) as directories:
inspect_checkpoint(tmp_path, limits=InspectionLimits(max_directories=1))
assert directories.value.detail == "max_directories"
with pytest.raises(CheckpointContractError) as total:
inspect_checkpoint(weights, limits=InspectionLimits(max_total_bytes=1))
assert total.value.detail == "max_total_bytes"
@pytest.mark.parametrize(
("header", "payload", "violation"),
[
([], b"", "safetensors_header_invalid"),
({"": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"),
({"x": []}, b"", "safetensors_header_invalid"),
({"x": {"dtype": "F32", "shape": [1], "data_offsets": [0]}}, b"\0" * 4, "safetensors_header_invalid"),
({"x": {"dtype": 1, "shape": [1], "data_offsets": [0, 4]}}, b"\0" * 4, "safetensors_header_invalid"),
({"x": {"dtype": "F32", "shape": [-1], "data_offsets": [0, 0]}}, b"", "safetensors_header_invalid"),
({"x": {"dtype": "F32", "shape": [1 << 62, 4], "data_offsets": [0, 0]}}, b"", "integer_overflow"),
({"x": {"dtype": "C128", "shape": [1 << 62], "data_offsets": [0, 0]}}, b"", "integer_overflow"),
(
{
"a": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]},
"b": {"dtype": "F32", "shape": [1], "data_offsets": [2, 6]},
},
b"\0" * 6,
"safetensors_range_overlap",
),
],
)
def test_additional_safetensors_header_corruptions_are_blocked(
tmp_path,
header,
payload,
violation,
):
_raw_safetensors(tmp_path / "model.safetensors", header, payload)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert violation in descriptor["safety"]["violations"]
assert descriptor["support_decision"] == "blocked"
def test_truncated_invalid_and_overlarge_safetensors_headers(tmp_path):
weights = tmp_path / "model.safetensors"
weights.write_bytes(b"tiny")
assert "safetensors_truncated" in _assert_contract(inspect_checkpoint(tmp_path))["safety"][
"violations"
]
weights.write_bytes((20).to_bytes(8, "little") + b"{}")
assert "safetensors_truncated" in _assert_contract(inspect_checkpoint(tmp_path))["safety"][
"violations"
]
weights.write_bytes((1).to_bytes(8, "little") + b"{")
assert "safetensors_header_invalid" in _assert_contract(inspect_checkpoint(tmp_path))[
"safety"
]["violations"]
_raw_safetensors(
weights,
{
"a": {"dtype": "F32", "shape": [0], "data_offsets": [0, 0]},
"b": {"dtype": "F32", "shape": [0], "data_offsets": [0, 0]},
},
)
with pytest.raises(CheckpointContractError) as tensor_limit:
inspect_checkpoint(tmp_path, limits=InspectionLimits(max_tensors=1))
assert tensor_limit.value.detail == "max_tensors"
@pytest.mark.parametrize(
("weight_map", "violation"),
[
({}, "hf_weight_map_invalid"),
({"": "model-00001-of-00001.safetensors"}, "hf_weight_map_invalid"),
({"other": "model-00001-of-00001.safetensors"}, "hf_weight_map_tensor_mismatch"),
],
)
def test_hf_index_shape_and_tensor_membership_are_validated(tmp_path, weight_map, violation):
_save(tmp_path / "model-00001-of-00001.safetensors", weight=torch.ones(1))
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map}),
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert violation in descriptor["safety"]["violations"]
def test_hf_index_rejects_unmapped_tensor_in_a_referenced_shard(tmp_path):
_save(
tmp_path / "model-00001-of-00001.safetensors",
declared=torch.ones(1),
undeclared=torch.ones(1),
)
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"declared": "model-00001-of-00001.safetensors",
}
}
),
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert "hf_weight_map_tensor_mismatch" in descriptor["safety"]["violations"]
assert descriptor["support_decision"] == "blocked"
def test_duplicate_json_keys_and_aggregate_tensor_limit_fail_closed(tmp_path):
(tmp_path / ".metadata").write_bytes(b"opaque")
(tmp_path / "fsdp_metadata.json").write_text(
'{"state_dict_type":"SHARDED_STATE_DICT","state_dict_type":"FULL_STATE_DICT"}',
encoding="utf-8",
)
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert "json_invalid" in descriptor["safety"]["violations"]
for path in tuple(tmp_path.iterdir()):
path.unlink()
_save(tmp_path / "model.safetensors", weight=torch.ones(1))
_save(tmp_path / "adapter_model.safetensors", lora_A=torch.ones(1))
(tmp_path / "adapter_config.json").write_text("{}", encoding="utf-8")
with pytest.raises(CheckpointContractError) as tensors:
inspect_checkpoint(tmp_path, limits=InspectionLimits(max_tensors=1))
assert tensors.value.code == "DCI_RESOURCE_LIMIT"
assert tensors.value.detail == "max_tensors"
def test_invalid_bounded_json_is_reported_without_vendor_reader(tmp_path):
(tmp_path / ".metadata").write_bytes(b"opaque")
metadata = tmp_path / "fsdp_metadata.json"
metadata.write_text("[]", encoding="utf-8")
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert "json_object_required" in descriptor["safety"]["violations"]
metadata.write_text("{", encoding="utf-8")
descriptor = _assert_contract(inspect_checkpoint(tmp_path))
assert "json_invalid" in descriptor["safety"]["violations"]
metadata.write_text("{}", encoding="utf-8")
with pytest.raises(CheckpointContractError) as json_limit:
inspect_checkpoint(tmp_path, limits=InspectionLimits(max_json_bytes=1))
assert json_limit.value.detail == "max_json_bytes"
def test_final_revalidation_detects_removed_source(tmp_path, monkeypatch):
weights = tmp_path / "model.safetensors"
_save(weights, weight=torch.ones(1))
original = inspection_module._format_components
def remove_after_probe(files, limits):
result = original(files, limits)
weights.unlink()
return result
monkeypatch.setattr(inspection_module, "_format_components", remove_after_probe)
with pytest.raises(CheckpointContractError) as caught:
inspect_checkpoint(tmp_path)
assert caught.value.code == "DCI_SOURCE_CHANGED"
def test_final_revalidation_detects_nested_inventory_mutation(tmp_path, monkeypatch):
nested = tmp_path / "nested"
nested.mkdir()
_save(nested / "model.safetensors", weight=torch.ones(1))
original = inspection_module._format_components
def add_after_probe(files, limits):
result = original(files, limits)
(nested / "late-file").write_text("changed", encoding="utf-8")
return result
monkeypatch.setattr(inspection_module, "_format_components", add_after_probe)
with pytest.raises(CheckpointContractError) as caught:
inspect_checkpoint(tmp_path)
assert caught.value.code == "DCI_SOURCE_CHANGED"
def test_descriptor_read_error_is_mapped_to_stable_source_change(tmp_path, monkeypatch):
weights = tmp_path / "model.safetensors"
_save(weights, weight=torch.ones(1))
real_read = inspection_module.os.read
calls = 0
def fail_after_inventory(descriptor, size):
nonlocal calls
calls += 1
if calls > 2:
raise OSError("injected read race")
return real_read(descriptor, size)
monkeypatch.setattr(inspection_module.os, "read", fail_after_inventory)
with pytest.raises(CheckpointContractError) as caught:
inspect_checkpoint(tmp_path)
assert caught.value.code == "DCI_SOURCE_CHANGED"
assert caught.value.detail == "source_changed"
tests/test_checkpoint_provenance.py
+524
View File
@@ -0,0 +1,524 @@
"""Versioned checkpoint provenance, lineage, and resume-state truth tests."""
from __future__ import annotations
import json
from copy import deepcopy
from hashlib import sha256
from pathlib import Path
import pytest
from jsonschema import Draft202012Validator
from obliteratus.checkpoint_provenance import (
AdapterIdentity,
ArtifactIdentity,
DatasetIdentity,
LineageEvent,
ProvenanceRecord,
ToolIdentity,
TrainingIdentity,
build_provenance,
classify_resume_state,
migrate_legacy_metadata,
sanitize_command,
verify_provenance_record,
)
from obliteratus.run_archive import RunArchive
ROOT = Path(__file__).resolve().parents[1]
SCHEMA = json.loads(
(ROOT / "docs/checkpoints/schemas/artifact-provenance-v1.schema.json").read_text()
)
DIGEST_A = "sha256:" + "a" * 64
DIGEST_B = "sha256:" + "b" * 64
COMMIT = "c" * 40
def _record(*, output_digests=(DIGEST_B,), command=("checkpoint", "convert")):
base = ArtifactIdentity("hub", "org/base", "0123456789abcdef", DIGEST_A)
return build_provenance(
sources=(ArtifactIdentity("local", "content-addressed-source", None, DIGEST_A),),
converter=ToolIdentity("obliteratus-neutral-writer", "1.0.0", COMMIT),
obliteratus_commit=COMMIT,
configuration_digest=DIGEST_A,
tokenizer=ArtifactIdentity("hub", "org/base", "0123456789abcdef", DIGEST_B),
base_model=base,
command=command,
environment={"python": "3.12.7", "platform": "linux", "packages": {"torch": "2.5"}},
source_topology={"world_size": 4, "tp": 2, "pp": 2},
lineage=(
LineageEvent(
event_id="event-consolidation",
event_type="consolidation",
parent_artifact_ids=("artifact-sha256:" + "d" * 64,),
tool="obliteratus-neutral-writer@1.0.0",
transformations=("deduplicate_declared_replicas",),
),
),
input_digests=(DIGEST_A,),
output_digests=output_digests,
transformations=("consolidation", "canonical_safetensors"),
observed_scopes=("model_weights",),
lost_state=("optimizer_state", "scheduler_state"),
adapter=AdapterIdentity(
adapter_type="lora",
base_model=base,
config_digest=DIGEST_B,
key_map_digest=DIGEST_A,
),
dataset=DatasetIdentity(
identifier="dataset/name",
revision="rev-1",
digest=DIGEST_A,
split="train",
subset=None,
record_count=42,
),
training=TrainingIdentity(
method="adapter_train",
framework="transformers",
framework_version="4.60.0",
hyperparameters_digest=DIGEST_B,
),
unknowns=("optimizer_producer_version",),
)
def test_provenance_is_strict_versioned_canonical_and_content_addressed():
record = _record()
payload = record.to_dict()
Draft202012Validator(SCHEMA).validate(payload)
assert payload["schema_id"] == "obliteratus.artifact-provenance"
assert payload["schema_version"] == "1.0.0"
assert payload["artifact_id"].startswith("artifact-sha256:")
assert payload["record_digest"].startswith("sha256:")
assert json.loads(record.to_json()) == payload
assert record.artifact_id == _record().artifact_id
assert record.to_json() == _record().to_json()
def test_canonical_sets_and_maps_do_not_depend_on_caller_order():
first = _record(output_digests=(DIGEST_A, DIGEST_B))
second = _record(output_digests=(DIGEST_B, DIGEST_A))
assert first.artifact_id == second.artifact_id
assert first.to_json() == second.to_json()
@pytest.mark.parametrize(
("scopes", "expected"),
[
(("model_weights",), "weights_only"),
(("model_weights", "optimizer_state"), "model_and_optimizer"),
(
(
"model_weights",
"optimizer_state",
"scheduler_state",
"rng_state",
"dataloader_state",
"framework_state",
),
"exact_resume",
),
(("optimizer_state",), "unknown"),
((), "unknown"),
],
)
def test_resume_classification_is_derived_only_from_observed_state(scopes, expected):
assert classify_resume_state(scopes) == expected
def test_lineage_vocabulary_keeps_surgery_distinct_from_finetuning():
allowed = {
"consolidation",
"reshard",
"pretrain",
"full_finetune",
"adapter_train",
"adapter_merge",
"quantization",
"dequantization",
"surgery",
}
for event_type in allowed:
assert LineageEvent("event", event_type, (), "tool@1", ()).event_type == event_type
with pytest.raises(ValueError, match="lineage event type"):
LineageEvent("event", "finetuning_surgery", (), "tool@1", ())
def test_command_redacts_secrets_prompts_and_private_paths():
sanitized = sanitize_command(
(
"checkpoint",
"convert",
"/home/alice/private/model",
"--token",
"hf_abcdefghijklmnopqrstuvwxyz",
"--prompt=raw private prompt",
)
)
rendered = json.dumps(sanitized)
assert "/home/alice" not in rendered
assert "hf_" not in rendered
assert "raw private prompt" not in rendered
assert "[REDACTED]" in sanitized
assert any(value.startswith("[LOCAL_PATH:sha256:") for value in sanitized)
def test_legacy_migration_preserves_declared_facts_and_never_invents_digests():
facts = migrate_legacy_metadata(
{
"model": "org/base",
"model_revision": "rev-1",
"tokenizer_revision": None,
"seed": "42",
"dataset_inputs": [{"identifier": "builtin", "sha256": "e" * 64}],
"unmapped_private_field": "must not leak",
}
)
payload = facts.to_dict()
assert payload["base_model"] == {
"identity": "org/base",
"revision": "rev-1",
"digest": None,
}
assert payload["tokenizer"] == {"revision": None, "digest": None}
assert payload["seed"] == "42"
assert payload["datasets"] == [
{"identifier": "builtin", "digest": "sha256:" + "e" * 64}
]
assert "unmapped_private_field" not in json.dumps(payload)
assert "base_model_digest" in payload["unknowns"]
assert "tokenizer_digest" in payload["unknowns"]
def test_run_archive_attaches_same_artifact_identity_without_raw_sensitive_data(tmp_path):
archive = RunArchive(tmp_path)
run_id = archive.begin(["org/base"])
record = _record(
command=("convert", "/home/alice/private/model", "--token", "hf_secretsecretsecret")
)
manifest = archive.attach_checkpoint_provenance(run_id, record)
provenance_path = tmp_path / run_id / "checkpoint-provenance.json"
raw = provenance_path.read_text(encoding="utf-8")
assert manifest["artifact_id"] == record.artifact_id
assert manifest["checkpoint_provenance"]["artifact_id"] == record.artifact_id
assert json.loads(raw)["artifact_id"] == record.artifact_id
assert "/home/alice" not in raw
assert "hf_secret" not in raw
def test_provenance_rejects_secret_bearing_environment_keys():
with pytest.raises(ValueError, match="sensitive key"):
build_provenance(
sources=(ArtifactIdentity("local", "source", None, DIGEST_A),),
converter=ToolIdentity("writer", "1", COMMIT),
obliteratus_commit=COMMIT,
configuration_digest=None,
tokenizer=None,
base_model=None,
command=("convert",),
environment={"API_TOKEN": "secret"},
source_topology={},
lineage=(),
input_digests=(DIGEST_A,),
output_digests=(DIGEST_B,),
transformations=(),
observed_scopes=("model_weights",),
lost_state=(),
)
@pytest.mark.parametrize(
("factory", "message"),
[
(lambda: ArtifactIdentity("other", "source", None, DIGEST_A), "kind is invalid"),
(lambda: ArtifactIdentity("local", "", None, DIGEST_A), "non-empty bounded"),
(
lambda: ArtifactIdentity("local", "hf_abcdefghijklmnopqrstuvwxyz", None, DIGEST_A),
"contains a secret",
),
(lambda: ArtifactIdentity("local", "/private/source", None, DIGEST_A), "private local path"),
(lambda: ArtifactIdentity("local", "source", None, "bad"), "sha256 digest"),
(lambda: ToolIdentity("tool", "1", "A" * 40), "40-character lowercase commit"),
(
lambda: LineageEvent("event", "surgery", ("bad-parent",), "tool@1", ()),
"parent artifact ID",
),
(
lambda: DatasetIdentity("dataset", None, DIGEST_A, None, None, -1),
"record count",
),
(lambda: TrainingIdentity("invalid", None, None, None), "training method"),
(lambda: TrainingIdentity("unknown", None, None, "bad"), "sha256 digest"),
],
)
def test_identity_records_reject_unverifiable_or_sensitive_fields(factory, message):
with pytest.raises(ValueError, match=message):
factory()
def _minimal_provenance(**overrides):
values = {
"sources": (ArtifactIdentity("local", "source", None, DIGEST_A),),
"converter": ToolIdentity("writer", "1", COMMIT),
"obliteratus_commit": COMMIT,
"configuration_digest": None,
"tokenizer": None,
"base_model": None,
"command": ("convert",),
"environment": {"python": "3.12", "platform": "linux", "packages": {}},
"source_topology": {},
"lineage": (),
"input_digests": (DIGEST_A,),
"output_digests": (DIGEST_B,),
"transformations": (),
"observed_scopes": ("model_weights",),
"lost_state": (),
}
values.update(overrides)
return build_provenance(**values)
@pytest.mark.parametrize(
("overrides", "message"),
[
({"sources": ()}, "at least one source"),
({"configuration_digest": "bad"}, "sha256 digest"),
({"environment": {"python": "3", "platform": "linux", "packages": {}, "extra": 1}}, "unsupported fields"),
({"source_topology": {"rank": 1 << 64}}, "outside int64"),
({"source_topology": {"path": "/private/checkpoint"}}, "private local path"),
({"source_topology": {"note": "hf_abcdefghijklmnopqrstuvwxyz"}}, "contains a secret"),
({"source_topology": {1: "value"}}, "non-string key"),
({"source_topology": {"prompt_text": "value"}}, "sensitive key"),
({"source_topology": {"opaque": object()}}, "non-JSON value"),
({"input_digests": ("bad",)}, "sha256 digest"),
],
)
def test_provenance_builder_rejects_incomplete_or_unsafe_evidence(overrides, message):
with pytest.raises(ValueError, match=message):
_minimal_provenance(**overrides)
def test_normalization_accepts_explicit_json_scalars_and_sequences():
record = _minimal_provenance(
source_topology={"active": True, "optional": None, "ranks": [0, 1]},
).to_dict()
assert record["source_topology"] == {
"active": True,
"optional": None,
"ranks": [0, 1],
}
def test_command_redaction_covers_equals_and_bare_sensitive_values():
sanitized = sanitize_command(
(
"convert",
"--output=/private/output",
"--note=ok",
"--credential=secret-value",
"bearer abcdefghijklmnop",
)
)
assert sanitized[1].startswith("--output=[LOCAL_PATH:sha256:")
assert sanitized[2] == "--note=ok"
assert sanitized[3] == "--credential=[REDACTED]"
assert sanitized[4] == "[REDACTED]"
def test_legacy_migration_marks_invalid_identity_and_ignores_invalid_dataset_rows():
payload = migrate_legacy_metadata(
{
"model": "/private/model",
"model_revision": "hf_abcdefghijklmnopqrstuvwxyz",
"tokenizer_revision": 42,
"dataset_inputs": ["invalid", {"identifier": "/private", "sha256": "a" * 64}],
}
).to_dict()
assert payload["base_model"] == {"identity": None, "revision": None, "digest": None}
assert payload["tokenizer"]["revision"] is None
assert payload["datasets"] == []
assert "base_model_identity" in payload["unknowns"]
def test_provenance_verifier_rejects_tampering_and_constructor_identity_disagreement():
record = _record()
tampered = record.to_dict()
tampered["unknowns"] = ["changed"]
with pytest.raises(ValueError, match="record digest mismatch"):
verify_provenance_record(tampered)
with pytest.raises(ValueError, match="identity fields disagree"):
ProvenanceRecord(
"artifact-sha256:" + "f" * 64,
record.record_digest,
record.to_json(),
)
with pytest.raises(ValueError, match="JSON is not canonical"):
ProvenanceRecord(
record.artifact_id,
record.record_digest,
json.dumps(record.to_dict()),
)
def test_public_metadata_bounds_mixed_keys_and_legacy_sensitive_values_fail_closed():
with pytest.raises(ValueError, match="non-string key"):
_minimal_provenance(source_topology={"rank": 0, 1: "invalid"})
with pytest.raises(ValueError, match="invalid key"):
_minimal_provenance(source_topology={"x" * 513: "invalid"})
deeply_nested = {}
cursor = deeply_nested
for _ in range(18):
cursor["next"] = {}
cursor = cursor["next"]
with pytest.raises(ValueError, match="nesting is too deep"):
_minimal_provenance(source_topology=deeply_nested)
with pytest.raises(ValueError, match="command argument"):
_minimal_provenance(command=(object(),))
legacy = migrate_legacy_metadata(
{
"model": "org/base",
"model_revision": "/private/revision",
"tokenizer_revision": "hf_abcdefghijklmnopqrstuvwxyz",
"seed": "/private/seed",
"dataset_inputs": None,
}
).to_dict()
rendered = json.dumps(legacy)
assert "/private" not in rendered
assert "hf_" not in rendered
assert legacy["seed"] is None
assert "seed" in legacy["unknowns"]
def test_public_provenance_parsers_reject_boundedness_and_structure_attacks():
record = _record()
with pytest.raises(ValueError, match="provenance JSON is invalid"):
ProvenanceRecord(record.artifact_id, record.record_digest, '{"a": 1, "a": 2}')
with pytest.raises(ValueError, match="text is too large"):
_minimal_provenance(source_topology={"note": "x" * 1025})
with pytest.raises(ValueError, match="too many fields"):
_minimal_provenance(
source_topology={f"field-{index}": index for index in range(4097)},
)
with pytest.raises(ValueError, match="too many items"):
_minimal_provenance(source_topology={"items": [None] * 4097})
with pytest.raises(ValueError, match="must be an object"):
verify_provenance_record([])
with pytest.raises(ValueError, match="bounded string collection"):
classify_resume_state("model_weights")
with pytest.raises(ValueError, match="collection is invalid"):
_minimal_provenance(input_digests=DIGEST_A)
with pytest.raises(ValueError, match="must contain strings"):
_minimal_provenance(input_digests=(1,))
with pytest.raises(ValueError, match="sources collection"):
_minimal_provenance(sources="source")
with pytest.raises(ValueError, match="artifact identities"):
_minimal_provenance(sources=(object(),))
with pytest.raises(ValueError, match="lineage events"):
_minimal_provenance(lineage=(object(),))
with pytest.raises(ValueError, match="bounded mapping"):
migrate_legacy_metadata([])
def test_provenance_verifier_rejects_each_identity_and_state_layer():
payload = _record().to_dict()
malformed = {**payload, "sources": []}
with pytest.raises(ValueError, match="structure is invalid"):
verify_provenance_record(malformed)
malformed = {**payload, "artifact_id": "invalid"}
with pytest.raises(ValueError, match="artifact ID is invalid"):
verify_provenance_record(malformed)
malformed = {**payload, "state": {}}
with pytest.raises(ValueError, match="state is invalid"):
verify_provenance_record(malformed)
malformed = json.loads(json.dumps(payload))
malformed["state"]["classification"] = "unknown"
with pytest.raises(ValueError, match="classification is not evidence-derived"):
verify_provenance_record(malformed)
malformed = {**payload, "artifact_id": "artifact-sha256:" + "f" * 64}
digest_input = {key: value for key, value in malformed.items() if key != "record_digest"}
encoded = json.dumps(
digest_input,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
allow_nan=False,
).encode("utf-8")
malformed["record_digest"] = f"sha256:{sha256(encoded).hexdigest()}"
with pytest.raises(ValueError, match="artifact ID mismatch"):
verify_provenance_record(malformed)
def test_command_and_legacy_scalar_paths_remain_public_and_canonical():
assert sanitize_command(("convert", f"--note=hf_{'a' * 20}")) == (
"convert",
"--note=[REDACTED]",
)
assert migrate_legacy_metadata({"seed": 7}).to_dict()["seed"] == "7"
@pytest.mark.parametrize(
("mutation", "message"),
[
(lambda value: value["sources"][0].update(kind="invalid"), "source identity"),
(lambda value: value["converter"].update(commit="invalid"), "converter"),
(lambda value: value["command"].append("/private/source"), "command"),
(lambda value: value["environment"].update(packages=[]), "packages"),
(lambda value: value.update(source_topology=[]), "source topology"),
(lambda value: value["lineage"][0].update(extra=True), "lineage event fields"),
(lambda value: value.update(input_digests=[]), "input digests"),
(
lambda value: value.update(output_digests=[DIGEST_B, DIGEST_B]),
"output digests",
),
(lambda value: value["transformations"].reverse(), "transformations"),
(
lambda value: value["state"].update(
observed_scopes=["model_weights", "model_weights"]
),
"observed scopes",
),
(lambda value: value["adapter"].update(config_digest="invalid"), "adapter identity"),
(lambda value: value["dataset"].update(record_count=True), "dataset identity"),
(lambda value: value["training"].update(framework=[]), "training framework"),
(lambda value: value.update(unknowns=["duplicate", "duplicate"]), "unknowns"),
],
)
def test_provenance_verifier_rejects_noncanonical_nested_records(mutation, message):
payload = deepcopy(_record().to_dict())
mutation(payload)
with pytest.raises(ValueError, match=message):
verify_provenance_record(payload)
def test_builder_cannot_emit_schema_invalid_environment_or_duplicate_lineage():
with pytest.raises(ValueError, match="environment python"):
_minimal_provenance(
environment={"python": [], "platform": "linux", "packages": {}},
)
event = LineageEvent("event", "consolidation", (), "writer@1", ())
with pytest.raises(ValueError, match="sorted and unique"):
_minimal_provenance(lineage=(event, event))
tests/test_checkpoint_service.py
+29
View File
@@ -0,0 +1,29 @@
"""Public producer-neutral checkpoint service contracts."""
from __future__ import annotations
import torch
from safetensors.torch import save_file
from obliteratus.checkpoint_capabilities import AdapterRegistry
from obliteratus.checkpoint_inspection import InspectionLimits
from obliteratus.checkpoint_service import CheckpointService
def test_service_inspects_canonical_checkpoint_with_injected_limits(tmp_path):
save_file({"weight": torch.ones(2)}, tmp_path / "model.safetensors")
service = CheckpointService(inspection_limits=InspectionLimits(max_files=4))
report = service.inspect(tmp_path)
assert report.primary_format == "hf_safetensors"
assert report.support_decision == "canonical_hf_ready"
def test_service_has_no_trusted_reader_adapter_or_conversion_entrypoint():
service = CheckpointService()
assert service.adapter_registry == AdapterRegistry()
assert not hasattr(service, "trusted_inspect")
assert not hasattr(service, "load_vendor_checkpoint")
assert not hasattr(service, "convert")
tests/test_checkpoint_writer.py
+779
View File
@@ -0,0 +1,779 @@
"""Bounded deterministic safetensors writer and transactional failure tests."""
from __future__ import annotations
import json
from hashlib import sha256
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from jsonschema import Draft202012Validator
from safetensors.torch import load_file
import obliteratus.checkpoint_writer as writer_module
from obliteratus.checkpoint_errors import CheckpointContractError
from obliteratus.checkpoint_fixtures import load_fixture_case
from obliteratus.checkpoint_provenance import (
ArtifactIdentity,
LineageEvent,
ToolIdentity,
build_provenance,
)
from obliteratus.checkpoint_writer import (
ImmutableCopy,
VerifiedSourceFile,
WriterLimits,
write_canonical_checkpoint,
)
ROOT = Path(__file__).resolve().parents[1]
FIXTURES = ROOT / "tests/fixtures/distributed_checkpoints/v1/cases"
MANIFEST_SCHEMA = json.loads(
(ROOT / "docs/checkpoints/schemas/conversion-manifest-v1.schema.json").read_text()
)
COMMIT = "c" * 40
def _digest_bytes(payload: bytes) -> str:
return f"sha256:{sha256(payload).hexdigest()}"
def _digest_file(path: Path) -> str:
return _digest_bytes(path.read_bytes())
def _tree(root: Path) -> dict[str, str]:
return {
path.relative_to(root).as_posix(): _digest_file(path)
for path in sorted(item for item in root.rglob("*") if item.is_file())
}
def _inputs(
tmp_path: Path,
case_name: str = "world1-complete",
*,
source_topology: dict | None = None,
):
case_root = FIXTURES / case_name
case = load_fixture_case(case_root)
sources = tuple(
VerifiedSourceFile(
path=path,
relative_path=f"fixture/{case_name}/{path.name}",
expected_sha256=_digest_file(path),
)
for path in sorted(item for item in case_root.iterdir() if item.is_file())
)
copy_root = tmp_path / "immutable-base"
copy_root.mkdir(parents=True)
config = copy_root / "config.json"
tokenizer = copy_root / "tokenizer_config.json"
config.write_text('{"architectures":["TinyModel"],"vocab_size":8}\n')
tokenizer.write_text('{"model_max_length":128,"tokenizer_class":"Tiny"}\n')
copies = (
ImmutableCopy(
relative_path="config.json",
source_path=config,
expected_sha256=_digest_file(config),
kind="configuration",
),
ImmutableCopy(
relative_path="tokenizer_config.json",
source_path=tokenizer,
expected_sha256=_digest_file(tokenizer),
kind="tokenizer",
),
)
input_digests = tuple(item.expected_sha256 for item in sources)
def provenance_factory(output_digests: tuple[str, ...]):
return build_provenance(
sources=(
ArtifactIdentity(
"local",
f"synthetic-fixture-{case_name}",
"v1",
_digest_bytes("".join(sorted(input_digests)).encode()),
),
),
converter=ToolIdentity("obliteratus-neutral-writer", "1.0.0", COMMIT),
obliteratus_commit=COMMIT,
configuration_digest=copies[0].expected_sha256,
tokenizer=ArtifactIdentity(
"generated",
"fixture-tokenizer",
"v1",
copies[1].expected_sha256,
),
base_model=ArtifactIdentity(
"generated",
"fixture-base-model",
"v1",
copies[0].expected_sha256,
),
command=("checkpoint", "write", f"fixture:{case_name}"),
environment={"python": "test", "platform": "cpu", "packages": {}},
source_topology=source_topology or {},
lineage=(
LineageEvent(
"event-consolidate",
"consolidation",
(),
"obliteratus-neutral-writer@1.0.0",
("canonical_safetensors",),
),
),
input_digests=input_digests,
output_digests=output_digests,
transformations=("canonical_safetensors", "consolidation"),
observed_scopes=("model_weights",),
lost_state=("optimizer_state", "scheduler_state", "rng_state"),
)
return case, sources, copies, provenance_factory
def _write(tmp_path: Path, destination: Path, **kwargs):
source_topology = {"world_size": 1}
case, sources, copies, provenance_factory = _inputs(
tmp_path,
kwargs.pop("case_name", "world1-complete"),
source_topology=source_topology,
)
result = write_canonical_checkpoint(
destination,
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology=source_topology,
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
**kwargs,
)
return case, result
def test_writer_is_deterministic_sharded_reloadable_and_contract_valid(tmp_path):
first = tmp_path / "first"
second = tmp_path / "second"
case, first_result = _write(
tmp_path / "one",
first,
limits=WriterLimits(max_shard_bytes=32),
)
_, second_result = _write(
tmp_path / "two",
second,
limits=WriterLimits(max_shard_bytes=32),
)
assert _tree(first) == _tree(second)
assert first_result.artifact_id == second_result.artifact_id
assert first_result.output_path == first.resolve()
assert not (first / "pytorch_model.bin").exists()
index = json.loads((first / "model.safetensors.index.json").read_text())
assert list(index["weight_map"]) == sorted(index["weight_map"])
assert len(set(index["weight_map"].values())) > 1
loaded = {}
for shard in sorted(set(index["weight_map"].values())):
loaded.update(load_file(first / shard, device="cpu"))
for logical_id, oracle in case.tensor_oracles.items():
assert torch.equal(loaded[logical_id], oracle.values)
assert str(loaded[logical_id].dtype).removeprefix("torch.") == oracle.dtype
manifest = json.loads((first / "conversion-manifest.json").read_text())
Draft202012Validator(MANIFEST_SCHEMA).validate(manifest)
provenance = json.loads((first / "checkpoint-provenance.json").read_text())
metadata = json.loads((first / "abliteration_metadata.json").read_text())
assert manifest["manifest_id"] == provenance["artifact_id"] == metadata["artifact_id"]
assert provenance["artifact_id"] == first_result.artifact_id
assert manifest["validation"] == {
"coverage": True,
"replicas": True,
"ties": True,
"hashes": True,
"index": True,
"safe_reload": True,
"source_unchanged": True,
"result": "passed",
}
assert manifest["resource_usage"]["actual_peak_ram_bytes"] is None
assert manifest["resource_usage"]["actual_temp_bytes"] is None
def test_single_shard_uses_canonical_direct_filename(tmp_path):
case, result = _write(
tmp_path / "work",
tmp_path / "output",
case_name="world2-uneven-1d",
limits=WriterLimits(max_shard_bytes=1024),
)
assert result.weight_files == ("model.safetensors",)
assert not (result.output_path / "model.safetensors.index.json").exists()
loaded = load_file(result.output_path / "model.safetensors", device="cpu")
assert torch.equal(loaded["model.weight"], case.tensor_oracles["model.weight"].values)
def test_writer_refuses_ties_without_an_explicit_validated_policy(tmp_path):
case, sources, copies, provenance_factory = _inputs(tmp_path)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
tmp_path / "output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
)
assert caught.value.code == "DCI_VALIDATION_FAILED"
assert caught.value.detail == "tie_policy_required"
def test_admission_denial_creates_no_staging_or_output(tmp_path, monkeypatch):
case, sources, copies, provenance_factory = _inputs(tmp_path)
destination = tmp_path / "output"
def forbidden_transaction(*_args, **_kwargs):
raise AssertionError("staging began before admission")
monkeypatch.setattr(writer_module, "atomic_checkpoint_directory", forbidden_transaction)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
destination,
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
limits=WriterLimits(max_output_bytes=1),
)
assert caught.value.code == "DCI_ADMISSION_DENIED"
assert caught.value.detail == "max_output_bytes"
assert not destination.exists()
assert list(tmp_path.glob(".output.staging-*")) == []
def test_copy_and_source_digests_are_verified_before_staging(tmp_path):
case, sources, copies, provenance_factory = _inputs(tmp_path)
copies[0].source_path.write_text("changed\n")
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
tmp_path / "output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
)
assert caught.value.code == "DCI_SOURCE_CHANGED"
assert not (tmp_path / "output").exists()
def test_enospc_preserves_source_and_prior_output_and_cleans_staging(tmp_path, monkeypatch):
case, sources, copies, provenance_factory = _inputs(tmp_path)
destination = tmp_path / "output"
destination.mkdir()
sentinel = destination / "sentinel"
sentinel.write_text("prior")
source_before = {item.relative_path: _digest_file(item.path) for item in sources}
def enospc(_tensors, path):
path.write_bytes(b"partial")
raise OSError(28, "No space left on device")
monkeypatch.setattr(writer_module, "_save_safetensors_file", enospc)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
destination,
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
)
assert caught.value.code == "DCI_MATERIALIZE_FAILED"
assert sentinel.read_text() == "prior"
assert {item.relative_path: _digest_file(item.path) for item in sources} == source_before
assert list(tmp_path.glob(".output.staging-*")) == []
assert list(tmp_path.glob(".output.backup-*")) == []
def test_cancellation_preserves_prior_output_and_cleans_staging(tmp_path, monkeypatch):
class Cancelled(BaseException):
pass
case, sources, copies, provenance_factory = _inputs(tmp_path)
destination = tmp_path / "output"
destination.mkdir()
(destination / "sentinel").write_text("prior")
def cancel(_tensors, _path):
raise Cancelled()
monkeypatch.setattr(writer_module, "_save_safetensors_file", cancel)
with pytest.raises(Cancelled):
write_canonical_checkpoint(
destination,
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
)
assert (destination / "sentinel").read_text() == "prior"
assert list(tmp_path.glob(".output.staging-*")) == []
def test_postwrite_validation_failure_never_promotes(tmp_path, monkeypatch):
case, sources, copies, provenance_factory = _inputs(tmp_path)
destination = tmp_path / "output"
destination.mkdir()
(destination / "sentinel").write_text("prior")
def reject(*_args, **_kwargs):
raise CheckpointContractError(
"DCI_VALIDATION_FAILED",
detail="injected_postwrite_failure",
)
monkeypatch.setattr(writer_module, "_verify_staging", reject)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
destination,
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
)
assert caught.value.detail == "injected_postwrite_failure"
assert (destination / "sentinel").read_text() == "prior"
assert list(tmp_path.glob(".output.staging-*")) == []
@pytest.mark.parametrize("target", ["index", "manifest"])
def test_corrupt_index_or_manifest_is_rejected_before_promotion(
tmp_path,
monkeypatch,
target,
):
case, sources, copies, provenance_factory = _inputs(
tmp_path / "inputs",
source_topology={},
)
original = writer_module._write_json
def corrupt(path, value):
if target == "index" and path.name == "model.safetensors.index.json":
value = {"metadata": {"total_size": 0}, "weight_map": value["weight_map"]}
if target == "manifest" and path.name == "conversion-manifest.json":
value = {**value, "source_topology": {"tampered": True}}
return original(path, value)
monkeypatch.setattr(writer_module, "_write_json", corrupt)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
tmp_path / "output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=provenance_factory,
tie_policy="duplicate_validated",
limits=WriterLimits(max_shard_bytes=32),
)
assert caught.value.code == "DCI_VALIDATION_FAILED"
assert caught.value.detail == (
"output_index_mismatch" if target == "index" else "output_manifest_mismatch"
)
assert not (tmp_path / "output").exists()
def _invoke(tmp_path, *, source_files=None, copies=None, factory=None, **kwargs):
source_topology = {"world_size": 1}
case, default_sources, default_copies, default_factory = _inputs(
tmp_path / "inputs",
source_topology=source_topology,
)
return write_canonical_checkpoint(
tmp_path / "output",
case.fragments,
source_files=default_sources if source_files is None else source_files,
copies=default_copies if copies is None else copies,
descriptor_digest=case.expected_manifest_digest,
source_topology=source_topology,
provenance_factory=default_factory if factory is None else factory,
tie_policy="duplicate_validated",
**kwargs,
)
def test_writer_limit_and_input_records_validate_before_io(tmp_path):
with pytest.raises(ValueError, match="positive integer"):
WriterLimits(max_shard_bytes=0)
with pytest.raises(ValueError, match="cannot exceed 100"):
WriterLimits(min_free_headroom_percent=101)
with pytest.raises(ValueError, match="relative path is unsafe"):
VerifiedSourceFile(tmp_path / "source", "../source", "sha256:" + "0" * 64)
with pytest.raises(ValueError, match="sha256 digest"):
VerifiedSourceFile(tmp_path / "source", "source", "bad")
with pytest.raises(ValueError, match="root-level"):
ImmutableCopy("nested/config.json", tmp_path / "config", "sha256:" + "0" * 64, "configuration")
with pytest.raises(ValueError, match="configuration or tokenizer"):
ImmutableCopy("config.json", tmp_path / "config", "sha256:" + "0" * 64, "other")
def test_writer_rejects_missing_duplicate_or_incomplete_evidence(tmp_path):
with pytest.raises(CheckpointContractError) as empty:
_invoke(tmp_path / "empty", source_files=())
assert empty.value.detail == "source_inventory_empty"
case, sources, copies, factory = _inputs(tmp_path / "duplicate-inputs")
duplicate_source = (*sources, sources[0])
with pytest.raises(ValueError, match="source relative paths must be unique"):
write_canonical_checkpoint(
tmp_path / "duplicate-source-output",
case.fragments,
source_files=duplicate_source,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
with pytest.raises(ValueError, match="copy output paths must be unique"):
write_canonical_checkpoint(
tmp_path / "duplicate-copy-output",
case.fragments,
source_files=sources,
copies=(*copies, copies[0]),
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
with pytest.raises(CheckpointContractError) as missing_copy:
write_canonical_checkpoint(
tmp_path / "missing-copy-output",
case.fragments,
source_files=sources,
copies=(copies[0],),
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert missing_copy.value.detail == "canonical_config_or_tokenizer_missing"
@pytest.mark.parametrize(
("limits", "detail"),
[
(WriterLimits(max_temp_bytes=1), "max_temp_bytes"),
(WriterLimits(max_peak_ram_bytes=1), "max_peak_ram_bytes"),
],
)
def test_every_writer_admission_limit_fails_before_staging(tmp_path, limits, detail):
with pytest.raises(CheckpointContractError) as caught:
_invoke(tmp_path, limits=limits)
assert caught.value.code == "DCI_ADMISSION_DENIED"
assert caught.value.detail == detail
assert not (tmp_path / "output").exists()
def test_tensor_larger_than_configured_shard_limit_is_refused(tmp_path):
with pytest.raises(CheckpointContractError) as caught:
_invoke(tmp_path, limits=WriterLimits(max_shard_bytes=1))
assert caught.value.code == "DCI_ADMISSION_DENIED"
assert caught.value.detail == "tensor_exceeds_max_shard_bytes"
assert not (tmp_path / "output").exists()
def test_post_write_size_limit_is_rechecked_before_promotion(tmp_path, monkeypatch):
monkeypatch.setattr(
writer_module,
"_admit",
lambda *_args, **_kwargs: writer_module._Admission(0, 0, 0, 0),
)
with pytest.raises(CheckpointContractError) as caught:
_invoke(
tmp_path,
limits=WriterLimits(max_output_bytes=1, max_temp_bytes=1),
)
assert caught.value.code == "DCI_ADMISSION_DENIED"
assert caught.value.detail == "post_write_size_limit"
assert not (tmp_path / "output").exists()
def test_filesystem_admission_and_destination_symlink_fail_closed(tmp_path, monkeypatch):
monkeypatch.setattr(writer_module.shutil, "disk_usage", lambda _path: SimpleNamespace(free=0))
with pytest.raises(CheckpointContractError) as capacity:
_invoke(tmp_path / "capacity")
assert capacity.value.detail == "filesystem_free_bytes"
parent_target = tmp_path / "parent-target"
parent_target.mkdir()
parent_link = tmp_path / "parent-link"
parent_link.symlink_to(parent_target, target_is_directory=True)
case, sources, copies, factory = _inputs(tmp_path / "symlink-inputs")
with pytest.raises(CheckpointContractError) as boundary:
write_canonical_checkpoint(
parent_link / "output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert boundary.value.detail == "destination_symlink"
def test_source_ancestor_symlink_and_open_race_fail_before_staging(tmp_path, monkeypatch):
actual = tmp_path / "actual"
case, sources, copies, factory = _inputs(actual)
alias = tmp_path / "alias"
alias.symlink_to(actual, target_is_directory=True)
aliased_sources = (
VerifiedSourceFile(
alias / copies[0].source_path.relative_to(actual),
"source/config.json",
copies[0].expected_sha256,
),
)
with pytest.raises(CheckpointContractError) as symlink:
write_canonical_checkpoint(
tmp_path / "symlink-output",
case.fragments,
source_files=aliased_sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert symlink.value.code == "DCI_SOURCE_BOUNDARY_VIOLATION"
assert symlink.value.detail == "source_symlink"
real_open = writer_module.os.open
blocked_path = sources[0].path
def fail_observed_open(path, flags):
if Path(path) == blocked_path:
raise OSError("injected source replacement")
return real_open(path, flags)
monkeypatch.setattr(writer_module.os, "open", fail_observed_open)
with pytest.raises(CheckpointContractError) as changed:
write_canonical_checkpoint(
tmp_path / "race-output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert changed.value.code == "DCI_SOURCE_CHANGED"
assert not list(tmp_path.glob(".race-output.staging-*"))
def test_source_missing_and_nonregular_inputs_fail_before_staging(tmp_path):
case, sources, copies, factory = _inputs(tmp_path / "source-inputs")
sources = (
VerifiedSourceFile(
tmp_path / "missing-source",
sources[0].relative_path,
sources[0].expected_sha256,
),
*sources[1:],
)
with pytest.raises(CheckpointContractError) as missing:
write_canonical_checkpoint(
tmp_path / "missing-output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert missing.value.code == "DCI_SOURCE_CHANGED"
case, sources, copies, factory = _inputs(tmp_path / "directory-inputs")
source_directory = tmp_path / "source-directory"
source_directory.mkdir()
sources = (
VerifiedSourceFile(
source_directory,
sources[0].relative_path,
sources[0].expected_sha256,
),
*sources[1:],
)
with pytest.raises(CheckpointContractError) as nonregular:
write_canonical_checkpoint(
tmp_path / "directory-output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology={},
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert nonregular.value.detail == "source_not_regular_file"
@pytest.mark.parametrize(
("record", "detail"),
[
({}, "provenance_contract_invalid"),
(
{
"schema_id": "obliteratus.artifact-provenance",
"output_digests": [],
"record_digest": "bad",
},
"provenance_contract_invalid",
),
],
)
def test_provenance_contract_failures_never_promote(tmp_path, record, detail):
fake = SimpleNamespace(artifact_id="artifact-sha256:" + "0" * 64, to_dict=lambda: record)
with pytest.raises(CheckpointContractError) as caught:
_invoke(tmp_path, factory=lambda _digests: fake)
assert caught.value.code == "DCI_EVIDENCE_UNAVAILABLE"
assert caught.value.detail == detail
assert not (tmp_path / "output").exists()
@pytest.mark.parametrize(
("mismatch", "detail"),
[
("input", "provenance_input_digest_mismatch"),
("configuration", "provenance_configuration_digest_mismatch"),
("topology", "provenance_source_topology_mismatch"),
],
)
def test_writer_binds_source_config_and_topology_to_provenance(
tmp_path,
mismatch,
detail,
):
source_topology = {"world_size": 1}
case, sources, copies, _ = _inputs(
tmp_path / "inputs",
source_topology=source_topology,
)
source_digests = tuple(item.expected_sha256 for item in sources)
def factory(output_digests):
return build_provenance(
sources=(ArtifactIdentity("local", "fixture", "v1", source_digests[0]),),
converter=ToolIdentity("writer", "1", COMMIT),
obliteratus_commit=COMMIT,
configuration_digest=(
"sha256:" + "f" * 64
if mismatch == "configuration"
else copies[0].expected_sha256
),
tokenizer=None,
base_model=None,
command=("checkpoint", "write"),
environment={"python": "test", "platform": "cpu", "packages": {}},
source_topology=(
{"world_size": 2} if mismatch == "topology" else source_topology
),
lineage=(),
input_digests=(
("sha256:" + "e" * 64,)
if mismatch == "input"
else source_digests
),
output_digests=output_digests,
transformations=("canonical_safetensors",),
observed_scopes=("model_weights",),
lost_state=(),
)
with pytest.raises(CheckpointContractError) as caught:
write_canonical_checkpoint(
tmp_path / "output",
case.fragments,
source_files=sources,
copies=copies,
descriptor_digest=case.expected_manifest_digest,
source_topology=source_topology,
provenance_factory=factory,
tie_policy="duplicate_validated",
)
assert caught.value.code == "DCI_EVIDENCE_UNAVAILABLE"
assert caught.value.detail == detail
assert not (tmp_path / "output").exists()
def test_provenance_factory_and_promotion_errors_are_stable(tmp_path, monkeypatch):
with pytest.raises(CheckpointContractError) as factory_error:
_invoke(tmp_path / "factory", factory=lambda _digests: (_ for _ in ()).throw(RuntimeError("boom")))
assert factory_error.value.detail == "provenance_factory_failed"
def promotion_failure(*_args, **_kwargs):
raise OSError("promotion unavailable")
monkeypatch.setattr(writer_module, "atomic_checkpoint_directory", promotion_failure)
with pytest.raises(CheckpointContractError) as promotion:
_invoke(tmp_path / "promotion")
assert promotion.value.code == "DCI_PROMOTION_FAILED"
def test_output_json_and_file_record_validation_rejects_unsafe_artifacts(tmp_path):
missing = tmp_path / "missing.json"
with pytest.raises(CheckpointContractError, match="output_json_invalid"):
writer_module._verify_json_object(missing)
malformed = tmp_path / "malformed.json"
malformed.write_text("{", encoding="utf-8")
with pytest.raises(CheckpointContractError, match="output_json_invalid"):
writer_module._verify_json_object(malformed)
array = tmp_path / "array.json"
array.write_text("[]", encoding="utf-8")
with pytest.raises(CheckpointContractError, match="output_json_invalid"):
writer_module._verify_json_object(array)
directory = tmp_path / "directory"
directory.mkdir()
with pytest.raises(CheckpointContractError, match="output_not_regular_file"):
writer_module._output_record(directory)
tests/test_cli.py
+52
View File
@@ -144,6 +144,58 @@ class TestCLIDispatch:
main(["interactive"])
mock_cmd.assert_called_once()
def test_distributed_preflight_is_an_explicit_separate_command(self, tmp_path):
profile = tmp_path / "profile.json"
profile.write_text("{}", encoding="utf-8")
with patch("obliteratus.cli._cmd_distributed") as mock_cmd:
main(["distributed", "preflight", str(profile), "--json"])
args_passed = mock_cmd.call_args.args[0]
assert args_passed.command == "distributed"
assert args_passed.distributed_command == "preflight"
assert args_passed.profile == profile
def test_ordinary_command_never_infers_distributed_mode(self, monkeypatch):
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("MASTER_ADDR", "10.10.0.10")
with (
patch("obliteratus.cli._cmd_abliterate") as ordinary,
patch("obliteratus.cli._cmd_distributed") as distributed,
):
main(["obliterate", "fake/model"])
ordinary.assert_called_once()
distributed.assert_not_called()
@pytest.mark.parametrize("option", ["--token", "--password", "--api-key", "--remote"])
def test_distributed_preflight_rejects_unknown_options_without_echoing_value(
self, option, capsys
):
secret = "hf_private_value_that_must_not_appear"
with pytest.raises(SystemExit) as error:
main(["distributed", "preflight", "profile.json", option, secret])
assert error.value.code == 2
captured = capsys.readouterr()
assert secret not in captured.out
assert secret not in captured.err
@pytest.mark.parametrize(
"argv",
[
["--token", "{secret}", "distributed", "preflight", "profile.json"],
["distributed", "preflight", "profile.json", "--token={secret}"],
],
)
def test_distributed_preflight_secret_prescan_cannot_be_bypassed(
self, argv, capsys
):
secret = "hf_private_value_that_must_not_appear"
with pytest.raises(SystemExit) as error:
main([item.replace("{secret}", secret) for item in argv])
assert error.value.code == 2
captured = capsys.readouterr()
assert secret not in captured.out
assert secret not in captured.err
# 9. --contribute and --contribute-notes are accepted on obliterate
def test_contribute_flags_on_obliterate(self):
"""Verify --contribute and --contribute-notes are accepted args."""
tests/test_cli_boundaries.py
+70
View File
@@ -9,6 +9,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, Mock
import pytest
import torch
from obliteratus import cli
@@ -21,6 +22,7 @@ def ns(**values):
("argv", "target"),
[
(["gpu-calc", "--params", "1"], "_cmd_gpu_calc"),
(["checkpoint", "inspect", "local/checkpoint"], "_cmd_checkpoint"),
(["run", "config.yml"], "_cmd_run"),
(["interactive"], "_cmd_interactive"),
(["models"], "_cmd_models"),
@@ -144,6 +146,74 @@ def test_version_is_stable_and_does_not_dispatch(capsys):
assert capsys.readouterr().out.endswith(f"obliteratus {__version__}\n")
def test_checkpoint_inspect_json_is_machine_readable_without_banner(tmp_path, capsys):
from safetensors.torch import save_file
save_file({"weight": torch.ones(1)}, tmp_path / "model.safetensors")
cli.main(["checkpoint", "inspect", str(tmp_path), "--json"])
payload = json.loads(capsys.readouterr().out)
assert payload["schema_id"] == "obliteratus.checkpoint-descriptor"
assert payload["primary_format"] == "hf_safetensors"
assert payload["safety"]["inspection_level"] == "safe_structure"
def test_checkpoint_inspect_boundary_error_is_stable_json(tmp_path, capsys):
target = tmp_path / "target"
target.write_bytes(b"payload")
(tmp_path / "model.safetensors").symlink_to(target.name)
with pytest.raises(SystemExit) as caught:
cli.main(["checkpoint", "inspect", str(tmp_path), "--json"])
assert caught.value.code == 2
payload = json.loads(capsys.readouterr().out)
assert payload["code"] == "DCI_SOURCE_BOUNDARY_VIOLATION"
assert payload["detail"] == "source_symlink"
assert str(tmp_path) not in json.dumps(payload)
def test_checkpoint_inspect_human_output_honors_explicit_limits(tmp_path, capsys):
cli.main(
[
"checkpoint",
"inspect",
str(tmp_path),
"--max-files",
"10",
"--max-total-bytes",
"1024",
"--max-json-bytes",
"512",
"--max-header-bytes",
"256",
]
)
output = capsys.readouterr().out
assert "format" in output
assert "confidence" in output
assert "unknown" in output
assert "blocked" in output
assert "DCI_UNSUPPORTED_FORMAT_OR_VERSION" in output
def test_checkpoint_inspect_human_boundary_error_is_actionable(tmp_path, capsys):
target = tmp_path / "target"
target.write_bytes(b"payload")
(tmp_path / "model.safetensors").symlink_to(target.name)
with pytest.raises(SystemExit) as caught:
cli.main(["checkpoint", "inspect", str(tmp_path)])
assert caught.value.code == 2
output = capsys.readouterr().out
assert "DCI_SOURCE_BOUNDARY_VIOLATION" in output
assert "source_symlink" in output
assert "Repair the immutable local source boundary" in output
def test_gpu_selection_contract(monkeypatch):
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
cli._apply_gpu_selection(ns(gpus=None, remote=None))
tests/test_distributed_contracts.py
+425
View File
@@ -0,0 +1,425 @@
"""Pure tests for bounded distributed runtime records and frames."""
from __future__ import annotations
from dataclasses import FrozenInstanceError
from enum import Enum
import pytest
import torch
from obliteratus.distributed.consensus import (
assert_rank_order,
decode_frame,
encode_frame,
gloo_all_gather_records,
require_consensus_digest,
require_record_consensus,
unanimous_vote,
)
from obliteratus.distributed.contracts import (
MAX_CONSENSUS_BYTES,
ContractError,
LogicalPlacement,
PlacementKind,
RankInventory,
RunIdentity,
RuntimeStage,
StageMessage,
TopologyPlan,
Vote,
advance_stage,
canonical_record,
contract_digest,
validate_inventory_consensus,
)
from obliteratus.distributed.numerical import distributed_project_weight
def _digest(character: str = "a") -> str:
return character * 64
def _identity(**overrides: object) -> RunIdentity:
values: dict[str, object] = {
"run_id": "1" * 32,
"config_digest": _digest("a"),
"source_digest": _digest("b"),
"model_digest": _digest("c"),
"tokenizer_digest": _digest("d"),
"commit_sha": "e" * 40,
"world_size": 2,
}
values.update(overrides)
return RunIdentity(**values) # type: ignore[arg-type]
def _inventory(rank: int = 0, **overrides: object) -> RankInventory:
values: dict[str, object] = {
"rank": rank,
"local_rank": rank,
"world_size": 2,
"host_digest": _digest(str(rank + 1)),
"device_digest": _digest(chr(ord("a") + rank)),
"device_kind": "cpu",
"total_memory_bytes": 1024,
"free_memory_bytes": 512,
"software_digest": _digest("e"),
"storage_digest": _digest("f"),
}
values.update(overrides)
return RankInventory(**values) # type: ignore[arg-type]
def test_contract_records_are_immutable_and_canonical():
identity = _identity()
with pytest.raises(FrozenInstanceError):
identity.world_size = 3 # type: ignore[misc]
first = canonical_record({"z": 1, "identity": identity, "items": (Vote.ABORT,)})
second = canonical_record({"items": ["abort"], "identity": identity, "z": 1})
assert first == second
assert contract_digest({"value": 1}) == contract_digest({"value": 1})
assert len(contract_digest(identity)) == 64
def test_identity_and_topology_fields_are_digest_bound():
identity = _identity()
for field, value in (
("run_id", "2" * 32),
("config_digest", _digest("1")),
("source_digest", _digest("2")),
("model_digest", _digest("3")),
("tokenizer_digest", _digest("4")),
):
assert contract_digest(identity) != contract_digest(_identity(**{field: value}))
topology = TopologyPlan(2, 0, "gloo", _digest("5"))
assert contract_digest(topology) != contract_digest(TopologyPlan(2, 1, "gloo", _digest("5")))
assert contract_digest(topology) != contract_digest(TopologyPlan(2, 0, "nccl", _digest("5")))
def test_public_records_reject_unknown_fields():
with pytest.raises(TypeError, match="unexpected keyword"):
_identity(unknown="value")
@pytest.mark.parametrize(
("overrides", "message"),
[
({"run_id": "not-a-run"}, "run_id has an invalid format"),
({"config_digest": "A" * 64}, "config_digest has an invalid format"),
({"commit_sha": "e" * 39}, "commit_sha has an invalid format"),
({"world_size": True}, "world_size must be an integer"),
({"world_size": 1}, "world_size must be between 2 and 4096"),
],
)
def test_run_identity_rejects_malformed_or_single_rank_values(overrides, message):
with pytest.raises(ContractError, match=message):
_identity(**overrides)
@pytest.mark.parametrize(
("overrides", "message"),
[
({"rank": 2}, "rank must be between 0 and 1"),
({"local_rank": -1}, "local_rank must be between 0 and 1"),
({"device_kind": "mps"}, "device_kind must be 'cpu' or 'cuda'"),
({"free_memory_bytes": 2048}, "free_memory_bytes cannot exceed"),
({"total_memory_bytes": 0}, "total_memory_bytes must be between"),
({"host_digest": "x" * 64}, "host_digest has an invalid format"),
],
)
def test_rank_inventory_is_bounded(overrides, message):
with pytest.raises(ContractError, match=message):
_inventory(**overrides)
def test_complete_homogeneous_inventory_is_accepted():
validate_inventory_consensus(_identity(), (_inventory(0), _inventory(1)))
@pytest.mark.parametrize(
("records", "message"),
[
((_inventory(0),), "exactly one record per rank"),
((_inventory(0), _inventory(0)), "ranks do not exactly cover"),
(
(_inventory(0), _inventory(1, world_size=3)),
"world_size disagrees",
),
(
(_inventory(0), _inventory(1, host_digest=_digest("1"), device_digest=_digest("a"))),
"unique host/device pair",
),
(
(_inventory(0), _inventory(1, host_digest=_digest("1"), local_rank=0)),
"local ranks must be unique",
),
(
(_inventory(0), _inventory(1, software_digest=_digest("0"))),
"software identities disagree",
),
(
(_inventory(0), _inventory(1, storage_digest=_digest("0"))),
"storage identities disagree",
),
],
)
def test_inventory_consensus_rejects_missing_duplicate_or_divergent_records(records, message):
with pytest.raises(ContractError, match=message):
validate_inventory_consensus(_identity(), records)
def test_inventory_consensus_rejects_wrong_record_types():
with pytest.raises(ContractError, match="identity must"):
validate_inventory_consensus(object(), ()) # type: ignore[arg-type]
with pytest.raises(ContractError, match="invalid rank record"):
validate_inventory_consensus(_identity(), (_inventory(0), object())) # type: ignore[arg-type]
@pytest.mark.parametrize("backend", ["gloo", "nccl"])
def test_topology_accepts_declared_backends(backend):
assert TopologyPlan(2, 0, backend, _digest()).backend == backend
@pytest.mark.parametrize(
("args", "message"),
[
((1, 0, "gloo", _digest()), "world_size"),
((2, 2, "gloo", _digest()), "coordinator_rank"),
((2, 0, "mpi", _digest()), "backend must"),
((2, 0, "gloo", "bad"), "placement_plan_digest"),
],
)
def test_topology_rejects_unqualified_values(args, message):
with pytest.raises(ContractError, match=message):
TopologyPlan(*args)
def _placement(kind: PlacementKind, rank: int, **overrides: object) -> LogicalPlacement:
shard_dim = (
None
if kind is PlacementKind.REPLICATED
else (0 if kind is PlacementKind.COLUMN_WISE else 1)
)
start, end = (0, 0) if shard_dim is None else (rank * 2, (rank + 1) * 2)
values: dict[str, object] = {
"logical_name": "model.layers.0.weight",
"global_shape": (4, 4),
"dtype": "float32",
"kind": kind,
"rank": rank,
"world_size": 2,
"direction_axis": 1,
"shard_dim": shard_dim,
"shard_start": start,
"shard_end": end,
}
values.update(overrides)
return LogicalPlacement(**values) # type: ignore[arg-type]
@pytest.mark.parametrize(
("kind", "expected"),
[
(PlacementKind.COLUMN_WISE, (2, 4)),
(PlacementKind.ROW_WISE, (4, 2)),
(PlacementKind.REPLICATED, (4, 4)),
],
)
def test_logical_placements_report_exact_local_shapes(kind, expected):
assert _placement(kind, 0).local_shape == expected
@pytest.mark.parametrize(
("kind", "overrides", "message"),
[
(PlacementKind.COLUMN_WISE, {"shard_dim": 1}, "requires shard_dim=0"),
(PlacementKind.ROW_WISE, {"shard_dim": 0}, "requires shard_dim=1"),
(PlacementKind.REPLICATED, {"shard_dim": 0}, "cannot declare"),
(PlacementKind.COLUMN_WISE, {"shard_start": 1}, "does not match"),
(PlacementKind.COLUMN_WISE, {"global_shape": (5, 4)}, "equal shard"),
(PlacementKind.COLUMN_WISE, {"global_shape": (4,)}, "two-dimensional"),
(PlacementKind.COLUMN_WISE, {"logical_name": "bad name"}, "invalid format"),
(PlacementKind.COLUMN_WISE, {"direction_axis": 2}, "direction_axis"),
],
)
def test_logical_placement_fails_closed_for_unknown_or_uneven_layouts(kind, overrides, message):
with pytest.raises(ContractError, match=message):
_placement(kind, 0, **overrides)
def test_logical_placement_rejects_unknown_dtype():
with pytest.raises(ContractError, match="dtype is not supported"):
_placement(PlacementKind.COLUMN_WISE, 0, dtype="float8_e4m3fn")
def test_lifecycle_accepts_only_the_documented_happy_path_and_abort_path():
happy = [
RuntimeStage.CREATED,
RuntimeStage.PREFLIGHTED,
RuntimeStage.LOADED,
RuntimeStage.PROBED,
RuntimeStage.DISTILLED,
RuntimeStage.PREPARED,
RuntimeStage.MUTATING,
RuntimeStage.VERIFIED,
RuntimeStage.STAGED,
RuntimeStage.PUBLISHED,
]
for current, requested in zip(happy, happy[1:]):
assert advance_stage(current, requested) is requested
assert advance_stage(RuntimeStage.LOADED, RuntimeStage.ABORTING) is RuntimeStage.ABORTING
assert advance_stage(RuntimeStage.ABORTING, RuntimeStage.ABORTED) is RuntimeStage.ABORTED
assert (
advance_stage(RuntimeStage.ABORTING, RuntimeStage.QUARANTINED) is RuntimeStage.QUARANTINED
)
@pytest.mark.parametrize(
("current", "requested"),
[
(RuntimeStage.CREATED, RuntimeStage.LOADED),
(RuntimeStage.MUTATING, RuntimeStage.STAGED),
(RuntimeStage.PUBLISHED, RuntimeStage.ABORTING),
(RuntimeStage.ABORTED, RuntimeStage.CREATED),
(RuntimeStage.QUARANTINED, RuntimeStage.PUBLISHED),
],
)
def test_lifecycle_rejects_skipped_or_post_terminal_transitions(current, requested):
with pytest.raises(ContractError, match="invalid distributed stage transition"):
advance_stage(current, requested)
with pytest.raises(ContractError, match="must be RuntimeStage"):
advance_stage(current.value, requested) # type: ignore[arg-type]
def test_stage_message_requires_sequenced_typed_abort_evidence():
message = StageMessage(
run_id="1" * 32,
identity_digest=_digest(),
rank=1,
sequence=7,
stage=RuntimeStage.ABORTING,
vote=Vote.ABORT,
error_code="LMS_TEST_FAILURE",
)
assert contract_digest(message)
assert StageMessage.from_bytes(message.to_bytes()) == message
with pytest.raises(ContractError, match="not canonical"):
StageMessage.from_bytes(b" " + message.to_bytes())
duplicate = message.to_bytes().replace(b'{"error_code":', b'{"rank":0,"error_code":', 1)
with pytest.raises(ContractError, match="duplicate"):
StageMessage.from_bytes(duplicate)
with pytest.raises(ContractError, match="requires an error_code"):
StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.ABORTING, Vote.ABORT)
with pytest.raises(ContractError, match="valid only with an abort vote"):
StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.LOADED, None, "LMS_FAIL")
with pytest.raises(ContractError, match="error_code has an invalid format"):
StageMessage("1" * 32, _digest(), 0, 1, RuntimeStage.ABORTING, Vote.ABORT, "bad")
@pytest.mark.parametrize(
"value",
[1.5, {1: "bad key"}, {"nested": object()}],
)
def test_canonical_record_rejects_ambiguous_types(value):
with pytest.raises(ContractError):
canonical_record(value)
def test_canonical_record_rejects_enum_type_bypass_cycles_and_resource_abuse():
class FloatEnum(Enum):
VALUE = 1.5
with pytest.raises(ContractError, match="unsupported type float"):
canonical_record(FloatEnum.VALUE)
cyclic: dict[str, object] = {}
cyclic["cycle"] = cyclic
with pytest.raises(ContractError, match="reference cycle"):
canonical_record(cyclic)
nested: object = None
for _ in range(18):
nested = [nested]
with pytest.raises(ContractError, match="nesting depth"):
canonical_record(nested)
with pytest.raises(ContractError, match="4096 items"):
canonical_record([None] * 4097)
with pytest.raises(ContractError, match="signed 64-bit"):
canonical_record(2**63)
def test_record_gather_rejects_noncanonical_raw_bytes_before_group_use():
with pytest.raises(ContractError, match="unsupported type bytes"):
gloo_all_gather_records(b"not-a-canonical-record")
def test_canonical_record_enforces_size_before_collective_allocation():
with pytest.raises(ContractError, match="exceeds 16 bytes"):
canonical_record({"value": "x" * 20}, max_bytes=16)
with pytest.raises(ContractError, match="max_bytes"):
canonical_record({}, max_bytes=MAX_CONSENSUS_BYTES + 1)
def test_fixed_frame_round_trip_and_zero_padding():
frame = encode_frame(b"record", capacity=16)
assert frame.dtype == torch.uint8
assert frame.numel() == 20
assert decode_frame(frame) == b"record"
@pytest.mark.parametrize(
("payload", "capacity", "message"),
[
("not bytes", 16, "payload must be bytes"),
(b"too long", 2, "exceeds 2 bytes"),
(b"ok", 0, "capacity must be between"),
(b"ok", MAX_CONSENSUS_BYTES + 1, "capacity must be between"),
],
)
def test_fixed_frame_rejects_invalid_input(payload, capacity, message):
with pytest.raises(ContractError, match=message):
encode_frame(payload, capacity=capacity) # type: ignore[arg-type]
def test_frame_decoder_rejects_type_shape_length_and_padding_corruption():
with pytest.raises(ContractError, match="one-dimensional uint8"):
decode_frame(torch.zeros((2, 2)))
with pytest.raises(ContractError, match="invalid capacity"):
decode_frame(torch.zeros(4, dtype=torch.uint8))
too_long = encode_frame(b"a", capacity=2)
too_long[:4] = torch.tensor(list((3).to_bytes(4, "big")), dtype=torch.uint8)
with pytest.raises(ContractError, match="length exceeds"):
decode_frame(too_long)
bad_padding = encode_frame(b"a", capacity=2)
bad_padding[-1] = 1
with pytest.raises(ContractError, match="padding must be zero"):
decode_frame(bad_padding)
def test_collective_helpers_refuse_without_a_gloo_group():
with pytest.raises(ContractError, match="must be initialized"):
gloo_all_gather_records({"rank": 0})
with pytest.raises(ContractError, match="64 lowercase"):
require_consensus_digest("BAD")
with pytest.raises(ContractError, match="must be initialized"):
require_record_consensus(_identity())
with pytest.raises(ContractError, match="vote sequence"):
unanimous_vote(-1, True)
with pytest.raises(ContractError, match="accepted must"):
unanimous_vote(1, 1) # type: ignore[arg-type]
with pytest.raises(ContractError, match="initialized process group"):
distributed_project_weight(
torch.eye(4),
torch.ones(4),
_placement(PlacementKind.REPLICATED, 0),
)
with pytest.raises(ContractError, match="placement must"):
distributed_project_weight(torch.eye(4), torch.ones(4), object()) # type: ignore[arg-type]
def test_rank_order_rejects_missing_duplicate_and_reordered_records():
records = (_inventory(0), _inventory(1))
assert_rank_order(records, world_size=2)
with pytest.raises(ContractError, match="record count"):
assert_rank_order(records[:1], world_size=2)
with pytest.raises(ContractError, match="global-rank order"):
assert_rank_order(tuple(reversed(records)), world_size=2)
tests/test_distributed_evidence.py
+113
View File
@@ -0,0 +1,113 @@
"""Redacted, atomic distributed-preflight evidence tests."""
from __future__ import annotations
import json
import stat
import pytest
from obliteratus.distributed.contracts import (
ContractError,
RuntimeStage,
StageMessage,
Vote,
)
from obliteratus.distributed.evidence import (
PreflightEvidence,
read_evidence,
read_stage_message,
write_evidence,
write_stage_message,
)
def test_evidence_is_allowlisted_redacted_and_private(tmp_path):
hostile = "Bearer hf_secret password=/private/model 10.10.0.10 host.internal"
evidence = PreflightEvidence.failure(
run_id="1" * 32,
config_digest="2" * 64,
code="LMS_DIAGNOSTIC_REDACTION_FAILED",
world_size=2,
evidence_tier="protocol_cpu",
detail=hostile,
)
path = tmp_path / "evidence.json"
write_evidence(path, evidence)
raw = path.read_text(encoding="utf-8")
parsed = json.loads(raw)
assert parsed["result"] == "failed"
assert parsed["error_code"] == "LMS_DIAGNOSTIC_REDACTION_FAILED"
assert hostile not in raw
assert "10.10.0.10" not in raw
assert stat.S_IMODE(path.stat().st_mode) == 0o600
assert read_evidence(path) == evidence
def test_cleanup_failure_is_quarantined():
evidence = PreflightEvidence.failure(
run_id="1" * 32,
config_digest="2" * 64,
code="LMS_CLEANUP_INCOMPLETE",
world_size=2,
evidence_tier="candidate_preflight",
detail="secret",
)
assert evidence.result == "quarantined"
assert evidence.error_code == "LMS_CLEANUP_INCOMPLETE"
def test_evidence_never_overwrites_an_existing_attempt_record(tmp_path):
evidence = PreflightEvidence.failure(
run_id="1" * 32,
config_digest="2" * 64,
code="LMS_EVIDENCE_UNAVAILABLE",
world_size=2,
evidence_tier="candidate_preflight",
)
path = tmp_path / "evidence.json"
write_evidence(path, evidence)
with pytest.raises(ContractError, match="already exists"):
write_evidence(path, evidence)
def test_evidence_parent_must_be_private_and_cannot_be_a_symlink(tmp_path):
evidence = PreflightEvidence.failure(
run_id="1" * 32,
config_digest="2" * 64,
code="LMS_EVIDENCE_UNAVAILABLE",
world_size=2,
evidence_tier="candidate_preflight",
)
public = tmp_path / "public"
public.mkdir(mode=0o755)
with pytest.raises(ContractError, match="private"):
write_evidence(public / "evidence.json", evidence)
private = tmp_path / "private"
private.mkdir(mode=0o700)
alias = tmp_path / "alias"
alias.symlink_to(private, target_is_directory=True)
with pytest.raises(ContractError):
write_evidence(alias / "evidence.json", evidence)
def test_evidence_decoder_rejects_noncanonical_and_duplicate_json():
duplicate = b'{"schema_version":1,"schema_version":1,"result":"failed"}\n'
with pytest.raises(ContractError, match="duplicate"):
PreflightEvidence.from_bytes(duplicate)
def test_lifecycle_receipt_uses_a_distinct_nonterminal_schema(tmp_path):
message = StageMessage(
run_id="1" * 32,
identity_digest="2" * 64,
rank=0,
sequence=1,
stage=RuntimeStage.PREFLIGHTED,
vote=Vote.PREPARED,
)
path = tmp_path / "prepared.stage.json"
write_stage_message(path, message)
assert read_stage_message(path) == message
with pytest.raises(ContractError):
read_evidence(path)
tests/test_distributed_gloo.py
+871
View File
@@ -0,0 +1,871 @@
"""Two-process CPU/Gloo semantic and failure tests for issue 58."""
from __future__ import annotations
import hashlib
import multiprocessing as mp
import os
import socket
import time
from dataclasses import replace
from datetime import timedelta
from pathlib import Path
from queue import Empty
from typing import Any
import pytest
import torch
import torch.distributed as dist
from obliteratus.analysis.numerical_contracts import project_weight_against_direction
from obliteratus.distributed.config import DistributedPreflightConfig
from obliteratus.distributed.consensus import (
gloo_all_gather_records,
require_consensus_digest,
require_record_consensus,
unanimous_vote,
)
from obliteratus.distributed.contracts import (
ContractError,
LogicalPlacement,
PlacementKind,
RunIdentity,
RuntimeStage,
Vote,
)
from obliteratus.distributed.evidence import read_evidence, read_stage_message
from obliteratus.distributed.numerical import distributed_project_weight
from obliteratus.distributed.launcher import TorchrunEnvironment
from obliteratus.distributed.preflight import (
LocalSnapshot,
SourceIdentity,
execute_preflight,
run_preflight,
)
WORLD_SIZE = 2
def _preflight_config(root: Path) -> DistributedPreflightConfig:
root.joinpath("staging").mkdir(exist_ok=True)
software = (
("accelerate", "test"),
("cuda", "unavailable"),
("driver", "unavailable"),
("machine", "test"),
("nccl", "unavailable"),
("platform", "test"),
("python", "test"),
("safetensors", "test"),
("torch", "test"),
("transformers", "test"),
)
return DistributedPreflightConfig(
run_id="1" * 32,
rendezvous_id="2" * 32,
world_size=2,
local_world_size=1,
source_digest="a" * 64,
model_digest="b" * 64,
tokenizer_digest="c" * 64,
commit_sha="d" * 40,
code_digest="0" * 64,
tensor_parallel_size=2,
coordinator_rank=0,
placement_plan_digest="e" * 64,
dimension_divisors=(2, 4),
master_addr="10.10.0.10",
master_port=29500,
network_interface="eth0",
allowed_master_cidrs=("10.10.0.0/24",),
source_path=root / "source",
staging_path=root / "staging",
storage_digest="f" * 64,
min_free_device_memory_bytes=1,
min_free_host_memory_bytes=1,
min_free_staging_bytes=1,
max_source_files=1000,
max_source_bytes=1024,
max_source_file_bytes=1024,
source_timeout_seconds=2,
init_timeout_seconds=2,
collective_timeout_seconds=2,
teardown_timeout_seconds=2,
software_versions=software,
device_kind="cpu",
device_name="cpu",
compute_capability="none",
evidence_tier="protocol_cpu",
allowed_environment_keys=(),
local_files_only=True,
trust_remote_code=False,
allow_runtime_install=False,
allow_plugins=False,
allow_compilation=False,
allow_adapters=False,
allow_quantization=False,
evidence_path=root / "staging" / ("1" * 32) / "preflight.json",
digest="9" * 64,
).validate()
class _FixedProbes:
def __init__(self, rank: int, software: tuple[tuple[str, str], ...]):
self.rank = rank
self.software = software
def collect(self, config, launch):
return LocalSnapshot(
host_identity=f"host-{self.rank}",
device_identity=f"cpu-{self.rank}",
device_name="cpu",
compute_capability="none",
device_kind="cpu",
total_device_memory_bytes=8192,
free_device_memory_bytes=4096,
total_host_memory_bytes=16384,
free_host_memory_bytes=8192,
free_staging_bytes=8192,
storage_identity=config.storage_digest,
source=SourceIdentity(
config.source_digest,
config.model_digest,
config.tokenizer_digest,
2,
2,
),
software_versions=self.software,
commit_sha=config.commit_sha,
code_digest=config.code_digest,
)
def _execute_worker(
rank: int,
root_text: str,
mode: str,
port: int,
queue: mp.Queue,
) -> None:
root = Path(root_text)
config = replace(
_preflight_config(root),
master_addr="127.0.0.1",
master_port=port,
init_timeout_seconds=10,
collective_timeout_seconds=10,
)
launch = TorchrunEnvironment(
rank=rank,
local_rank=0,
world_size=2,
local_world_size=1,
group_rank=rank,
role_rank=rank,
role_world_size=2,
master_endpoint_digest="8" * 64,
run_id=config.run_id,
rendezvous_id=config.rendezvous_id,
network_interface=config.network_interface,
)
os.environ["MASTER_ADDR"] = config.master_addr
os.environ["MASTER_PORT"] = str(config.master_port)
probes = _FixedProbes(rank, config.software_versions)
if mode == "identity_failure" and rank == 1:
original_collect = probes.collect
def collect_with_wrong_identity(candidate_config, candidate_launch):
snapshot = original_collect(candidate_config, candidate_launch)
return replace(
snapshot,
source=replace(snapshot.source, source_digest="7" * 64),
)
probes.collect = collect_with_wrong_identity # type: ignore[method-assign]
if mode == "storage_failure" and rank == 1:
original_collect = probes.collect
def collect_with_wrong_storage(candidate_config, candidate_launch):
snapshot = original_collect(candidate_config, candidate_launch)
return replace(snapshot, storage_identity="7" * 64)
probes.collect = collect_with_wrong_storage # type: ignore[method-assign]
if mode == "resource_failure" and rank == 1:
original_collect = probes.collect
def collect_without_headroom(candidate_config, candidate_launch):
snapshot = original_collect(candidate_config, candidate_launch)
return replace(snapshot, free_device_memory_bytes=0)
probes.collect = collect_without_headroom # type: ignore[method-assign]
if mode == "commit_failure" and rank == 1:
original_collect = probes.collect
def collect_with_wrong_commit(candidate_config, candidate_launch):
snapshot = original_collect(candidate_config, candidate_launch)
return replace(snapshot, commit_sha="7" * 40)
probes.collect = collect_with_wrong_commit # type: ignore[method-assign]
if mode == "code_failure" and rank == 1:
original_collect = probes.collect
def collect_with_wrong_code(candidate_config, candidate_launch):
snapshot = original_collect(candidate_config, candidate_launch)
return replace(snapshot, code_digest="7" * 64)
probes.collect = collect_with_wrong_code # type: ignore[method-assign]
if mode == "stage_timeout":
def collect_after_timeout(candidate_config, candidate_launch):
del candidate_config, candidate_launch
raise TimeoutError("secret-bearing timeout detail")
probes.collect = collect_after_timeout # type: ignore[method-assign]
if mode == "cancelled":
def collect_after_cancel(candidate_config, candidate_launch):
del candidate_config, candidate_launch
raise KeyboardInterrupt
probes.collect = collect_after_cancel # type: ignore[method-assign]
if mode == "native_stderr":
original_collect = probes.collect
def collect_with_native_diagnostic(candidate_config, candidate_launch):
os.write(2, b"secret-bearing native child diagnostic\n")
return original_collect(candidate_config, candidate_launch)
probes.collect = collect_with_native_diagnostic # type: ignore[method-assign]
if mode == "rank_exception" and rank == 1:
def collect_after_error(candidate_config, candidate_launch):
del candidate_config, candidate_launch
raise RuntimeError("secret-bearing rank exception")
probes.collect = collect_after_error # type: ignore[method-assign]
if mode == "execute_hang" and rank == 1:
original_collect = probes.collect
def collect_after_hang(candidate_config, candidate_launch):
time.sleep(60)
return original_collect(candidate_config, candidate_launch)
probes.collect = collect_after_hang # type: ignore[method-assign]
if mode == "execute_early_exit" and rank == 1:
def exit_before_attestation(candidate_config, candidate_launch):
del candidate_config, candidate_launch
os._exit(17)
probes.collect = exit_before_attestation # type: ignore[method-assign]
import obliteratus.distributed.launcher as launcher_module
import obliteratus.distributed.preflight as preflight_module
original_network_validator = launcher_module.validate_network_interface
original_writer = preflight_module.write_stage_message
original_destroy = dist.destroy_process_group
launcher_module.validate_network_interface = lambda *args, **kwargs: None
if mode == "evidence_failure" and rank == 0:
def fail_prepared(path, evidence):
if Path(path).name == ".preflight.prepared.stage.json":
raise OSError("injected secret-bearing sink failure")
return original_writer(path, evidence)
preflight_module.write_stage_message = fail_prepared
if mode == "teardown_failure" and rank == 1:
def fail_destroy():
raise RuntimeError("injected secret-bearing teardown failure")
dist.destroy_process_group = fail_destroy # type: ignore[method-assign]
try:
evidence = execute_preflight(config, launch, probes=probes)
queue.put((rank, "ok", (evidence.result, evidence.error_code)))
except Exception as exc:
queue.put((rank, "error", (getattr(exc, "code", None), str(exc))))
finally:
launcher_module.validate_network_interface = original_network_validator
preflight_module.write_stage_message = original_writer
dist.destroy_process_group = original_destroy # type: ignore[method-assign]
if dist.is_available() and dist.is_initialized():
original_destroy()
def _run_execute_workers(tmp_path: Path, mode: str, *, timeout: float = 35.0):
context = mp.get_context("spawn")
queue = context.Queue()
listener = socket.socket()
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
listener.close()
processes = [
context.Process(
target=_execute_worker,
args=(rank, str(tmp_path), mode, port, queue),
)
for rank in range(WORLD_SIZE)
]
for process in processes:
process.start()
for process in processes:
process.join(timeout)
if process.is_alive():
process.terminate()
process.join(5)
pytest.fail(f"{mode} execute worker {process.pid} did not terminate")
assert process.exitcode == 0
results = sorted(queue.get(timeout=2) for _ in range(WORLD_SIZE))
queue.close()
queue.join_thread()
return results
def _run_execute_disruption(
tmp_path: Path,
mode: str,
*,
ranks: tuple[int, ...] = (0, 1),
timeout: float = 40.0,
):
context = mp.get_context("spawn")
queue = context.Queue()
listener = socket.socket()
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
listener.close()
processes = [
context.Process(
target=_execute_worker,
args=(rank, str(tmp_path), mode, port, queue),
)
for rank in ranks
]
for process in processes:
process.start()
for process in processes:
process.join(timeout)
if process.is_alive():
process.terminate()
process.join(5)
pytest.fail(f"{mode} execute worker {process.pid} did not terminate")
results = [queue.get(timeout=2)]
queue.close()
queue.join_thread()
return sorted(results), tuple(process.exitcode for process in processes)
def _placement(kind: PlacementKind, rank: int, *, direction_axis: int = 1) -> LogicalPlacement:
shard_dim = (
None
if kind is PlacementKind.REPLICATED
else (0 if kind is PlacementKind.COLUMN_WISE else 1)
)
global_shape = (4, 6) if direction_axis == 0 else (4, 4)
shard_size = 0 if shard_dim is None else global_shape[shard_dim] // WORLD_SIZE
start, end = (0, 0) if shard_dim is None else (rank * shard_size, (rank + 1) * shard_size)
return LogicalPlacement(
logical_name="model.layers.0.proj.weight",
global_shape=global_shape,
dtype="float64",
kind=kind,
rank=rank,
world_size=WORLD_SIZE,
direction_axis=direction_axis,
shard_dim=shard_dim,
shard_start=start,
shard_end=end,
)
def _full_inputs(transposed: bool = False) -> tuple[torch.Tensor, torch.Tensor]:
shape = (4, 6) if transposed else (4, 4)
element_count = shape[0] * shape[1]
weight = torch.arange(1, element_count + 1, dtype=torch.float64).reshape(shape)
weight = (weight - (element_count + 1) / 2) / 7.0
return weight, torch.tensor((1.0, -2.0, 0.5, 3.0), dtype=torch.float64)
def _projection_payload(rank: int, mode: str) -> dict[str, Any]:
transposed = mode.startswith("transposed")
full_weight, direction = _full_inputs(transposed)
if mode == "column":
placement = _placement(PlacementKind.COLUMN_WISE, rank)
local_weight = full_weight[rank * 2 : (rank + 1) * 2]
elif mode == "row":
placement = _placement(PlacementKind.ROW_WISE, rank)
local_weight = full_weight[:, rank * 2 : (rank + 1) * 2]
elif mode == "transposed_column":
placement = _placement(PlacementKind.COLUMN_WISE, rank, direction_axis=0)
local_weight = full_weight[rank * 2 : (rank + 1) * 2]
elif mode == "transposed_row":
placement = _placement(PlacementKind.ROW_WISE, rank, direction_axis=0)
local_weight = full_weight[:, rank * 3 : (rank + 1) * 3]
elif mode in {"nonfinite", "zero_direction", "zero_weight", "column_no_norm"}:
placement = _placement(PlacementKind.COLUMN_WISE, rank)
local_weight = full_weight[rank * 2 : (rank + 1) * 2].clone()
elif mode == "replicated":
placement = _placement(PlacementKind.REPLICATED, rank)
local_weight = full_weight.clone()
else:
raise AssertionError(f"unknown projection mode {mode}")
if mode == "nonfinite" and rank == 1:
local_weight[0, 0] = float("nan")
if mode == "zero_direction":
direction = torch.zeros_like(direction)
if mode == "zero_weight":
local_weight.zero_()
result = distributed_project_weight(
local_weight,
direction,
placement,
norm_preserve=mode != "column_no_norm",
regularization=0.2,
projection_row_fraction=0.5,
)
return {
"weight": result.weight.tolist(),
"projected": result.projected,
"coefficient_norm_sq": result.coefficient_norm_sq,
"layout": result.layout,
}
def _validation_messages(rank: int) -> list[str]:
placement = _placement(PlacementKind.COLUMN_WISE, rank)
weight = _full_inputs()[0][rank * 2 : (rank + 1) * 2]
direction = _full_inputs()[1]
cases: tuple[tuple[torch.Tensor, object, dict[str, Any], str], ...] = (
(weight[:1], direction, {}, "local weight shape"),
(torch.ones_like(weight, dtype=torch.int64), direction, {}, "floating-point"),
(weight.float(), direction, {}, "weight dtype"),
(weight, object(), {}, "direction does not match"),
(weight, direction[:3], {}, "direction does not match"),
(weight, torch.ones(4, dtype=torch.int64), {}, "direction does not match"),
(weight, direction, {"regularization": True}, "finite number"),
(weight, direction, {"regularization": 2.0}, "in [0, 1]"),
(weight, direction, {"projection_row_fraction": False}, "finite number"),
(weight, direction, {"projection_row_fraction": 0.0}, "in (0, 1]"),
(weight, direction, {"max_norm_ratio": True}, "positive and finite"),
)
messages = []
for candidate_weight, candidate_direction, kwargs, expected in cases:
try:
distributed_project_weight(
candidate_weight,
candidate_direction, # type: ignore[arg-type]
placement,
**kwargs,
)
except ContractError as exc:
assert expected in str(exc)
messages.append(str(exc))
else:
raise AssertionError(f"validation case unexpectedly passed: {expected}")
return messages
def _worker(rank: int, init_file: str, mode: str, queue: mp.Queue) -> None:
try:
dist.init_process_group(
"gloo",
init_method=f"file://{init_file}",
rank=rank,
world_size=WORLD_SIZE,
timeout=timedelta(seconds=5),
)
if mode == "success_matrix":
records = gloo_all_gather_records({"rank": rank}, capacity=64)
digest = hashlib.sha256(b"same").hexdigest()
agreed = require_consensus_digest(digest)
vote = unanimous_vote(3, True)
payload: dict[str, Any] = {
"records": ([item.decode() for item in records], agreed, vote),
"vote_no": unanimous_vote(4, rank == 0),
}
for projection_mode in (
"column",
"row",
"replicated",
"transposed_column",
"transposed_row",
"nonfinite",
"zero_direction",
"zero_weight",
"column_no_norm",
):
payload[projection_mode] = _projection_payload(rank, projection_mode)
payload["validation"] = _validation_messages(rank)
queue.put((rank, "ok", payload))
elif mode == "digest_mismatch":
digest = hashlib.sha256(f"rank-{rank}".encode()).hexdigest()
require_consensus_digest(digest)
queue.put((rank, "unexpected", None))
elif mode == "identity_mismatch":
identity = RunIdentity(
run_id="1" * 32,
config_digest=str(rank + 1) * 64,
source_digest="a" * 64,
model_digest="b" * 64,
tokenizer_digest="c" * 64,
commit_sha="d" * 40,
world_size=WORLD_SIZE,
)
require_record_consensus(identity)
queue.put((rank, "unexpected", None))
elif mode == "stale_sequence":
unanimous_vote(rank, True)
queue.put((rank, "unexpected", None))
elif mode == "placement_disagreement":
if rank == 0:
placement = _placement(PlacementKind.COLUMN_WISE, rank)
weight = _full_inputs()[0][:2]
else:
placement = _placement(PlacementKind.ROW_WISE, rank)
weight = _full_inputs()[0][:, 2:]
distributed_project_weight(weight, _full_inputs()[1], placement)
queue.put((rank, "unexpected", None))
elif mode == "placement_name_disagreement":
placement = _placement(PlacementKind.COLUMN_WISE, rank)
if rank == 1:
placement = replace(placement, logical_name="model.layers.1.proj.weight")
distributed_project_weight(
_full_inputs()[0][rank * 2 : (rank + 1) * 2], _full_inputs()[1], placement
)
queue.put((rank, "unexpected", None))
elif mode == "one_rank_error":
if rank == 1:
raise RuntimeError("injected rank error")
gloo_all_gather_records({"rank": rank}, capacity=64)
queue.put((rank, "unexpected", None))
elif mode == "early_exit":
if rank == 1:
queue.put((rank, "exited", None))
return
gloo_all_gather_records({"rank": rank}, capacity=64)
queue.put((rank, "unexpected", None))
elif mode == "hang":
if rank == 1:
time.sleep(7)
gloo_all_gather_records({"rank": rank}, capacity=64)
queue.put((rank, "unexpected", None))
elif mode == "preflight_success":
config = _preflight_config(Path(init_file).parent)
launch = TorchrunEnvironment(
rank=rank,
local_rank=0,
world_size=2,
local_world_size=1,
group_rank=rank,
role_rank=rank,
role_world_size=2,
master_endpoint_digest="8" * 64,
run_id=config.run_id,
rendezvous_id=config.rendezvous_id,
network_interface=config.network_interface,
)
result = run_preflight(
config,
launch,
probes=_FixedProbes(rank, config.software_versions),
)
queue.put(
(
rank,
"ok",
{
"accepted": len(result.attestations),
"identity_digest": result.identity_digest,
"backend": result.topology.backend,
},
)
)
else:
raise AssertionError(f"unknown worker mode {mode}")
except Exception as exc:
queue.put((rank, "error", (type(exc).__name__, str(exc))))
finally:
if dist.is_available() and dist.is_initialized():
dist.destroy_process_group()
def _run_workers(tmp_path: Path, mode: str, *, timeout: float = 30.0):
context = mp.get_context("spawn")
queue = context.Queue()
init_file = tmp_path / f"{mode}.rendezvous"
processes = [
context.Process(target=_worker, args=(rank, str(init_file), mode, queue))
for rank in range(WORLD_SIZE)
]
for process in processes:
process.start()
for process in processes:
process.join(timeout)
if process.is_alive():
process.terminate()
process.join(5)
pytest.fail(f"{mode} worker {process.pid} did not terminate")
assert process.exitcode == 0
results = []
for _ in range(WORLD_SIZE):
try:
results.append(queue.get(timeout=2))
except Empty:
pytest.fail(f"{mode} did not report one result per rank")
queue.close()
queue.join_thread()
return sorted(results)
@pytest.fixture(scope="module")
def success_results(tmp_path_factory):
return _run_workers(
tmp_path_factory.mktemp("distributed-success"), "success_matrix", timeout=20.0
)
def _case_results(success_results, key: str):
return [(rank, status, payload[key]) for rank, status, payload in success_results]
def test_bounded_records_digest_and_unanimous_vote_succeed(success_results):
results = _case_results(success_results, "records")
assert [status for _rank, status, _payload in results] == ["ok", "ok"]
expected_records = ['{"rank":0}', '{"rank":1}']
for _rank, _status, payload in results:
records, digest, vote = payload
assert records == expected_records
assert digest == hashlib.sha256(b"same").hexdigest()
assert vote is True
@pytest.mark.parametrize(
("mode", "message"),
[
("digest_mismatch", "rank digests disagree"),
("identity_mismatch", "rank digests disagree"),
("stale_sequence", "rank vote sequences disagree"),
("placement_disagreement", "rank placement metadata disagrees"),
("placement_name_disagreement", "rank digests disagree"),
],
)
def test_rank_disagreement_fails_on_every_participant(tmp_path, mode, message):
results = _run_workers(tmp_path, mode)
assert [status for _rank, status, _payload in results] == ["error", "error"]
assert all(message in payload[1] for _rank, _status, payload in results)
def test_one_negative_vote_aborts_unanimity_for_every_rank(success_results):
results = _case_results(success_results, "vote_no")
assert [payload for _rank, status, payload in results if status == "ok"] == [False, False]
@pytest.mark.parametrize("mode", ["one_rank_error", "early_exit", "hang"])
def test_rank_exit_or_timeout_terminates_and_reaps_the_worker_group(tmp_path, mode):
results = _run_workers(tmp_path, mode)
statuses = {rank: status for rank, status, _payload in results}
assert statuses[0] == "error"
assert statuses[1] in {"error", "exited"}
@pytest.mark.parametrize(
("mode", "concat_dim", "transposed"),
[
("column", 0, False),
("row", 1, False),
("transposed_column", 0, True),
("transposed_row", 1, True),
],
)
def test_distributed_shards_match_complete_tensor_projection(
success_results,
mode,
concat_dim,
transposed,
):
results = _case_results(success_results, mode)
assert all(status == "ok" for _rank, status, _payload in results)
shards = [
torch.tensor(payload["weight"], dtype=torch.float64) for _rank, _status, payload in results
]
actual = torch.cat(shards, dim=concat_dim)
full_weight, direction = _full_inputs(transposed)
expected = project_weight_against_direction(
full_weight,
direction,
norm_preserve=True,
regularization=0.2,
projection_row_fraction=0.5,
)
torch.testing.assert_close(actual, expected.weight, rtol=1e-10, atol=1e-10)
assert all(payload["projected"] is True for _rank, _status, payload in results)
assert all(payload["layout"] == expected.layout for _rank, _status, payload in results)
assert all(
payload["coefficient_norm_sq"] == pytest.approx(expected.coefficient_norm_sq)
for _rank, _status, payload in results
)
def test_replicated_projection_is_identical_on_every_rank(success_results):
results = _case_results(success_results, "replicated")
weights = [
torch.tensor(payload["weight"], dtype=torch.float64)
for _rank, status, payload in results
if status == "ok"
]
assert len(weights) == WORLD_SIZE
torch.testing.assert_close(weights[0], weights[1], rtol=0, atol=0)
full_weight, direction = _full_inputs()
expected = project_weight_against_direction(
full_weight,
direction,
norm_preserve=True,
regularization=0.2,
projection_row_fraction=0.5,
)
torch.testing.assert_close(weights[0], expected.weight, rtol=1e-10, atol=1e-10)
def test_nonfinite_value_on_one_rank_prevents_mutation_everywhere(success_results):
results = _case_results(success_results, "nonfinite")
assert all(status == "ok" for _rank, status, _payload in results)
assert all(payload["projected"] is False for _rank, _status, payload in results)
@pytest.mark.parametrize("mode", ["zero_direction", "zero_weight"])
def test_degenerate_global_inputs_are_deterministic(success_results, mode):
results = _case_results(success_results, mode)
assert all(status == "ok" for _rank, status, _payload in results)
expected_projected = mode == "zero_weight"
assert all(payload["projected"] is expected_projected for _rank, _status, payload in results)
def test_projection_without_norm_preservation_reports_no_global_norm(success_results):
results = _case_results(success_results, "column_no_norm")
assert all(status == "ok" for _rank, status, _payload in results)
actual = torch.cat(
[
torch.tensor(payload["weight"], dtype=torch.float64)
for _rank, _status, payload in results
],
dim=0,
)
full_weight, direction = _full_inputs()
expected = project_weight_against_direction(
full_weight,
direction,
norm_preserve=False,
regularization=0.2,
projection_row_fraction=0.5,
)
torch.testing.assert_close(actual, expected.weight, rtol=1e-10, atol=1e-10)
assert all(payload["coefficient_norm_sq"] == 0.0 for _rank, _status, payload in results)
def test_invalid_numerical_requests_fail_closed_on_both_ranks(success_results):
results = _case_results(success_results, "validation")
assert [status for _rank, status, _payload in results] == ["ok", "ok"]
assert all(len(payload) == 11 for _rank, _status, payload in results)
def test_real_gloo_preflight_admits_the_complete_fixed_world(tmp_path):
results = _run_workers(tmp_path, "preflight_success")
assert [status for _rank, status, _payload in results] == ["ok", "ok"]
assert {payload["accepted"] for _rank, _status, payload in results} == {2}
assert len({payload["identity_digest"] for _rank, _status, payload in results}) == 1
assert {payload["backend"] for _rank, _status, payload in results} == {"gloo"}
def test_execute_preflight_publishes_success_only_after_all_teardown_acknowledgements(
tmp_path,
):
results = _run_execute_workers(tmp_path, "success")
assert [status for _rank, status, _payload in results] == ["ok", "ok"]
assert {payload for _rank, _status, payload in results} == {("preflighted", None)}
attempt = tmp_path / "staging" / ("1" * 32)
prepared = read_stage_message(attempt / ".preflight.prepared.stage.json")
assert prepared.stage is RuntimeStage.PREFLIGHTED
assert prepared.vote is Vote.PREPARED
assert not (attempt / ".preflight.prepared.json").exists()
acknowledgements = tuple(
read_stage_message(attempt / f".rank-{rank}.teardown.stage.json")
for rank in range(WORLD_SIZE)
)
assert {item.rank for item in acknowledgements} == {0, 1}
assert {item.vote for item in acknowledgements} == {Vote.COMMITTED}
@pytest.mark.parametrize(
("mode", "code"),
[
("identity_failure", "LMS_IDENTITY_MISMATCH"),
("commit_failure", "LMS_IDENTITY_MISMATCH"),
("code_failure", "LMS_IDENTITY_MISMATCH"),
("storage_failure", "LMS_STORAGE_PROFILE_MISMATCH"),
("resource_failure", "LMS_RESOURCE_ADMISSION_DENIED"),
("stage_timeout", "LMS_STAGE_TIMEOUT"),
("cancelled", "LMS_ATTEMPT_CANCELLED"),
("native_stderr", "LMS_DIAGNOSTIC_REDACTION_FAILED"),
("evidence_failure", "LMS_EVIDENCE_UNAVAILABLE"),
("teardown_failure", "LMS_CLEANUP_INCOMPLETE"),
],
)
def test_execute_preflight_faults_never_report_success_and_reap_workers(
tmp_path, mode, code, capfd
):
results = _run_execute_workers(tmp_path, mode)
assert [status for _rank, status, _payload in results] == ["error", "error"]
assert {payload[0] for _rank, _status, payload in results} == {code}
assert all("secret-bearing" not in payload[1] for _rank, _status, payload in results)
assert "secret-bearing native child" not in capfd.readouterr().err
if mode != "teardown_failure":
attempt = tmp_path / "staging" / ("1" * 32)
aborting = tuple(
read_stage_message(attempt / f".rank-{rank}.aborting.stage.json")
for rank in range(WORLD_SIZE)
)
terminal = tuple(
read_stage_message(attempt / f".rank-{rank}.terminal.stage.json")
for rank in range(WORLD_SIZE)
)
assert {item.stage for item in aborting} == {RuntimeStage.ABORTING}
assert {item.stage for item in terminal} == {RuntimeStage.ABORTED}
assert {item.vote for item in (*aborting, *terminal)} == {Vote.ABORT}
@pytest.mark.parametrize(
("mode", "ranks", "exit_codes"),
[
("missing_rank", (0,), (0,)),
("execute_early_exit", (0, 1), (0, 17)),
],
)
def test_execute_preflight_missing_or_exited_rank_quarantines_and_reaps(
tmp_path, mode, ranks, exit_codes, capfd
):
results, observed_exit_codes = _run_execute_disruption(tmp_path, mode, ranks=ranks)
assert observed_exit_codes == exit_codes
assert len(results) == 1
assert results[0][1] == "error"
assert results[0][2][0] == "LMS_CLEANUP_INCOMPLETE"
evidence = read_evidence(tmp_path / "staging" / ("1" * 32) / "preflight.json")
assert evidence.result == "quarantined"
assert evidence.error_code == "LMS_CLEANUP_INCOMPLETE"
assert "secret-bearing" not in capfd.readouterr().err
@pytest.mark.parametrize("mode", ["rank_exception", "execute_hang"])
def test_execute_preflight_rank_error_or_hang_is_bounded_and_never_succeeds(tmp_path, mode, capfd):
started = time.monotonic()
results = _run_execute_workers(tmp_path, mode, timeout=20)
elapsed = time.monotonic() - started
assert [status for _rank, status, _payload in results] == ["error", "error"]
evidence = read_evidence(tmp_path / "staging" / ("1" * 32) / "preflight.json")
assert evidence.result != "preflighted"
assert evidence.error_code is not None
assert "secret-bearing" not in capfd.readouterr().err
if mode == "execute_hang":
assert elapsed < 20
tests/test_distributed_launcher.py
+392
View File
@@ -0,0 +1,392 @@
"""Fixed-membership launcher and configuration contracts for issue 59."""
from __future__ import annotations
import json
import multiprocessing as mp
import os
import time
from dataclasses import replace
import pytest
from obliteratus.distributed.config import DistributedPreflightConfig
from obliteratus.distributed.contracts import ContractError, RuntimeContractError
from obliteratus.distributed.launcher import (
TorchrunEnvironment,
control_group,
)
HEX = "a" * 64
def _teardown_overrun_worker(config, launch, sink_path: str, ready) -> None:
import obliteratus.distributed.launcher as launcher_module
import torch.distributed as child_dist
descriptor = os.open(sink_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
os.dup2(descriptor, 2)
os.close(descriptor)
state = {"initialized": False}
launcher_module.validate_network_interface = lambda *args, **kwargs: None
child_dist.is_available = lambda: True # type: ignore[method-assign]
child_dist.is_initialized = lambda: state["initialized"] # type: ignore[method-assign]
child_dist.init_process_group = ( # type: ignore[method-assign]
lambda *args, **kwargs: state.update(initialized=True)
)
def delayed_native_write() -> None:
time.sleep(config.teardown_timeout_seconds + 0.25)
os.write(2, b"LATE_NATIVE_MARKER_FROM_TIMED_OUT_TEARDOWN\n")
child_dist.destroy_process_group = delayed_native_write # type: ignore[method-assign]
ready.set()
with control_group(config, launch):
pass
os._exit(99)
def _profile(tmp_path, **overrides):
source = tmp_path / "source"
staging = tmp_path / "staging"
evidence = staging / ("1" * 32) / "preflight.json"
source.mkdir(exist_ok=True)
staging.mkdir(exist_ok=True)
payload = {
"schema_version": 1,
"run": {
"run_id": "1" * 32,
"rendezvous_id": "2" * 32,
"world_size": 2,
"local_world_size": 1,
},
"identity": {
"source_digest": HEX,
"model_digest": "b" * 64,
"tokenizer_digest": "c" * 64,
"commit_sha": "d" * 40,
"code_digest": "0" * 64,
},
"topology": {
"tensor_parallel_size": 2,
"coordinator_rank": 0,
"placement_plan_digest": "e" * 64,
"dimension_divisors": [2, 4],
},
"network": {
"master_addr": "10.10.0.10",
"master_port": 29500,
"interface": "eth0",
"allowed_master_cidrs": ["10.10.0.0/24"],
},
"source": {"path": str(source)},
"staging": {"path": str(staging), "storage_digest": "f" * 64},
"resources": {
"min_free_device_memory_bytes": 1024,
"min_free_host_memory_bytes": 2048,
"min_free_staging_bytes": 4096,
"max_source_files": 1000,
"max_source_bytes": 1099511627776,
"max_source_file_bytes": 1099511627776,
},
"timeouts": {
"source_seconds": 5,
"init_seconds": 5,
"collective_seconds": 5,
"teardown_seconds": 5,
},
"software": {
"python": "3.12.11",
"platform": "Linux-test",
"machine": "x86_64",
"torch": "2.13.0",
"transformers": "5.15.0",
"accelerate": "1.10.0",
"safetensors": "0.6.2",
"cuda": "13.0",
"nccl": "2.28.3",
"driver": "580.65",
},
"execution": {
"device_kind": "cuda",
"device_name": "NVIDIA Test GPU",
"compute_capability": "10.0",
"evidence_tier": "candidate_preflight",
"allowed_environment_keys": [],
"local_files_only": True,
"trust_remote_code": False,
"allow_runtime_install": False,
"allow_plugins": False,
"allow_compilation": False,
"allow_adapters": False,
"allow_quantization": False,
},
"evidence": {"path": str(evidence)},
}
payload.update(overrides)
path = tmp_path / "profile.json"
path.write_text(json.dumps(payload), encoding="utf-8")
return DistributedPreflightConfig.from_file(path)
def _environment(**overrides):
values = {
"RANK": "0",
"LOCAL_RANK": "0",
"WORLD_SIZE": "2",
"LOCAL_WORLD_SIZE": "1",
"GROUP_RANK": "0",
"ROLE_RANK": "0",
"ROLE_WORLD_SIZE": "2",
"MASTER_ADDR": "10.10.0.10",
"MASTER_PORT": "29500",
"TORCHELASTIC_RUN_ID": "2" * 32,
"TORCHELASTIC_RESTART_COUNT": "0",
"TORCHELASTIC_MAX_RESTARTS": "0",
"OBLITERATUS_RUN_ID": "1" * 32,
"GLOO_SOCKET_IFNAME": "eth0",
"NCCL_SOCKET_IFNAME": "eth0",
}
values.update(overrides)
return values
def test_profile_is_strict_bounded_and_order_independent(tmp_path):
config = _profile(tmp_path)
reordered = json.loads((tmp_path / "profile.json").read_text(encoding="utf-8"))
(tmp_path / "profile.json").write_text(
json.dumps(dict(reversed(list(reordered.items())))), encoding="utf-8"
)
assert DistributedPreflightConfig.from_file(tmp_path / "profile.json").digest == config.digest
reordered["password"] = "do-not-echo"
(tmp_path / "profile.json").write_text(json.dumps(reordered), encoding="utf-8")
with pytest.raises(ContractError, match="unknown profile field"):
DistributedPreflightConfig.from_file(tmp_path / "profile.json")
def test_profile_byte_bound_is_checked_before_json_parsing(tmp_path):
profile = tmp_path / "profile.json"
profile.write_bytes(b" " * (64 * 1024 + 1))
with pytest.raises(ContractError, match="regular file"):
DistributedPreflightConfig.from_file(profile)
@pytest.mark.parametrize(
"field",
[
"RANK",
"LOCAL_RANK",
"WORLD_SIZE",
"LOCAL_WORLD_SIZE",
"GROUP_RANK",
"ROLE_RANK",
"ROLE_WORLD_SIZE",
"MASTER_ADDR",
"MASTER_PORT",
"TORCHELASTIC_RUN_ID",
"TORCHELASTIC_RESTART_COUNT",
"TORCHELASTIC_MAX_RESTARTS",
"OBLITERATUS_RUN_ID",
"GLOO_SOCKET_IFNAME",
"NCCL_SOCKET_IFNAME",
],
)
def test_torchrun_environment_requires_every_fixed_field(tmp_path, field):
config = _profile(tmp_path)
environ = _environment()
del environ[field]
with pytest.raises(ContractError, match="required torchrun environment is incomplete"):
TorchrunEnvironment.from_environ(environ, config)
@pytest.mark.parametrize(
("overrides", "message"),
[
({"RANK": "2"}, "rank must be smaller"),
({"LOCAL_RANK": "1"}, "local_rank must be smaller"),
({"WORLD_SIZE": "3"}, "world_size disagrees"),
({"ROLE_WORLD_SIZE": "1"}, "role_world_size disagrees"),
({"MASTER_PORT": "0"}, "master_port"),
({"MASTER_ADDR": "8.8.8.8"}, "master endpoint disagrees"),
({"MASTER_ADDR": "0.0.0.0"}, "master endpoint disagrees"),
({"TORCHELASTIC_RESTART_COUNT": "1"}, "restarts are forbidden"),
({"TORCHELASTIC_MAX_RESTARTS": "1"}, "restarts are forbidden"),
({"TORCHELASTIC_RUN_ID": "3" * 32}, "rendezvous_id disagrees"),
({"OBLITERATUS_RUN_ID": "3" * 32}, "run_id disagrees"),
({"GLOO_SOCKET_IFNAME": "eth1"}, "interface disagrees"),
({"RANK": "9" * 10000}, "unsigned decimal integer"),
],
)
def test_torchrun_environment_rejects_dynamic_or_unapproved_membership(
tmp_path, overrides, message
):
with pytest.raises(ContractError, match=message):
TorchrunEnvironment.from_environ(_environment(**overrides), _profile(tmp_path))
def test_torchrun_environment_parses_immutable_identity(tmp_path):
launch = TorchrunEnvironment.from_environ(_environment(), _profile(tmp_path))
assert (launch.rank, launch.local_rank, launch.world_size) == (0, 0, 2)
assert launch.master_endpoint_digest != "10.10.0.10"
with pytest.raises(Exception):
launch.rank = 1 # type: ignore[misc]
def test_control_group_always_destroys_and_uses_explicit_timeout(tmp_path, monkeypatch):
config = _profile(tmp_path)
launch = TorchrunEnvironment.from_environ(_environment(), config)
calls = []
monkeypatch.setattr(
"obliteratus.distributed.launcher.validate_network_interface",
lambda *args, **kwargs: None,
)
monkeypatch.setattr("torch.distributed.is_available", lambda: True)
monkeypatch.setattr("torch.distributed.is_initialized", lambda: bool(calls))
monkeypatch.setattr("torch.distributed.init_process_group", lambda *a, **kw: calls.append(kw))
monkeypatch.setattr("torch.distributed.destroy_process_group", lambda: calls.append("destroy"))
with pytest.raises(RuntimeError, match="injected"):
with control_group(config, launch):
raise RuntimeError("injected")
assert calls[0]["backend"] == "gloo"
assert calls[0]["rank"] == 0
assert calls[0]["world_size"] == 2
assert calls[0]["timeout"].total_seconds() == 5
assert calls[-1] == "destroy"
def test_control_group_teardown_failure_replaces_success(tmp_path, monkeypatch):
config = _profile(tmp_path)
launch = TorchrunEnvironment.from_environ(_environment(), config)
state = {"initialized": False}
monkeypatch.setattr(
"obliteratus.distributed.launcher.validate_network_interface",
lambda *args, **kwargs: None,
)
monkeypatch.setattr("torch.distributed.is_available", lambda: True)
monkeypatch.setattr("torch.distributed.is_initialized", lambda: state["initialized"])
monkeypatch.setattr(
"torch.distributed.init_process_group",
lambda *args, **kwargs: state.update(initialized=True),
)
def fail_destroy():
raise RuntimeError("secret-bearing backend diagnostic")
monkeypatch.setattr("torch.distributed.destroy_process_group", fail_destroy)
with pytest.raises(ContractError, match="control group teardown failed"):
with control_group(config, launch):
pass
def test_control_group_refuses_unbound_interface_before_backend_init(tmp_path, monkeypatch):
config = _profile(tmp_path)
launch = TorchrunEnvironment.from_environ(_environment(), config)
calls = []
monkeypatch.setattr("torch.distributed.is_available", lambda: True)
monkeypatch.setattr("torch.distributed.is_initialized", lambda: False)
monkeypatch.setattr(
"torch.distributed.init_process_group", lambda *args, **kwargs: calls.append(kwargs)
)
def refuse(*args, **kwargs):
raise RuntimeContractError(
"LMS_NETWORK_PROFILE_DENIED", "configured interface is unavailable"
)
monkeypatch.setattr("obliteratus.distributed.launcher.validate_network_interface", refuse)
with pytest.raises(RuntimeContractError) as error:
with control_group(config, launch):
pass
assert error.value.code == "LMS_NETWORK_PROFILE_DENIED"
assert calls == []
@pytest.mark.parametrize("phase", ["init", "collective", "timeout", "teardown"])
def test_control_group_suppresses_native_fd2_and_emits_only_stable_code(
tmp_path, monkeypatch, capfd, phase
):
config = _profile(tmp_path)
launch = TorchrunEnvironment.from_environ(_environment(), config)
state = {"initialized": False}
monkeypatch.setattr(
"obliteratus.distributed.launcher.validate_network_interface",
lambda *args, **kwargs: None,
)
monkeypatch.setattr("torch.distributed.is_available", lambda: True)
monkeypatch.setattr("torch.distributed.is_initialized", lambda: state["initialized"])
def initialize(*args, **kwargs):
if phase in {"init", "timeout"}:
os.write(2, b"secret-bearing native backend diagnostic\n")
if phase == "timeout":
raise TimeoutError("raw private endpoint")
state["initialized"] = True
def destroy():
if phase == "teardown":
os.write(2, b"secret-bearing native backend diagnostic\n")
state["initialized"] = False
monkeypatch.setattr("torch.distributed.init_process_group", initialize)
monkeypatch.setattr("torch.distributed.destroy_process_group", destroy)
with pytest.raises(RuntimeContractError) as error:
with control_group(config, launch):
if phase == "collective":
os.write(2, b"secret-bearing native backend diagnostic\n")
assert error.value.code == "LMS_DIAGNOSTIC_REDACTION_FAILED"
assert "secret-bearing" not in str(error.value)
assert "secret-bearing" not in capfd.readouterr().err
def test_teardown_overrun_terminates_worker_before_late_native_write(tmp_path):
config = replace(_profile(tmp_path), teardown_timeout_seconds=1)
launch = TorchrunEnvironment.from_environ(_environment(), config)
sink = tmp_path / "worker-stderr.bin"
context = mp.get_context("spawn")
ready = context.Event()
process = context.Process(
target=_teardown_overrun_worker,
args=(config, launch, str(sink), ready),
)
process.start()
assert ready.wait(20), "teardown-overrun worker did not finish cold startup"
started = time.monotonic()
process.join(3)
elapsed = time.monotonic() - started
if process.is_alive():
process.terminate()
process.join(2)
pytest.fail("teardown-overrun worker did not terminate within its bound")
assert process.exitcode == 70
assert elapsed < 3
assert sink.read_bytes() == b""
def test_config_rejects_weakened_execution_policy(tmp_path):
config = _profile(tmp_path)
with pytest.raises(ContractError, match="trust_remote_code must remain false"):
replace(config, trust_remote_code=True).validate()
@pytest.mark.parametrize("name", ["HF_TOKEN", "AWS_ACCESS_KEY_ID", "HTTPS_PROXY"])
def test_secret_or_proxy_environment_is_rejected_without_echo(tmp_path, name):
environ = _environment()
environ[name] = "private-value"
with pytest.raises(ContractError, match="secret-bearing") as error:
TorchrunEnvironment.from_environ(environ, _profile(tmp_path))
assert "private-value" not in str(error.value)
@pytest.mark.parametrize("address", ["8.8.8.8", "0.0.0.0", "224.0.0.1"])
def test_profiled_endpoint_must_still_be_private_and_allowlisted(tmp_path, address):
network = {
"master_addr": address,
"master_port": 29500,
"interface": "eth0",
"allowed_master_cidrs": [f"{address}/32"],
}
with pytest.raises(ContractError, match="private"):
config = _profile(tmp_path, network=network)
TorchrunEnvironment.from_environ(_environment(MASTER_ADDR=address), config)
tests/test_distributed_preflight.py
+383
View File
@@ -0,0 +1,383 @@
"""Admission and source-safety tests for the distributed preflight."""
from __future__ import annotations
import json
import socket
import time
from dataclasses import replace
from types import SimpleNamespace
import pytest
from obliteratus.distributed.contracts import (
ContractError,
RuntimeContractError,
RuntimeStage,
StageMessage,
canonical_record,
contract_digest,
)
from obliteratus.distributed.launcher import validate_network_interface
from obliteratus.distributed.preflight import (
_validate_stage_messages,
RankAttestation,
checkout_code_digest,
checkout_commit,
inspect_source,
storage_mount_digest,
validate_attestations,
)
def _attestation(rank: int = 0, **overrides) -> RankAttestation:
values = {
"rank": rank,
"local_rank": 0,
"local_world_size": 1,
"group_rank": rank,
"world_size": 2,
"host_digest": ("1" if rank == 0 else "2") * 64,
"device_digest": ("3" if rank == 0 else "4") * 64,
"device_profile_digest": "0" * 64,
"device_config_digest": contract_digest(
{"kind": "cuda", "name": "test-device", "compute_capability": "10.0"}
),
"device_kind": "cuda",
"total_device_memory_bytes": 8192,
"free_device_memory_bytes": 4096,
"total_host_memory_bytes": 16384,
"free_host_memory_bytes": 8192,
"free_staging_bytes": 8192,
"software_digest": "5" * 64,
"storage_digest": "6" * 64,
"source_digest": "7" * 64,
"model_digest": "8" * 64,
"tokenizer_digest": "9" * 64,
"config_digest": "a" * 64,
"commit_sha": "b" * 40,
"code_digest": "f" * 64,
"placement_plan_digest": "c" * 64,
"network_interface_digest": "d" * 64,
}
values.update(overrides)
return RankAttestation(**values)
def _validate(records):
validate_attestations(
tuple(records),
world_size=2,
tensor_parallel_size=2,
dimension_divisors=(2, 4),
expected_device_kind="cuda",
expected_device_config_digest=contract_digest(
{"kind": "cuda", "name": "test-device", "compute_capability": "10.0"}
),
expected_software_digest="5" * 64,
expected_source_digest="7" * 64,
expected_model_digest="8" * 64,
expected_tokenizer_digest="9" * 64,
expected_config_digest="a" * 64,
expected_commit_sha="b" * 40,
expected_code_digest="f" * 64,
expected_placement_plan_digest="c" * 64,
expected_storage_digest="6" * 64,
expected_network_interface_digest="d" * 64,
local_world_size=1,
min_free_device_memory_bytes=4096,
min_free_host_memory_bytes=8192,
min_free_staging_bytes=8192,
)
def test_complete_fixed_inventory_passes_exact_boundaries():
_validate((_attestation(0), _attestation(1)))
def test_rank_attestation_decoder_requires_exact_canonical_bytes():
record = _attestation()
assert RankAttestation.from_bytes(canonical_record(record)) == record
with pytest.raises(ContractError, match="not canonical"):
RankAttestation.from_bytes(b" " + canonical_record(record))
duplicate = canonical_record(record).replace(b'{"code_digest":', b'{"rank":0,"code_digest":', 1)
with pytest.raises(ContractError, match="duplicate"):
RankAttestation.from_bytes(duplicate)
def test_lifecycle_validation_rejects_a_record_bound_to_another_run():
identity_digest = "a" * 64
records = (
StageMessage("1" * 32, identity_digest, 0, 0, RuntimeStage.CREATED),
StageMessage("2" * 32, identity_digest, 1, 0, RuntimeStage.CREATED),
)
with pytest.raises(RuntimeContractError) as error:
_validate_stage_messages(
records,
run_id="1" * 32,
world_size=2,
stage=RuntimeStage.CREATED,
sequence=0,
identity_digest=identity_digest,
vote=None,
)
assert error.value.code == "LMS_LIFECYCLE_INVALID"
@pytest.mark.parametrize(
("records", "message"),
[
((_attestation(0),), "exactly one attestation"),
((_attestation(0), _attestation(0)), "rank order"),
(
(_attestation(0), replace(_attestation(1), device_digest="3" * 64)),
"device identities",
),
(
(_attestation(0), replace(_attestation(1), device_profile_digest="e" * 64)),
"device profiles",
),
(
(_attestation(0), replace(_attestation(1), local_rank=0, host_digest="1" * 64)),
"local ranks",
),
(
(_attestation(0), replace(_attestation(1), software_digest="e" * 64)),
"software identities",
),
(
(_attestation(0), replace(_attestation(1), storage_digest="e" * 64)),
"storage identities",
),
(
(_attestation(0), replace(_attestation(1), source_digest="e" * 64)),
"source_digest",
),
(
(_attestation(0), replace(_attestation(1), free_device_memory_bytes=4095)),
"device memory headroom",
),
(
(_attestation(0), replace(_attestation(1), free_host_memory_bytes=8191)),
"host memory headroom",
),
(
(_attestation(0), replace(_attestation(1), free_staging_bytes=8191)),
"staging headroom",
),
],
)
def test_inventory_disagreement_fails_closed(records, message):
with pytest.raises(ContractError, match=message):
_validate(records)
def test_topology_dimensions_must_be_divisible():
with pytest.raises(ContractError, match="dimension divisor"):
validate_attestations(
(_attestation(0), _attestation(1)),
world_size=2,
tensor_parallel_size=2,
dimension_divisors=(3,),
expected_device_kind="cuda",
expected_device_config_digest=contract_digest(
{"kind": "cuda", "name": "test-device", "compute_capability": "10.0"}
),
expected_software_digest="5" * 64,
expected_source_digest="7" * 64,
expected_model_digest="8" * 64,
expected_tokenizer_digest="9" * 64,
expected_config_digest="a" * 64,
expected_commit_sha="b" * 40,
expected_code_digest="f" * 64,
expected_placement_plan_digest="c" * 64,
expected_storage_digest="6" * 64,
expected_network_interface_digest="d" * 64,
local_world_size=1,
min_free_device_memory_bytes=1,
min_free_host_memory_bytes=1,
min_free_staging_bytes=1,
)
def test_source_inspection_accepts_only_immutable_local_safetensors(tmp_path):
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
tokenizer = source / "tokenizer.json"
header = json.dumps(
{"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}},
separators=(",", ":"),
).encode()
weights.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4)
tokenizer.write_text('{"model":"fixture"}', encoding="utf-8")
weights.chmod(0o444)
tokenizer.chmod(0o444)
source.chmod(0o555)
first = inspect_source(source)
second = inspect_source(source)
assert first == second
assert first.file_count == 2
assert len({first.source_digest, first.model_digest, first.tokenizer_digest}) == 3
def test_source_inspection_rejects_arbitrary_bytes_with_safetensors_suffix(tmp_path):
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
tokenizer = source / "tokenizer.json"
weights.write_bytes(b"not-a-safetensors-file")
tokenizer.write_text("{}", encoding="utf-8")
weights.chmod(0o444)
tokenizer.chmod(0o444)
source.chmod(0o555)
with pytest.raises(ContractError, match="safe-structure inspection"):
inspect_source(source)
def test_source_inspection_rejects_executable_serialization(tmp_path):
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
pickle_file = source / "pytorch_model.bin"
weights.write_bytes(b"safe")
pickle_file.write_bytes(b"not-executed")
weights.chmod(0o444)
pickle_file.chmod(0o444)
source.chmod(0o555)
with pytest.raises(ContractError, match="outside the safetensors envelope"):
inspect_source(source)
def test_source_inspection_rejects_symlinked_directories(tmp_path):
outside = tmp_path / "outside"
outside.mkdir()
(outside / "tokenizer.json").write_text("{}", encoding="utf-8")
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
weights.write_bytes(b"safe")
(source / "linked").symlink_to(outside, target_is_directory=True)
weights.chmod(0o444)
source.chmod(0o555)
with pytest.raises(RuntimeContractError, match="symbolic links") as error:
inspect_source(source)
assert error.value.code == "LMS_SOURCE_BOUNDARY_VIOLATION"
def test_source_inspection_enforces_byte_bounds_before_hashing(tmp_path):
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
tokenizer = source / "tokenizer.json"
weights.write_bytes(b"12345")
tokenizer.write_bytes(b"{}")
weights.chmod(0o444)
tokenizer.chmod(0o444)
source.chmod(0o555)
with pytest.raises(RuntimeContractError, match="configured byte bound") as error:
inspect_source(source, max_file_bytes=4, max_total_bytes=10)
assert error.value.code == "LMS_RESOURCE_ADMISSION_DENIED"
def test_source_inspection_enforces_exact_file_and_total_bounds(tmp_path):
source = tmp_path / "source"
source.mkdir()
header = json.dumps(
{"weight": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}},
separators=(",", ":"),
).encode()
weights = source / "model.safetensors"
tokenizer = source / "tokenizer.json"
weights.write_bytes(len(header).to_bytes(8, "little") + header + b"\0" * 4)
tokenizer.write_text("{}", encoding="utf-8")
total = weights.stat().st_size + tokenizer.stat().st_size
weights.chmod(0o444)
tokenizer.chmod(0o444)
source.chmod(0o555)
assert (
inspect_source(source, max_files=2, max_total_bytes=total, max_file_bytes=total).file_count
== 2
)
with pytest.raises(ContractError, match="file count"):
inspect_source(source, max_files=1, max_total_bytes=total, max_file_bytes=total)
with pytest.raises(ContractError, match="total-byte"):
inspect_source(
source,
max_files=2,
max_total_bytes=total - 1,
max_file_bytes=total - 1,
)
def test_source_inspection_timeout_is_deterministic_before_io(tmp_path, monkeypatch):
source = tmp_path / "source"
source.mkdir()
source.chmod(0o555)
moments = iter((0.0, 0.0, 2.0))
monkeypatch.setattr("obliteratus.distributed.preflight.time.monotonic", lambda: next(moments))
with pytest.raises(RuntimeContractError, match="explicit timeout") as error:
inspect_source(source, timeout_seconds=1)
assert error.value.code == "LMS_STAGE_TIMEOUT"
def test_source_inspection_hard_deadline_interrupts_the_structural_inspector(tmp_path, monkeypatch):
source = tmp_path / "source"
source.mkdir()
weights = source / "model.safetensors"
tokenizer = source / "tokenizer.json"
weights.write_bytes(b"bounded")
tokenizer.write_text("{}", encoding="utf-8")
weights.chmod(0o444)
tokenizer.chmod(0o444)
source.chmod(0o555)
def block(*args, **kwargs):
time.sleep(5)
raise AssertionError("deadline did not interrupt the inspector")
monkeypatch.setattr("obliteratus.distributed.preflight.inspect_checkpoint", block)
started = time.monotonic()
with pytest.raises(RuntimeContractError) as error:
inspect_source(source, timeout_seconds=1)
assert error.value.code == "LMS_STAGE_TIMEOUT"
assert time.monotonic() - started < 2
def test_checkout_commit_resolves_a_worktree_reference_without_a_child_process(tmp_path):
checkout = tmp_path / "checkout"
git_dir = tmp_path / "common" / "worktrees" / "candidate"
common = tmp_path / "common"
reference = common / "refs" / "heads" / "candidate"
checkout.mkdir()
git_dir.mkdir(parents=True)
reference.parent.mkdir(parents=True)
(checkout / ".git").write_text(f"gitdir: {git_dir}\n", encoding="utf-8")
(git_dir / "HEAD").write_text("ref: refs/heads/candidate\n", encoding="utf-8")
(git_dir / "commondir").write_text("../..\n", encoding="utf-8")
reference.write_text("a" * 40 + "\n", encoding="utf-8")
assert checkout_commit(checkout) == "a" * 40
def test_checkout_code_digest_changes_with_executable_source(tmp_path):
package = tmp_path / "obliteratus"
package.mkdir()
module = package / "module.py"
module.write_text("VALUE = 1\n", encoding="utf-8")
first = checkout_code_digest(tmp_path)
module.write_text("VALUE = 2\n", encoding="utf-8")
assert checkout_code_digest(tmp_path) != first
def test_storage_mount_digest_is_measured_and_stable(tmp_path):
assert storage_mount_digest(tmp_path) == storage_mount_digest(tmp_path)
def test_network_interface_requires_allowlisted_address_and_coordinator_binding(
monkeypatch,
):
addresses = {"eth0": [SimpleNamespace(family=socket.AF_INET, address="10.10.0.10")]}
monkeypatch.setattr("psutil.net_if_addrs", lambda: addresses)
validate_network_interface("eth0", ("10.10.0.0/24",), "10.10.0.10", coordinator=True)
with pytest.raises(ContractError, match="not bound"):
validate_network_interface("eth0", ("10.10.0.0/24",), "10.10.0.11", coordinator=True)
tests/test_package_export_contracts.py
+1
View File
@@ -28,6 +28,7 @@ import obliteratus.analysis as analysis
"Watchtower",
"get_watchtower",
"AutoObliterator",
"CheckpointService",
],
)
def test_documented_lazy_export_resolves(name):
tests/test_peft_artifacts.py
+374
View File
@@ -0,0 +1,374 @@
"""Canonical PEFT LoRA export and truthful legacy-format contracts."""
from __future__ import annotations
import json
from dataclasses import replace
from hashlib import sha256
from pathlib import Path
import pytest
import torch
from jsonschema import Draft202012Validator
from safetensors.torch import load_file
from obliteratus.checkpoint_provenance import LineageEvent, ToolIdentity, build_provenance
from obliteratus.lora_ablation import (
BaseModelIdentity,
load_lora_adapters,
save_legacy_pickle_adapters_trusted,
save_lora_adapters,
save_unsupported_obliteratus_adapters,
validate_adapter_base,
)
DIGEST_A = "sha256:" + "a" * 64
DIGEST_B = "sha256:" + "b" * 64
COMMIT = "c" * 40
MANIFEST_SCHEMA = json.loads(
(
Path(__file__).resolve().parents[1]
/ "docs/checkpoints/schemas/peft-adapter-manifest-v1.schema.json"
).read_text()
)
def _base() -> BaseModelIdentity:
return BaseModelIdentity(
repo_id="org/exact-base",
revision="d" * 40,
weights_digest=DIGEST_A,
tokenizer_digest=DIGEST_B,
vocab_size=32000,
architecture="TinyForCausalLM",
tied_embeddings=True,
)
def _adapters():
return {
"model.layers.0.self_attn.q_proj": (
torch.arange(8, dtype=torch.float32).reshape(4, 2),
torch.arange(6, dtype=torch.float32).reshape(2, 3),
),
"model.layers.1.mlp.down_proj": (
torch.arange(10, dtype=torch.float32).reshape(5, 2),
torch.arange(8, dtype=torch.float32).reshape(2, 4),
),
}
def _factory(base: BaseModelIdentity):
def create(output_digests, adapter_identity):
return build_provenance(
sources=(base.to_artifact_identity(),),
converter=ToolIdentity("obliteratus-peft-export", "1.0.0", COMMIT),
obliteratus_commit=COMMIT,
configuration_digest=adapter_identity.config_digest,
tokenizer=base.tokenizer_artifact_identity(),
base_model=base.to_artifact_identity(),
command=("adapter", "export", base.repo_id),
environment={"python": "test", "platform": "cpu", "packages": {}},
source_topology={"world_size": 1},
lineage=(
LineageEvent(
"event-surgery",
"surgery",
(),
"obliteratus-peft-export@1.0.0",
("refusal_direction_ablation",),
),
),
input_digests=(base.weights_digest,),
output_digests=output_digests,
transformations=("lora_adapter_export", "surgery"),
observed_scopes=("adapter_weights",),
lost_state=("optimizer_state", "scheduler_state"),
adapter=adapter_identity,
training=None,
unknowns=("training_dataset",),
)
return create
def _tree(root):
return {
path.name: sha256(path.read_bytes()).hexdigest()
for path in sorted(item for item in root.iterdir() if item.is_file())
}
def test_canonical_export_is_deterministic_standard_named_and_fully_identified(tmp_path):
first = tmp_path / "first"
second = tmp_path / "second"
base = _base()
first_artifact = save_lora_adapters(
_adapters(),
first,
base_model=base,
provenance_factory=_factory(base),
lora_alpha=4,
)
second_artifact = save_lora_adapters(
dict(reversed(list(_adapters().items()))),
second,
base_model=base,
provenance_factory=_factory(base),
lora_alpha=4,
)
assert _tree(first) == _tree(second)
assert first_artifact.artifact_id == second_artifact.artifact_id
assert set(_tree(first)) == {
"README.md",
"adapter_config.json",
"adapter_manifest.json",
"adapter_model.safetensors",
"checkpoint-provenance.json",
}
assert not list(first.glob("*.pt"))
config = json.loads((first / "adapter_config.json").read_text())
assert config["peft_type"] == "LORA"
assert config["base_model_name_or_path"] == base.repo_id
assert config["revision"] == base.revision
assert config["r"] == 2
assert config["lora_alpha"] == 4
assert config["target_modules"] == [
"model.layers.0.self_attn.q_proj",
"model.layers.1.mlp.down_proj",
]
manifest = json.loads((first / "adapter_manifest.json").read_text())
Draft202012Validator(MANIFEST_SCHEMA).validate(manifest)
assert manifest["base_model"] == base.to_dict()
assert manifest["scaling"] == 2.0
assert manifest["merged"] is False
assert manifest["bias"] == "none"
assert manifest["modules_to_save"] == []
assert manifest["tie_policy"] == "base_model_declared"
assert "exact-base" in (first / "README.md").read_text()
assert first_artifact.weights_path == first / "adapter_model.safetensors"
def test_saved_peft_scaling_reproduces_each_exact_internal_delta(tmp_path):
base = _base()
adapters = _adapters()
save_lora_adapters(
adapters,
tmp_path,
base_model=base,
provenance_factory=_factory(base),
lora_alpha=4,
)
state = load_file(tmp_path / "adapter_model.safetensors", device="cpu")
manifest = json.loads((tmp_path / "adapter_manifest.json").read_text())
for mapping in manifest["key_map"]:
original_b, original_a = adapters[mapping["module_name"]]
saved_a = state[mapping["lora_A_key"]]
saved_b = state[mapping["lora_B_key"]]
actual_delta = saved_b @ saved_a * manifest["scaling"]
assert torch.equal(actual_delta, original_b @ original_a)
loaded = load_lora_adapters(tmp_path, base_model=base)
assert set(loaded) == set(adapters)
for key in adapters:
loaded_b, loaded_a = loaded[key]
original_b, original_a = adapters[key]
assert torch.equal(loaded_b @ loaded_a, original_b @ original_a)
@pytest.mark.parametrize(
("field", "value", "detail"),
[
("repo_id", "other/base", "base_model_identity_mismatch"),
("revision", "e" * 40, "base_model_revision_mismatch"),
("weights_digest", "sha256:" + "f" * 64, "base_model_digest_mismatch"),
("tokenizer_digest", "sha256:" + "f" * 64, "tokenizer_digest_mismatch"),
("vocab_size", 32001, "vocab_size_mismatch"),
("architecture", "OtherModel", "architecture_mismatch"),
],
)
def test_wrong_base_or_tokenizer_is_rejected_before_adapter_loading(
tmp_path,
field,
value,
detail,
):
base = _base()
save_lora_adapters(
_adapters(),
tmp_path,
base_model=base,
provenance_factory=_factory(base),
)
with pytest.raises(ValueError, match=detail):
validate_adapter_base(tmp_path, replace(base, **{field: value}))
def test_missing_or_malformed_canonical_artifact_fails_before_loading(tmp_path):
base = _base()
save_lora_adapters(
_adapters(),
tmp_path,
base_model=base,
provenance_factory=_factory(base),
)
(tmp_path / "adapter_config.json").write_text("{", encoding="utf-8")
with pytest.raises(ValueError, match="adapter_config_invalid"):
load_lora_adapters(tmp_path, base_model=base)
@pytest.mark.parametrize(
"relative_path",
[
"README.md",
"adapter_model.safetensors",
"adapter_config.json",
"adapter_manifest.json",
],
)
def test_tampered_peft_artifact_fails_digest_check_before_weight_loading(
tmp_path,
monkeypatch,
relative_path,
):
import obliteratus.lora_ablation as lora_ablation
base = _base()
save_lora_adapters(
_adapters(),
tmp_path,
base_model=base,
provenance_factory=_factory(base),
)
artifact = tmp_path / relative_path
if relative_path.endswith(".json"):
record = json.loads(artifact.read_text(encoding="utf-8"))
record["tampered"] = True
artifact.write_text(json.dumps(record), encoding="utf-8")
else:
artifact.write_bytes(artifact.read_bytes() + b"tampered")
monkeypatch.setattr(
lora_ablation,
"load_file",
lambda *_args, **_kwargs: pytest.fail("weights loaded before integrity check"),
)
with pytest.raises(ValueError, match="adapter_artifact_digest_mismatch"):
validate_adapter_base(tmp_path, base)
def test_tampered_provenance_digest_fails_before_weight_loading(tmp_path, monkeypatch):
import obliteratus.lora_ablation as lora_ablation
base = _base()
save_lora_adapters(
_adapters(),
tmp_path,
base_model=base,
provenance_factory=_factory(base),
)
provenance_path = tmp_path / "checkpoint-provenance.json"
provenance = json.loads(provenance_path.read_text(encoding="utf-8"))
provenance["record_digest"] = DIGEST_A
provenance_path.write_text(json.dumps(provenance), encoding="utf-8")
monkeypatch.setattr(
lora_ablation,
"load_file",
lambda *_args, **_kwargs: pytest.fail("weights loaded before integrity check"),
)
with pytest.raises(ValueError, match="adapter_provenance_digest_mismatch"):
validate_adapter_base(tmp_path, base)
def test_provenance_failure_preserves_prior_destination_and_leaks_no_staging(tmp_path):
destination = tmp_path / "adapter"
destination.mkdir()
sentinel = destination / "prior.txt"
sentinel.write_text("prior", encoding="utf-8")
def fail_provenance(_digests, _adapter_identity):
raise RuntimeError("injected provenance failure")
with pytest.raises(FileExistsError, match="absent or empty"):
save_lora_adapters(
_adapters(),
destination,
base_model=_base(),
provenance_factory=fail_provenance,
)
assert sentinel.read_text(encoding="utf-8") == "prior"
assert not list(tmp_path.glob(".adapter.staging-*"))
empty_destination = tmp_path / "empty"
empty_destination.mkdir()
with pytest.raises(RuntimeError, match="injected provenance failure"):
save_lora_adapters(
_adapters(),
empty_destination,
base_model=_base(),
provenance_factory=fail_provenance,
)
assert empty_destination.is_dir()
assert not list(empty_destination.iterdir())
assert not list(tmp_path.glob(".empty.staging-*"))
def test_safe_loader_never_calls_torch_load(tmp_path, monkeypatch):
base = _base()
save_lora_adapters(
_adapters(),
tmp_path,
base_model=base,
provenance_factory=_factory(base),
)
monkeypatch.setattr(torch, "load", lambda *_args, **_kwargs: pytest.fail("pickle loaded"))
loaded = load_lora_adapters(tmp_path, base_model=base)
assert loaded
def test_unknown_base_uses_truthful_safe_legacy_format_not_peft_or_pickle(tmp_path):
artifact = save_unsupported_obliteratus_adapters(
_adapters(),
tmp_path,
reason="exact base digest unavailable",
)
assert artifact.name == "obliteratus_unsupported_adapter.safetensors"
assert not (tmp_path / "adapter_config.json").exists()
assert not list(tmp_path.glob("*.pt"))
record = json.loads((tmp_path / "obliteratus_unsupported_adapter.json").read_text())
assert record["support_status"] == "unsupported_legacy"
assert record["safe_serialization"] is True
assert record["peft_compatible"] is False
assert record["reason"] == "exact base digest unavailable"
for unsafe_reason in ("Bearer abcdefghijklmnop", "/private/model/path"):
with pytest.raises(ValueError, match="reason is invalid"):
save_unsupported_obliteratus_adapters(
_adapters(),
tmp_path / "unsafe",
reason=unsafe_reason,
)
def test_pickle_legacy_export_requires_an_explicit_trust_gate(tmp_path, monkeypatch):
with pytest.raises(PermissionError, match="allow_pickle"):
save_legacy_pickle_adapters_trusted(_adapters(), tmp_path, allow_pickle=False)
observed = []
monkeypatch.setattr(torch, "save", lambda state, path: observed.append((state, path)))
path = save_legacy_pickle_adapters_trusted(_adapters(), tmp_path, allow_pickle=True)
assert path.name == "obliteratus_legacy_adapter_unsafe.pt"
assert observed and observed[0][1] == path
tests/test_persistence_pipeline.py
+13 -2
View File
@@ -263,7 +263,11 @@ def test_write_local_checkpoint_strips_runtime_only_state_and_writes_metadata(
checkpoint_dir = tmp_path / "staging"
adapter_path = checkpoint_dir / "lora"
save_adapters = MagicMock(return_value=adapter_path)
monkeypatch.setattr(lora_ablation, "save_lora_adapters", save_adapters)
monkeypatch.setattr(
lora_ablation,
"save_unsupported_obliteratus_adapters",
save_adapters,
)
state_dict = {"weight": torch.ones(1)}
metadata_json = '{"schema": 1}'
checkpoint_dir.mkdir()
@@ -281,7 +285,14 @@ def test_write_local_checkpoint_strips_runtime_only_state_and_writes_metadata(
},
)
tokenizer.save_pretrained.assert_called_once_with(checkpoint_dir)
save_adapters.assert_called_once_with(pipeline._lora_adapters, checkpoint_dir)
save_adapters.assert_called_once_with(
pipeline._lora_adapters,
checkpoint_dir,
reason=(
"This run did not retain an exact base-model commit, weights digest, "
"tokenizer digest, vocabulary, and architecture identity."
),
)
assert (checkpoint_dir / "abliteration_metadata.json").read_text(
encoding="utf-8",
) == metadata_json
tests/test_quant_dequant.py
+14
View File
@@ -215,6 +215,20 @@ def test_load_json_local_missing_returns_none(tmp_path):
assert qd._load_json_from_checkpoint(str(tmp_path), "missing.json") is None
def test_load_json_remote_unexpected_download_failure_returns_none(monkeypatch):
def fail_download(*args, **kwargs):
raise OSError("offline")
monkeypatch.setattr("huggingface_hub.hf_hub_download", fail_download)
assert qd._load_json_from_checkpoint(
"org/model",
"config.json",
revision="immutable-sha",
local_files_only=True,
) is None
def test_safetensors_key_names_reads_single_file_and_index(tmp_path):
from safetensors.torch import save_file
tests/test_run_archive.py
+242 -4
View File
@@ -4,9 +4,26 @@ import hashlib
import json
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
from obliteratus.run_archive import RunArchive, _worker
from obliteratus.checkpoint_provenance import (
ArtifactIdentity,
ToolIdentity,
build_provenance,
)
from obliteratus.run_archive import (
RunArchive,
_atomic_json,
_checkpoint_metrics,
_dataset_inputs,
_option_value,
_process_start_ticks,
_redact_arguments,
_worker,
default_archive_root,
main,
)
class FakeProcess:
@@ -84,7 +101,9 @@ def test_cancel_validates_identity_then_signals_only_worker_group(tmp_path, monk
run_id = archive.launch(["org/model"], popen=lambda *_args, **_kwargs: FakeProcess(5252))
monkeypatch.setattr(archive, "_worker_matches", lambda _manifest: True)
signals = []
monkeypatch.setattr("obliteratus.run_archive.os.killpg", lambda pid, sig: signals.append((pid, sig)))
monkeypatch.setattr(
"obliteratus.run_archive.os.killpg", lambda pid, sig: signals.append((pid, sig))
)
status = archive.cancel(run_id)
@@ -314,8 +333,10 @@ def test_worker_success_writes_metrics_inventory_and_atomic_marker(tmp_path, mon
assert result["result"]["metrics"] == {"refusal_rate": 0.2, "coherence": 0.8}
assert (tmp_path / run_id / "COMPLETE").is_file()
assert lifecycle.events == [
("loading", "org/model"), ("resize", 0), ("ready", None),
("release", "complete")
("loading", "org/model"),
("resize", 0),
("ready", None),
("release", "complete"),
]
@@ -397,3 +418,220 @@ def test_restart_recovery_preserves_partial_save_and_logs(tmp_path, monkeypatch)
assert recovered["failure"]["type"] == "WorkerLost"
assert (run_dir / "checkpoint" / "partial.bin").is_file()
assert (run_dir / "run.log").read_text() == "last durable phase\n"
def test_archive_helpers_cover_fallbacks_redaction_and_atomic_directory_sync(
tmp_path,
monkeypatch,
):
monkeypatch.delenv("OBLITERATUS_RUN_ARCHIVE", raising=False)
monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state"))
assert default_archive_root() == tmp_path / "state" / "obliteratus" / "runs"
assert _redact_arguments(["model", "--api-key=value", "--revision", "safe"]) == [
"model",
"--api-key=[REDACTED]",
"--revision",
"safe",
]
assert _process_start_ticks(2**31 - 1) is None
target = tmp_path / "record.json"
monkeypatch.setattr(
"obliteratus.run_archive.os.open",
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("unsupported")),
)
_atomic_json(target, {"ok": True})
assert json.loads(target.read_text(encoding="utf-8")) == {"ok": True}
def test_manifest_operations_record_running_log_and_revisions(tmp_path):
archive = RunArchive(tmp_path)
run_id = archive.begin(["org/model"])
running = archive.mark_running(run_id, phase="loading")
archive.append_log(run_id, "first line\n")
revised = archive.record_revisions(
run_id,
model_revision="model-sha",
tokenizer_revision="tokenizer-sha",
)
assert running["status"] == "running"
assert revised["model_revision"] == "model-sha"
assert revised["tokenizer_revision"] == "tokenizer-sha"
assert (archive._run_dir(run_id) / "run.log").read_text() == "first line\n"
def test_checkpoint_provenance_attachment_rejects_invalid_and_conflicting_records(
tmp_path,
):
archive = RunArchive(tmp_path)
run_id = archive.begin(["org/model"])
with pytest.raises(ValueError, match="canonical provenance"):
archive.attach_checkpoint_provenance(run_id, object())
invalid = SimpleNamespace(to_json=lambda: json.dumps({"artifact_id": "artifact-a"}))
with pytest.raises(ValueError, match="canonical provenance"):
archive.attach_checkpoint_provenance(run_id, invalid)
def provenance(output_digest: str):
return build_provenance(
sources=(
ArtifactIdentity(
"generated",
"run-archive-test",
"v1",
"sha256:" + "a" * 64,
),
),
converter=ToolIdentity("test", "1", "c" * 40),
obliteratus_commit="c" * 40,
configuration_digest=None,
tokenizer=None,
base_model=None,
command=("checkpoint", "attach"),
environment={"python": "test", "platform": "cpu", "packages": {}},
source_topology={},
lineage=(),
input_digests=("sha256:" + "a" * 64,),
output_digests=(output_digest,),
transformations=(),
observed_scopes=("model_weights",),
lost_state=(),
)
first = provenance("sha256:" + "b" * 64)
attached = archive.attach_checkpoint_provenance(run_id, first)
assert attached["artifact_id"] == first.artifact_id
assert attached["checkpoint_provenance"]["sha256"].startswith("sha256:")
second = provenance("sha256:" + "d" * 64)
with pytest.raises(ValueError, match="different artifact identity"):
archive.attach_checkpoint_provenance(run_id, second)
def test_evaluation_validation_and_failed_terminal_record(tmp_path):
archive = RunArchive(tmp_path)
run_id = archive.begin(["org/model"])
with pytest.raises(ValueError, match="unsupported evaluation partition"):
archive.begin_evaluation(run_id, partition="training", evaluator="test-v1")
checkpoint = archive._run_dir(run_id) / "checkpoint"
checkpoint.mkdir()
(checkpoint / "weights.bin").write_bytes(b"weights")
archive.complete(run_id, checkpoint=checkpoint, metrics={})
with pytest.raises(ValueError, match="requires qwen38-v1"):
archive.begin_evaluation(run_id, partition="optimizer_tune", evaluator="test-v1")
archive.record_experiment_protocol(
run_id,
{"protocol": "qwen38-v1", "manifest_sha256": "a" * 64, "counts": {}},
)
reservation = archive.begin_evaluation(
run_id,
partition="optimizer_tune",
evaluator="test-v1",
)
failed = archive.finish_evaluation(
run_id,
reservation["evaluation_id"],
failure=RuntimeError("Bearer abcdefghijklmnop"),
log=["failed safely"],
)
assert failed["status"] == "failed"
assert failed["failure"] == {"type": "RuntimeError", "message": "[REDACTED]"}
with pytest.raises(ValueError, match="invalid evaluation ID"):
archive.finish_evaluation(run_id, "invalid")
with pytest.raises(KeyError, match="unknown evaluation ID"):
archive.finish_evaluation(run_id, "eval-" + "e" * 32)
with pytest.raises(ValueError, match="already terminal"):
archive.finish_evaluation(run_id, reservation["evaluation_id"])
def test_complete_cancel_prune_and_worker_identity_fail_closed(tmp_path, monkeypatch):
archive = RunArchive(tmp_path)
run_id = archive.begin(["org/model"])
with pytest.raises(ValueError, match="managed run checkpoint"):
archive.complete(run_id, checkpoint=tmp_path)
with pytest.raises(ValueError, match="operator reason"):
archive.prune_checkpoint(run_id, reason=" ")
queued = archive._load(run_id)
assert archive._worker_matches({"worker": {"pid": 1}}) is False
queued["worker"] = {"pid": 9001, "uid": -1, "start_ticks": 10}
assert archive._worker_matches(queued) is False
queued["worker"] = {"pid": 9001, "uid": os.getuid(), "start_ticks": 10}
monkeypatch.setattr("obliteratus.run_archive._process_start_ticks", lambda _pid: 10)
assert archive._worker_matches(queued) is True
archive.fail(run_id, RuntimeError("failed"), phase="pipeline")
assert archive.cancel(run_id)["status"] == "failed"
live_run = archive.begin(["org/model"])
live = archive._load(live_run)
live["worker"] = {"pid": 9002, "uid": os.getuid(), "start_ticks": 11}
archive._save(live)
monkeypatch.setattr("obliteratus.run_archive._process_start_ticks", lambda _pid: 11)
monkeypatch.setattr(
"obliteratus.run_archive.os.killpg",
lambda *_args: (_ for _ in ()).throw(PermissionError("denied")),
)
with pytest.raises(RuntimeError, match="cannot cancel worker"):
archive.cancel(live_run)
def test_option_dataset_metric_and_main_edge_contracts(tmp_path, monkeypatch):
payload = tmp_path / "pairs.json"
payload.write_bytes(b"pairs")
arguments = [
"model",
"--dataset=custom",
"--prompt-pairs-file",
str(payload),
"--residue-file=missing.json",
]
assert _option_value(arguments, "--dataset") == "custom"
assert _option_value(arguments, "--absent") is None
inputs = _dataset_inputs(arguments)
assert inputs[0]["identifier"] == "custom"
assert inputs[1]["sha256"] == hashlib.sha256(b"pairs").hexdigest()
assert inputs[2]["sha256"] is None
checkpoint = tmp_path / "checkpoint"
checkpoint.mkdir()
assert _checkpoint_metrics(checkpoint) == {}
(checkpoint / "abliteration_metadata.json").write_text(
json.dumps({"metrics": ["invalid"]}),
encoding="utf-8",
)
assert _checkpoint_metrics(checkpoint) == {}
with pytest.raises(SystemExit, match="requires obliteration arguments"):
main(["worker", "--archive-root", str(tmp_path), "--run-id", "run-" + "f" * 32])
observed = []
monkeypatch.setattr(
"obliteratus.run_archive._worker",
lambda archive, run_id, args: observed.append((archive.root, run_id, args)) or 7,
)
run_id = "run-" + "f" * 32
assert (
main(
[
"worker",
"--archive-root",
str(tmp_path),
"--run-id",
run_id,
"--",
"org/model",
]
)
== 7
)
assert observed == [(tmp_path.resolve(), run_id, ["org/model"])]