mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-09-21 17:00:50 +02:00
542 lines
17 KiB
JSON
542 lines
17 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://obliteratus.dev/schemas/checkpoint-descriptor-v1.schema.json",
|
|
"title": "OBLITERATUS Checkpoint Descriptor v1",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"schema_id",
|
|
"schema_version",
|
|
"descriptor_id",
|
|
"primary_format",
|
|
"classification_confidence",
|
|
"components",
|
|
"producer",
|
|
"evidence",
|
|
"source_inventory",
|
|
"safety",
|
|
"state",
|
|
"topologies",
|
|
"tensor_fragments",
|
|
"adapter_resolution",
|
|
"conversion_plan",
|
|
"resource_estimate",
|
|
"support_decision",
|
|
"blockers"
|
|
],
|
|
"properties": {
|
|
"schema_id": {"const": "obliteratus.checkpoint-descriptor"},
|
|
"schema_version": {"const": "1.0.0"},
|
|
"descriptor_id": {"type": "string", "minLength": 1, "maxLength": 160},
|
|
"primary_format": {"$ref": "#/$defs/checkpointFormat"},
|
|
"classification_confidence": {
|
|
"enum": ["verified", "declared", "inferred", "unknown"]
|
|
},
|
|
"components": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"maxItems": 1024,
|
|
"items": {"$ref": "#/$defs/component"}
|
|
},
|
|
"producer": {"$ref": "#/$defs/producer"},
|
|
"evidence": {
|
|
"type": "array",
|
|
"maxItems": 100000,
|
|
"items": {"$ref": "#/$defs/evidence"}
|
|
},
|
|
"source_inventory": {"$ref": "#/$defs/sourceInventory"},
|
|
"safety": {"$ref": "#/$defs/safety"},
|
|
"state": {"$ref": "#/$defs/state"},
|
|
"topologies": {
|
|
"type": "array",
|
|
"maxItems": 128,
|
|
"items": {"$ref": "#/$defs/topology"}
|
|
},
|
|
"tensor_fragments": {
|
|
"type": "array",
|
|
"maxItems": 10000000,
|
|
"items": {"$ref": "#/$defs/tensorFragment"}
|
|
},
|
|
"adapter_resolution": {"$ref": "#/$defs/adapterResolution"},
|
|
"conversion_plan": {"$ref": "#/$defs/conversionPlan"},
|
|
"resource_estimate": {"$ref": "#/$defs/resourceEstimate"},
|
|
"support_decision": {
|
|
"enum": [
|
|
"canonical_hf_ready",
|
|
"conversion_supported",
|
|
"trusted_inspection_required",
|
|
"blocked"
|
|
]
|
|
},
|
|
"blockers": {
|
|
"type": "array",
|
|
"maxItems": 10000,
|
|
"items": {"$ref": "#/$defs/blocker"}
|
|
}
|
|
},
|
|
"$defs": {
|
|
"nonNegativeInt64": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"maximum": 9223372036854775807
|
|
},
|
|
"digest": {
|
|
"type": "string",
|
|
"pattern": "^sha256:[0-9a-f]{64}$"
|
|
},
|
|
"identifier": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"maxLength": 512
|
|
},
|
|
"checkpointFormat": {
|
|
"enum": [
|
|
"hf_safetensors",
|
|
"hf_pytorch_pickle",
|
|
"pytorch_dcp",
|
|
"fsdp_state_dict",
|
|
"megatron_torch_dist",
|
|
"megatron_torch_dcp",
|
|
"megatron_fsdp_dtensor",
|
|
"deepspeed_zero",
|
|
"deepspeed_universal",
|
|
"peft_safetensors",
|
|
"unknown",
|
|
"ambiguous"
|
|
]
|
|
},
|
|
"component": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"component_id",
|
|
"kind",
|
|
"format",
|
|
"producer",
|
|
"state_scopes",
|
|
"inventory_refs",
|
|
"tensor_fragment_refs"
|
|
],
|
|
"properties": {
|
|
"component_id": {"$ref": "#/$defs/identifier"},
|
|
"kind": {
|
|
"enum": [
|
|
"model",
|
|
"optimizer",
|
|
"scheduler",
|
|
"rng",
|
|
"scaler",
|
|
"progress",
|
|
"dataloader",
|
|
"peft_adapter",
|
|
"configuration",
|
|
"tokenizer",
|
|
"unknown"
|
|
]
|
|
},
|
|
"format": {"$ref": "#/$defs/checkpointFormat"},
|
|
"producer": {"$ref": "#/$defs/producer"},
|
|
"state_scopes": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
},
|
|
"topology_ref": {"type": ["string", "null"], "maxLength": 512},
|
|
"inventory_refs": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
},
|
|
"tensor_fragment_refs": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
},
|
|
"producer": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["name", "version", "format_version", "evidence_refs"],
|
|
"properties": {
|
|
"name": {"type": ["string", "null"], "maxLength": 256},
|
|
"version": {"type": ["string", "null"], "maxLength": 128},
|
|
"format_version": {"type": ["string", "null"], "maxLength": 128},
|
|
"evidence_refs": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
},
|
|
"evidence": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"evidence_id",
|
|
"subject",
|
|
"kind",
|
|
"file_ref",
|
|
"location",
|
|
"confidence",
|
|
"verifier"
|
|
],
|
|
"properties": {
|
|
"evidence_id": {"$ref": "#/$defs/identifier"},
|
|
"subject": {"type": "string", "pattern": "^/"},
|
|
"kind": {
|
|
"enum": ["explicit_metadata", "header", "json", "filename", "operator", "derived"]
|
|
},
|
|
"file_ref": {"type": ["string", "null"], "maxLength": 512},
|
|
"location": {"type": "string", "maxLength": 1024},
|
|
"confidence": {"enum": ["verified", "declared", "inferred", "unknown"]},
|
|
"verifier": {"type": "string", "maxLength": 256}
|
|
}
|
|
},
|
|
"sourceInventory": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["inventory_id", "files", "total_bytes", "observation_complete"],
|
|
"properties": {
|
|
"inventory_id": {"$ref": "#/$defs/identifier"},
|
|
"files": {
|
|
"type": "array",
|
|
"maxItems": 1000000,
|
|
"items": {"$ref": "#/$defs/inventoryFile"}
|
|
},
|
|
"total_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"observation_complete": {"type": "boolean"}
|
|
}
|
|
},
|
|
"inventoryFile": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"file_id",
|
|
"relative_path",
|
|
"role",
|
|
"size_bytes",
|
|
"sha256",
|
|
"regular_file",
|
|
"observation_id"
|
|
],
|
|
"properties": {
|
|
"file_id": {"$ref": "#/$defs/identifier"},
|
|
"relative_path": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"maxLength": 4096,
|
|
"not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}
|
|
},
|
|
"role": {"type": "string", "minLength": 1, "maxLength": 128},
|
|
"size_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"sha256": {"$ref": "#/$defs/digest"},
|
|
"regular_file": {"const": true},
|
|
"observation_id": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
},
|
|
"safety": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"inspection_level",
|
|
"trust_required",
|
|
"inventory_revalidated",
|
|
"unsafe_serialization_findings",
|
|
"violations"
|
|
],
|
|
"properties": {
|
|
"inspection_level": {"enum": ["safe_structure", "trusted_metadata"]},
|
|
"trust_required": {"type": "boolean"},
|
|
"inventory_revalidated": {"type": "boolean"},
|
|
"unsafe_serialization_findings": {
|
|
"type": "array",
|
|
"items": {"type": "string", "maxLength": 1024}
|
|
},
|
|
"violations": {
|
|
"type": "array",
|
|
"items": {"type": "string", "maxLength": 1024}
|
|
}
|
|
}
|
|
},
|
|
"state": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["observed_scopes", "classification"],
|
|
"properties": {
|
|
"observed_scopes": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
},
|
|
"classification": {
|
|
"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]
|
|
}
|
|
}
|
|
},
|
|
"fact": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["value", "provenance", "evidence_refs"],
|
|
"properties": {
|
|
"value": {"anyOf": [{"$ref": "#/$defs/nonNegativeInt64"}, {"type": "null"}]},
|
|
"provenance": {
|
|
"enum": ["explicit", "filename_inferred", "operator_supplied", "unknown"]
|
|
},
|
|
"evidence_refs": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
},
|
|
"topology": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"topology_id",
|
|
"kind",
|
|
"node_count",
|
|
"global_world_size",
|
|
"machine_rank",
|
|
"global_rank",
|
|
"local_rank",
|
|
"tp_size",
|
|
"pp_size",
|
|
"dp_size",
|
|
"cp_size",
|
|
"ep_size",
|
|
"etp_size",
|
|
"zero_stage"
|
|
],
|
|
"properties": {
|
|
"topology_id": {"$ref": "#/$defs/identifier"},
|
|
"kind": {"enum": ["saved", "target"]},
|
|
"node_count": {"$ref": "#/$defs/fact"},
|
|
"global_world_size": {"$ref": "#/$defs/fact"},
|
|
"machine_rank": {"$ref": "#/$defs/fact"},
|
|
"global_rank": {"$ref": "#/$defs/fact"},
|
|
"local_rank": {"$ref": "#/$defs/fact"},
|
|
"tp_size": {"$ref": "#/$defs/fact"},
|
|
"pp_size": {"$ref": "#/$defs/fact"},
|
|
"dp_size": {"$ref": "#/$defs/fact"},
|
|
"cp_size": {"$ref": "#/$defs/fact"},
|
|
"ep_size": {"$ref": "#/$defs/fact"},
|
|
"etp_size": {"$ref": "#/$defs/fact"},
|
|
"zero_stage": {"$ref": "#/$defs/fact"}
|
|
}
|
|
},
|
|
"tensorFragment": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"fragment_id",
|
|
"component_id",
|
|
"fqn",
|
|
"role",
|
|
"dtype",
|
|
"global_shape",
|
|
"local_shape",
|
|
"element_offset",
|
|
"element_extent",
|
|
"padding",
|
|
"shard_file_id",
|
|
"shard_digest_ref",
|
|
"replica",
|
|
"partition_axes",
|
|
"logical_tensor_id",
|
|
"storage_locations",
|
|
"evidence_refs"
|
|
],
|
|
"properties": {
|
|
"fragment_id": {"$ref": "#/$defs/identifier"},
|
|
"component_id": {"$ref": "#/$defs/identifier"},
|
|
"fqn": {"$ref": "#/$defs/identifier"},
|
|
"role": {"enum": ["parameter", "persistent_buffer"]},
|
|
"dtype": {"$ref": "#/$defs/identifier"},
|
|
"global_shape": {"$ref": "#/$defs/shape"},
|
|
"local_shape": {"$ref": "#/$defs/shape"},
|
|
"element_offset": {"$ref": "#/$defs/shape"},
|
|
"element_extent": {"$ref": "#/$defs/shape"},
|
|
"padding": {"$ref": "#/$defs/padding"},
|
|
"shard_file_id": {"$ref": "#/$defs/identifier"},
|
|
"shard_digest_ref": {"$ref": "#/$defs/identifier"},
|
|
"fragment_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]},
|
|
"replica": {"$ref": "#/$defs/replica"},
|
|
"partition_axes": {
|
|
"type": "array",
|
|
"items": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"uniqueItems": true
|
|
},
|
|
"logical_tensor_id": {"$ref": "#/$defs/identifier"},
|
|
"tie_group_id": {"type": ["string", "null"], "maxLength": 512},
|
|
"shared_storage_id": {"type": ["string", "null"], "maxLength": 512},
|
|
"storage_locations": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {"$ref": "#/$defs/storageLocation"}
|
|
},
|
|
"evidence_refs": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
},
|
|
"shape": {
|
|
"type": "array",
|
|
"maxItems": 32,
|
|
"items": {"$ref": "#/$defs/nonNegativeInt64"}
|
|
},
|
|
"padding": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["before", "after", "semantic"],
|
|
"properties": {
|
|
"before": {"$ref": "#/$defs/shape"},
|
|
"after": {"$ref": "#/$defs/shape"},
|
|
"semantic": {"enum": ["none", "producer_declared", "model_mapping_declared"]}
|
|
}
|
|
},
|
|
"replica": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["group_id", "member_index", "member_count"],
|
|
"properties": {
|
|
"group_id": {"type": ["string", "null"], "maxLength": 512},
|
|
"member_index": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"member_count": {"$ref": "#/$defs/nonNegativeInt64"}
|
|
}
|
|
},
|
|
"storageLocation": {
|
|
"oneOf": [
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["kind", "element_offset", "element_extent"],
|
|
"properties": {
|
|
"kind": {"const": "logical_element_range"},
|
|
"element_offset": {"$ref": "#/$defs/shape"},
|
|
"element_extent": {"$ref": "#/$defs/shape"}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["kind", "global_offset", "rank_offsets"],
|
|
"properties": {
|
|
"kind": {"const": "megatron_rank_offsets"},
|
|
"global_offset": {"$ref": "#/$defs/shape"},
|
|
"rank_offsets": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "array",
|
|
"prefixItems": [
|
|
{"$ref": "#/$defs/nonNegativeInt64"},
|
|
{"$ref": "#/$defs/nonNegativeInt64"},
|
|
{"$ref": "#/$defs/nonNegativeInt64"}
|
|
],
|
|
"items": false,
|
|
"minItems": 3,
|
|
"maxItems": 3
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["kind", "begin", "end"],
|
|
"properties": {
|
|
"kind": {"const": "safetensors_byte_range"},
|
|
"begin": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"end": {"$ref": "#/$defs/nonNegativeInt64"}
|
|
}
|
|
},
|
|
{
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["kind", "tensor_name", "file_id"],
|
|
"properties": {
|
|
"kind": {"const": "hf_weight_map"},
|
|
"tensor_name": {"$ref": "#/$defs/identifier"},
|
|
"file_id": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"adapterResolution": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["status", "adapter_id", "adapter_version", "capability_digest", "reason"],
|
|
"properties": {
|
|
"status": {"enum": ["matched", "not_required", "missing", "ambiguous", "unsupported"]},
|
|
"adapter_id": {"type": ["string", "null"], "maxLength": 256},
|
|
"adapter_version": {"type": ["string", "null"], "maxLength": 128},
|
|
"capability_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]},
|
|
"reason": {"type": "string", "maxLength": 2048}
|
|
}
|
|
},
|
|
"conversionPlan": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["eligible", "target_format", "state_scope", "dropped_scopes"],
|
|
"properties": {
|
|
"eligible": {"type": "boolean"},
|
|
"target_format": {"const": "hf_safetensors"},
|
|
"state_scope": {"const": "weights_only"},
|
|
"dropped_scopes": {
|
|
"type": "array",
|
|
"uniqueItems": true,
|
|
"items": {"$ref": "#/$defs/identifier"}
|
|
}
|
|
}
|
|
},
|
|
"resourceEstimate": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": [
|
|
"source_bytes",
|
|
"logical_bytes",
|
|
"output_bytes",
|
|
"temporary_bytes",
|
|
"peak_ram_bytes",
|
|
"peak_vram_bytes",
|
|
"file_count",
|
|
"tensor_count",
|
|
"shard_count",
|
|
"assumptions",
|
|
"confidence",
|
|
"admission"
|
|
],
|
|
"properties": {
|
|
"source_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"logical_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"output_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"temporary_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"peak_ram_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"peak_vram_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"file_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"tensor_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"shard_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
|
"assumptions": {"type": "array", "items": {"type": "string", "maxLength": 1024}},
|
|
"confidence": {"enum": ["verified", "declared", "inferred", "unknown"]},
|
|
"admission": {"enum": ["admitted", "denied", "unknown"]}
|
|
}
|
|
},
|
|
"blocker": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["code", "category", "phase", "affected_refs", "retryable", "next_action"],
|
|
"properties": {
|
|
"code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"},
|
|
"category": {"enum": ["cleanup", "concurrency", "evidence", "output", "protocol", "resource", "runtime", "source", "trust", "unsupported", "validation"]},
|
|
"phase": {"enum": ["admission", "classification", "cleanup", "evidence", "materialization", "policy", "preflight", "promotion", "protocol", "reader", "source", "validation"]},
|
|
"affected_refs": {"type": "array", "items": {"$ref": "#/$defs/identifier"}},
|
|
"retryable": {"type": "boolean"},
|
|
"next_action": {"type": "string", "maxLength": 2048}
|
|
}
|
|
}
|
|
}
|
|
}
|