mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-09-27 11:41:55 +02:00
feat: add safe distributed checkpoint intake and preflight
This commit is contained in:
Joseph Magly
@@ -0,0 +1,148 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/adapter-capability-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint Adapter Capability v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_id",
|
||||
"schema_version",
|
||||
"adapter_id",
|
||||
"adapter_version",
|
||||
"contract_version",
|
||||
"supported_producer_versions",
|
||||
"tested_producer_versions",
|
||||
"required_extras",
|
||||
"formats",
|
||||
"model_families",
|
||||
"state_scopes",
|
||||
"topology_capabilities",
|
||||
"quantization_capabilities",
|
||||
"safe_inspection",
|
||||
"trusted_inspection",
|
||||
"conversion",
|
||||
"evidence"
|
||||
],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.adapter-capability"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"adapter_id": {"$ref": "#/$defs/id"},
|
||||
"adapter_version": {"$ref": "#/$defs/version"},
|
||||
"contract_version": {"const": "1.0.0"},
|
||||
"supported_producer_versions": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"$ref": "#/$defs/versionBand"}
|
||||
},
|
||||
"tested_producer_versions": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/$defs/testedVersion"}
|
||||
},
|
||||
"required_extras": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/id"}
|
||||
},
|
||||
"formats": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"enum": [
|
||||
"hf_safetensors",
|
||||
"pytorch_dcp",
|
||||
"fsdp_state_dict",
|
||||
"megatron_torch_dist",
|
||||
"megatron_torch_dcp",
|
||||
"megatron_fsdp_dtensor",
|
||||
"deepspeed_zero",
|
||||
"deepspeed_universal",
|
||||
"peft_safetensors"
|
||||
]
|
||||
}
|
||||
},
|
||||
"model_families": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/id"}
|
||||
},
|
||||
"state_scopes": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/id"}
|
||||
},
|
||||
"topology_capabilities": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["axes", "saved_to_canonical", "saved_to_changed_topology"],
|
||||
"properties": {
|
||||
"axes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"enum": ["nodes", "world", "tp", "pp", "dp", "cp", "ep", "etp", "fsdp", "zero"]}
|
||||
},
|
||||
"saved_to_canonical": {"type": "boolean"},
|
||||
"saved_to_changed_topology": {"type": "boolean"}
|
||||
}
|
||||
},
|
||||
"quantization_capabilities": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["format", "status", "evidence_refs"],
|
||||
"properties": {
|
||||
"format": {"$ref": "#/$defs/id"},
|
||||
"status": {"enum": ["supported", "refused", "deferred"]},
|
||||
"evidence_refs": {"type": "array", "items": {"$ref": "#/$defs/id"}}
|
||||
}
|
||||
}
|
||||
},
|
||||
"safe_inspection": {"type": "boolean"},
|
||||
"trusted_inspection": {"type": "boolean"},
|
||||
"conversion": {"type": "boolean"},
|
||||
"evidence": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/$defs/evidence"}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"version": {"type": "string", "minLength": 1, "maxLength": 128},
|
||||
"versionBand": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["producer", "specifier", "rationale"],
|
||||
"properties": {
|
||||
"producer": {"$ref": "#/$defs/id"},
|
||||
"specifier": {"$ref": "#/$defs/version"},
|
||||
"rationale": {"type": "string", "minLength": 1, "maxLength": 2048}
|
||||
}
|
||||
},
|
||||
"testedVersion": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["producer", "version", "fixture_digest", "candidate_commit", "environment", "topology"],
|
||||
"properties": {
|
||||
"producer": {"$ref": "#/$defs/id"},
|
||||
"version": {"$ref": "#/$defs/version"},
|
||||
"fixture_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"candidate_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"environment": {"$ref": "#/$defs/id"},
|
||||
"topology": {"$ref": "#/$defs/id"}
|
||||
}
|
||||
},
|
||||
"evidence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "reference", "status"],
|
||||
"properties": {
|
||||
"kind": {"enum": ["primary_documentation", "upstream_source", "fixture", "test_result", "review"]},
|
||||
"reference": {"$ref": "#/$defs/id"},
|
||||
"status": {"enum": ["required", "present", "expired", "missing"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/artifact-provenance-v1.schema.json",
|
||||
"title": "OBLITERATUS Artifact Provenance v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_id",
|
||||
"schema_version",
|
||||
"artifact_id",
|
||||
"record_digest",
|
||||
"sources",
|
||||
"converter",
|
||||
"obliteratus_commit",
|
||||
"configuration_digest",
|
||||
"tokenizer",
|
||||
"base_model",
|
||||
"command",
|
||||
"environment",
|
||||
"source_topology",
|
||||
"lineage",
|
||||
"input_digests",
|
||||
"output_digests",
|
||||
"transformations",
|
||||
"state",
|
||||
"adapter",
|
||||
"dataset",
|
||||
"training",
|
||||
"unknowns"
|
||||
],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.artifact-provenance"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"artifact_id": {"type": "string", "pattern": "^artifact-sha256:[0-9a-f]{64}$"},
|
||||
"record_digest": {"$ref": "#/$defs/digest"},
|
||||
"sources": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/artifactIdentity"}},
|
||||
"converter": {"$ref": "#/$defs/toolIdentity"},
|
||||
"obliteratus_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"configuration_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]},
|
||||
"tokenizer": {"anyOf": [{"$ref": "#/$defs/artifactIdentity"}, {"type": "null"}]},
|
||||
"base_model": {"anyOf": [{"$ref": "#/$defs/artifactIdentity"}, {"type": "null"}]},
|
||||
"command": {"type": "array", "maxItems": 256, "items": {"type": "string", "maxLength": 4096}},
|
||||
"environment": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["packages", "platform", "python"],
|
||||
"properties": {
|
||||
"python": {"type": ["string", "null"], "maxLength": 128},
|
||||
"platform": {"type": ["string", "null"], "maxLength": 256},
|
||||
"packages": {"type": "object", "additionalProperties": {"type": "string", "maxLength": 128}}
|
||||
}
|
||||
},
|
||||
"source_topology": {"type": "object"},
|
||||
"lineage": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/lineageEvent"}},
|
||||
"input_digests": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/digest"}},
|
||||
"output_digests": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/digest"}},
|
||||
"transformations": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}},
|
||||
"state": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["classification", "observed_scopes", "lost_state"],
|
||||
"properties": {
|
||||
"classification": {"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]},
|
||||
"observed_scopes": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}},
|
||||
"lost_state": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}
|
||||
}
|
||||
},
|
||||
"adapter": {"anyOf": [{"$ref": "#/$defs/adapterIdentity"}, {"type": "null"}]},
|
||||
"dataset": {"anyOf": [{"$ref": "#/$defs/datasetIdentity"}, {"type": "null"}]},
|
||||
"training": {"anyOf": [{"$ref": "#/$defs/trainingIdentity"}, {"type": "null"}]},
|
||||
"unknowns": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1, "maxLength": 1024}}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"artifactIdentity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "identity", "revision", "digest"],
|
||||
"properties": {
|
||||
"kind": {"enum": ["local", "hub", "generated"]},
|
||||
"identity": {"$ref": "#/$defs/id"},
|
||||
"revision": {"type": ["string", "null"], "maxLength": 256},
|
||||
"digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
},
|
||||
"toolIdentity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "version", "commit"],
|
||||
"properties": {
|
||||
"name": {"$ref": "#/$defs/id"},
|
||||
"version": {"$ref": "#/$defs/id"},
|
||||
"commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}
|
||||
}
|
||||
},
|
||||
"lineageEvent": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["event_id", "event_type", "parent_artifact_ids", "tool", "transformations"],
|
||||
"properties": {
|
||||
"event_id": {"$ref": "#/$defs/id"},
|
||||
"event_type": {"enum": ["consolidation", "reshard", "pretrain", "full_finetune", "adapter_train", "adapter_merge", "quantization", "dequantization", "surgery"]},
|
||||
"parent_artifact_ids": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^artifact-sha256:[0-9a-f]{64}$"}},
|
||||
"tool": {"$ref": "#/$defs/id"},
|
||||
"transformations": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}
|
||||
}
|
||||
},
|
||||
"adapterIdentity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["adapter_type", "base_model", "config_digest", "key_map_digest"],
|
||||
"properties": {
|
||||
"adapter_type": {"$ref": "#/$defs/id"},
|
||||
"base_model": {"$ref": "#/$defs/artifactIdentity"},
|
||||
"config_digest": {"$ref": "#/$defs/digest"},
|
||||
"key_map_digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
},
|
||||
"datasetIdentity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["identifier", "revision", "digest", "split", "subset", "record_count"],
|
||||
"properties": {
|
||||
"identifier": {"$ref": "#/$defs/id"},
|
||||
"revision": {"type": ["string", "null"], "maxLength": 256},
|
||||
"digest": {"$ref": "#/$defs/digest"},
|
||||
"split": {"type": ["string", "null"], "maxLength": 256},
|
||||
"subset": {"type": ["string", "null"], "maxLength": 256},
|
||||
"record_count": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}
|
||||
}
|
||||
},
|
||||
"trainingIdentity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["method", "framework", "framework_version", "hyperparameters_digest"],
|
||||
"properties": {
|
||||
"method": {"enum": ["pretrain", "full_finetune", "adapter_train", "unknown"]},
|
||||
"framework": {"type": ["string", "null"], "maxLength": 256},
|
||||
"framework_version": {"type": ["string", "null"], "maxLength": 128},
|
||||
"hyperparameters_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/checkpoint-descriptor-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint Descriptor v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_id",
|
||||
"schema_version",
|
||||
"descriptor_id",
|
||||
"primary_format",
|
||||
"classification_confidence",
|
||||
"components",
|
||||
"producer",
|
||||
"evidence",
|
||||
"source_inventory",
|
||||
"safety",
|
||||
"state",
|
||||
"topologies",
|
||||
"tensor_fragments",
|
||||
"adapter_resolution",
|
||||
"conversion_plan",
|
||||
"resource_estimate",
|
||||
"support_decision",
|
||||
"blockers"
|
||||
],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.checkpoint-descriptor"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"descriptor_id": {"type": "string", "minLength": 1, "maxLength": 160},
|
||||
"primary_format": {"$ref": "#/$defs/checkpointFormat"},
|
||||
"classification_confidence": {
|
||||
"enum": ["verified", "declared", "inferred", "unknown"]
|
||||
},
|
||||
"components": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 1024,
|
||||
"items": {"$ref": "#/$defs/component"}
|
||||
},
|
||||
"producer": {"$ref": "#/$defs/producer"},
|
||||
"evidence": {
|
||||
"type": "array",
|
||||
"maxItems": 100000,
|
||||
"items": {"$ref": "#/$defs/evidence"}
|
||||
},
|
||||
"source_inventory": {"$ref": "#/$defs/sourceInventory"},
|
||||
"safety": {"$ref": "#/$defs/safety"},
|
||||
"state": {"$ref": "#/$defs/state"},
|
||||
"topologies": {
|
||||
"type": "array",
|
||||
"maxItems": 128,
|
||||
"items": {"$ref": "#/$defs/topology"}
|
||||
},
|
||||
"tensor_fragments": {
|
||||
"type": "array",
|
||||
"maxItems": 10000000,
|
||||
"items": {"$ref": "#/$defs/tensorFragment"}
|
||||
},
|
||||
"adapter_resolution": {"$ref": "#/$defs/adapterResolution"},
|
||||
"conversion_plan": {"$ref": "#/$defs/conversionPlan"},
|
||||
"resource_estimate": {"$ref": "#/$defs/resourceEstimate"},
|
||||
"support_decision": {
|
||||
"enum": [
|
||||
"canonical_hf_ready",
|
||||
"conversion_supported",
|
||||
"trusted_inspection_required",
|
||||
"blocked"
|
||||
]
|
||||
},
|
||||
"blockers": {
|
||||
"type": "array",
|
||||
"maxItems": 10000,
|
||||
"items": {"$ref": "#/$defs/blocker"}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"nonNegativeInt64": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9223372036854775807
|
||||
},
|
||||
"digest": {
|
||||
"type": "string",
|
||||
"pattern": "^sha256:[0-9a-f]{64}$"
|
||||
},
|
||||
"identifier": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 512
|
||||
},
|
||||
"checkpointFormat": {
|
||||
"enum": [
|
||||
"hf_safetensors",
|
||||
"hf_pytorch_pickle",
|
||||
"pytorch_dcp",
|
||||
"fsdp_state_dict",
|
||||
"megatron_torch_dist",
|
||||
"megatron_torch_dcp",
|
||||
"megatron_fsdp_dtensor",
|
||||
"deepspeed_zero",
|
||||
"deepspeed_universal",
|
||||
"peft_safetensors",
|
||||
"unknown",
|
||||
"ambiguous"
|
||||
]
|
||||
},
|
||||
"component": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"component_id",
|
||||
"kind",
|
||||
"format",
|
||||
"producer",
|
||||
"state_scopes",
|
||||
"inventory_refs",
|
||||
"tensor_fragment_refs"
|
||||
],
|
||||
"properties": {
|
||||
"component_id": {"$ref": "#/$defs/identifier"},
|
||||
"kind": {
|
||||
"enum": [
|
||||
"model",
|
||||
"optimizer",
|
||||
"scheduler",
|
||||
"rng",
|
||||
"scaler",
|
||||
"progress",
|
||||
"dataloader",
|
||||
"peft_adapter",
|
||||
"configuration",
|
||||
"tokenizer",
|
||||
"unknown"
|
||||
]
|
||||
},
|
||||
"format": {"$ref": "#/$defs/checkpointFormat"},
|
||||
"producer": {"$ref": "#/$defs/producer"},
|
||||
"state_scopes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
},
|
||||
"topology_ref": {"type": ["string", "null"], "maxLength": 512},
|
||||
"inventory_refs": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
},
|
||||
"tensor_fragment_refs": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"producer": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["name", "version", "format_version", "evidence_refs"],
|
||||
"properties": {
|
||||
"name": {"type": ["string", "null"], "maxLength": 256},
|
||||
"version": {"type": ["string", "null"], "maxLength": 128},
|
||||
"format_version": {"type": ["string", "null"], "maxLength": 128},
|
||||
"evidence_refs": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"evidence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"evidence_id",
|
||||
"subject",
|
||||
"kind",
|
||||
"file_ref",
|
||||
"location",
|
||||
"confidence",
|
||||
"verifier"
|
||||
],
|
||||
"properties": {
|
||||
"evidence_id": {"$ref": "#/$defs/identifier"},
|
||||
"subject": {"type": "string", "pattern": "^/"},
|
||||
"kind": {
|
||||
"enum": ["explicit_metadata", "header", "json", "filename", "operator", "derived"]
|
||||
},
|
||||
"file_ref": {"type": ["string", "null"], "maxLength": 512},
|
||||
"location": {"type": "string", "maxLength": 1024},
|
||||
"confidence": {"enum": ["verified", "declared", "inferred", "unknown"]},
|
||||
"verifier": {"type": "string", "maxLength": 256}
|
||||
}
|
||||
},
|
||||
"sourceInventory": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["inventory_id", "files", "total_bytes", "observation_complete"],
|
||||
"properties": {
|
||||
"inventory_id": {"$ref": "#/$defs/identifier"},
|
||||
"files": {
|
||||
"type": "array",
|
||||
"maxItems": 1000000,
|
||||
"items": {"$ref": "#/$defs/inventoryFile"}
|
||||
},
|
||||
"total_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"observation_complete": {"type": "boolean"}
|
||||
}
|
||||
},
|
||||
"inventoryFile": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"file_id",
|
||||
"relative_path",
|
||||
"role",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"regular_file",
|
||||
"observation_id"
|
||||
],
|
||||
"properties": {
|
||||
"file_id": {"$ref": "#/$defs/identifier"},
|
||||
"relative_path": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 4096,
|
||||
"not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}
|
||||
},
|
||||
"role": {"type": "string", "minLength": 1, "maxLength": 128},
|
||||
"size_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"sha256": {"$ref": "#/$defs/digest"},
|
||||
"regular_file": {"const": true},
|
||||
"observation_id": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"inspection_level",
|
||||
"trust_required",
|
||||
"inventory_revalidated",
|
||||
"unsafe_serialization_findings",
|
||||
"violations"
|
||||
],
|
||||
"properties": {
|
||||
"inspection_level": {"enum": ["safe_structure", "trusted_metadata"]},
|
||||
"trust_required": {"type": "boolean"},
|
||||
"inventory_revalidated": {"type": "boolean"},
|
||||
"unsafe_serialization_findings": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "maxLength": 1024}
|
||||
},
|
||||
"violations": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "maxLength": 1024}
|
||||
}
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["observed_scopes", "classification"],
|
||||
"properties": {
|
||||
"observed_scopes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
},
|
||||
"classification": {
|
||||
"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"fact": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["value", "provenance", "evidence_refs"],
|
||||
"properties": {
|
||||
"value": {"anyOf": [{"$ref": "#/$defs/nonNegativeInt64"}, {"type": "null"}]},
|
||||
"provenance": {
|
||||
"enum": ["explicit", "filename_inferred", "operator_supplied", "unknown"]
|
||||
},
|
||||
"evidence_refs": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"topology": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"topology_id",
|
||||
"kind",
|
||||
"node_count",
|
||||
"global_world_size",
|
||||
"machine_rank",
|
||||
"global_rank",
|
||||
"local_rank",
|
||||
"tp_size",
|
||||
"pp_size",
|
||||
"dp_size",
|
||||
"cp_size",
|
||||
"ep_size",
|
||||
"etp_size",
|
||||
"zero_stage"
|
||||
],
|
||||
"properties": {
|
||||
"topology_id": {"$ref": "#/$defs/identifier"},
|
||||
"kind": {"enum": ["saved", "target"]},
|
||||
"node_count": {"$ref": "#/$defs/fact"},
|
||||
"global_world_size": {"$ref": "#/$defs/fact"},
|
||||
"machine_rank": {"$ref": "#/$defs/fact"},
|
||||
"global_rank": {"$ref": "#/$defs/fact"},
|
||||
"local_rank": {"$ref": "#/$defs/fact"},
|
||||
"tp_size": {"$ref": "#/$defs/fact"},
|
||||
"pp_size": {"$ref": "#/$defs/fact"},
|
||||
"dp_size": {"$ref": "#/$defs/fact"},
|
||||
"cp_size": {"$ref": "#/$defs/fact"},
|
||||
"ep_size": {"$ref": "#/$defs/fact"},
|
||||
"etp_size": {"$ref": "#/$defs/fact"},
|
||||
"zero_stage": {"$ref": "#/$defs/fact"}
|
||||
}
|
||||
},
|
||||
"tensorFragment": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"fragment_id",
|
||||
"component_id",
|
||||
"fqn",
|
||||
"role",
|
||||
"dtype",
|
||||
"global_shape",
|
||||
"local_shape",
|
||||
"element_offset",
|
||||
"element_extent",
|
||||
"padding",
|
||||
"shard_file_id",
|
||||
"shard_digest_ref",
|
||||
"replica",
|
||||
"partition_axes",
|
||||
"logical_tensor_id",
|
||||
"storage_locations",
|
||||
"evidence_refs"
|
||||
],
|
||||
"properties": {
|
||||
"fragment_id": {"$ref": "#/$defs/identifier"},
|
||||
"component_id": {"$ref": "#/$defs/identifier"},
|
||||
"fqn": {"$ref": "#/$defs/identifier"},
|
||||
"role": {"enum": ["parameter", "persistent_buffer"]},
|
||||
"dtype": {"$ref": "#/$defs/identifier"},
|
||||
"global_shape": {"$ref": "#/$defs/shape"},
|
||||
"local_shape": {"$ref": "#/$defs/shape"},
|
||||
"element_offset": {"$ref": "#/$defs/shape"},
|
||||
"element_extent": {"$ref": "#/$defs/shape"},
|
||||
"padding": {"$ref": "#/$defs/padding"},
|
||||
"shard_file_id": {"$ref": "#/$defs/identifier"},
|
||||
"shard_digest_ref": {"$ref": "#/$defs/identifier"},
|
||||
"fragment_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]},
|
||||
"replica": {"$ref": "#/$defs/replica"},
|
||||
"partition_axes": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"uniqueItems": true
|
||||
},
|
||||
"logical_tensor_id": {"$ref": "#/$defs/identifier"},
|
||||
"tie_group_id": {"type": ["string", "null"], "maxLength": 512},
|
||||
"shared_storage_id": {"type": ["string", "null"], "maxLength": 512},
|
||||
"storage_locations": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"$ref": "#/$defs/storageLocation"}
|
||||
},
|
||||
"evidence_refs": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"shape": {
|
||||
"type": "array",
|
||||
"maxItems": 32,
|
||||
"items": {"$ref": "#/$defs/nonNegativeInt64"}
|
||||
},
|
||||
"padding": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["before", "after", "semantic"],
|
||||
"properties": {
|
||||
"before": {"$ref": "#/$defs/shape"},
|
||||
"after": {"$ref": "#/$defs/shape"},
|
||||
"semantic": {"enum": ["none", "producer_declared", "model_mapping_declared"]}
|
||||
}
|
||||
},
|
||||
"replica": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["group_id", "member_index", "member_count"],
|
||||
"properties": {
|
||||
"group_id": {"type": ["string", "null"], "maxLength": 512},
|
||||
"member_index": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"member_count": {"$ref": "#/$defs/nonNegativeInt64"}
|
||||
}
|
||||
},
|
||||
"storageLocation": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "element_offset", "element_extent"],
|
||||
"properties": {
|
||||
"kind": {"const": "logical_element_range"},
|
||||
"element_offset": {"$ref": "#/$defs/shape"},
|
||||
"element_extent": {"$ref": "#/$defs/shape"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "global_offset", "rank_offsets"],
|
||||
"properties": {
|
||||
"kind": {"const": "megatron_rank_offsets"},
|
||||
"global_offset": {"$ref": "#/$defs/shape"},
|
||||
"rank_offsets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"prefixItems": [
|
||||
{"$ref": "#/$defs/nonNegativeInt64"},
|
||||
{"$ref": "#/$defs/nonNegativeInt64"},
|
||||
{"$ref": "#/$defs/nonNegativeInt64"}
|
||||
],
|
||||
"items": false,
|
||||
"minItems": 3,
|
||||
"maxItems": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "begin", "end"],
|
||||
"properties": {
|
||||
"kind": {"const": "safetensors_byte_range"},
|
||||
"begin": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"end": {"$ref": "#/$defs/nonNegativeInt64"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "tensor_name", "file_id"],
|
||||
"properties": {
|
||||
"kind": {"const": "hf_weight_map"},
|
||||
"tensor_name": {"$ref": "#/$defs/identifier"},
|
||||
"file_id": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"adapterResolution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["status", "adapter_id", "adapter_version", "capability_digest", "reason"],
|
||||
"properties": {
|
||||
"status": {"enum": ["matched", "not_required", "missing", "ambiguous", "unsupported"]},
|
||||
"adapter_id": {"type": ["string", "null"], "maxLength": 256},
|
||||
"adapter_version": {"type": ["string", "null"], "maxLength": 128},
|
||||
"capability_digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]},
|
||||
"reason": {"type": "string", "maxLength": 2048}
|
||||
}
|
||||
},
|
||||
"conversionPlan": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["eligible", "target_format", "state_scope", "dropped_scopes"],
|
||||
"properties": {
|
||||
"eligible": {"type": "boolean"},
|
||||
"target_format": {"const": "hf_safetensors"},
|
||||
"state_scope": {"const": "weights_only"},
|
||||
"dropped_scopes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"$ref": "#/$defs/identifier"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"resourceEstimate": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"source_bytes",
|
||||
"logical_bytes",
|
||||
"output_bytes",
|
||||
"temporary_bytes",
|
||||
"peak_ram_bytes",
|
||||
"peak_vram_bytes",
|
||||
"file_count",
|
||||
"tensor_count",
|
||||
"shard_count",
|
||||
"assumptions",
|
||||
"confidence",
|
||||
"admission"
|
||||
],
|
||||
"properties": {
|
||||
"source_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"logical_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"output_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"temporary_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"peak_ram_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"peak_vram_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"file_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"tensor_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"shard_count": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"assumptions": {"type": "array", "items": {"type": "string", "maxLength": 1024}},
|
||||
"confidence": {"enum": ["verified", "declared", "inferred", "unknown"]},
|
||||
"admission": {"enum": ["admitted", "denied", "unknown"]}
|
||||
}
|
||||
},
|
||||
"blocker": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["code", "category", "phase", "affected_refs", "retryable", "next_action"],
|
||||
"properties": {
|
||||
"code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"},
|
||||
"category": {"enum": ["cleanup", "concurrency", "evidence", "output", "protocol", "resource", "runtime", "source", "trust", "unsupported", "validation"]},
|
||||
"phase": {"enum": ["admission", "classification", "cleanup", "evidence", "materialization", "policy", "preflight", "promotion", "protocol", "reader", "source", "validation"]},
|
||||
"affected_refs": {"type": "array", "items": {"$ref": "#/$defs/identifier"}},
|
||||
"retryable": {"type": "boolean"},
|
||||
"next_action": {"type": "string", "maxLength": 2048}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_id": "obliteratus.checkpoint-error-registry",
|
||||
"schema_version": "1.0.0",
|
||||
"entries": [
|
||||
{"code": "DCI_UNSUPPORTED_FORMAT_OR_VERSION", "phase": "classification", "category": "unsupported", "degraded_modes": ["F01"]},
|
||||
{"code": "DCI_TRUST_POLICY_REQUIRED", "phase": "policy", "category": "trust", "degraded_modes": ["F02"]},
|
||||
{"code": "DCI_TRUST_POLICY_MISMATCH", "phase": "policy", "category": "trust", "degraded_modes": ["F02", "F18"]},
|
||||
{"code": "DCI_SOURCE_BOUNDARY_VIOLATION", "phase": "source", "category": "source", "degraded_modes": ["F03"]},
|
||||
{"code": "DCI_SOURCE_CHANGED", "phase": "source", "category": "source", "degraded_modes": ["F04"]},
|
||||
{"code": "DCI_TRUST_RUNTIME_UNAVAILABLE", "phase": "preflight", "category": "runtime", "degraded_modes": ["F05", "F19"]},
|
||||
{"code": "DCI_RUNTIME_IDENTITY_MISMATCH", "phase": "preflight", "category": "runtime", "degraded_modes": ["F06", "F20"]},
|
||||
{"code": "DCI_FORBIDDEN_READER_CAPABILITY", "phase": "reader", "category": "runtime", "degraded_modes": ["F07"]},
|
||||
{"code": "DCI_RESOURCE_LIMIT", "phase": "reader", "category": "resource", "degraded_modes": ["F08", "F09"]},
|
||||
{"code": "DCI_TRUSTED_READER_FAILED", "phase": "reader", "category": "runtime", "degraded_modes": ["F09"]},
|
||||
{"code": "DCI_WORKER_PROTOCOL_INVALID", "phase": "protocol", "category": "protocol", "degraded_modes": ["F10"]},
|
||||
{"code": "DCI_VALIDATION_FAILED", "phase": "validation", "category": "validation", "degraded_modes": ["F11", "F14"]},
|
||||
{"code": "DCI_ADMISSION_DENIED", "phase": "admission", "category": "resource", "degraded_modes": ["F12"]},
|
||||
{"code": "DCI_MATERIALIZE_FAILED", "phase": "materialization", "category": "output", "degraded_modes": ["F13"]},
|
||||
{"code": "DCI_PROMOTION_FAILED", "phase": "promotion", "category": "output", "degraded_modes": ["F13", "F14"]},
|
||||
{"code": "DCI_EVIDENCE_UNAVAILABLE", "phase": "evidence", "category": "evidence", "degraded_modes": ["F15"]},
|
||||
{"code": "DCI_DIAGNOSTIC_REDACTION_FAILED", "phase": "evidence", "category": "evidence", "degraded_modes": ["F16"]},
|
||||
{"code": "DCI_CLEANUP_INCOMPLETE", "phase": "cleanup", "category": "cleanup", "degraded_modes": ["F17"]},
|
||||
{"code": "DCI_CONCURRENT_OPERATION_CONFLICT", "phase": "admission", "category": "concurrency", "degraded_modes": ["F18"]},
|
||||
{"code": "DCI_HOST_TRUST_UNSATISFIED", "phase": "preflight", "category": "trust", "degraded_modes": ["F19"]},
|
||||
{"code": "DCI_SECURITY_BASELINE_REVOKED", "phase": "preflight", "category": "runtime", "degraded_modes": ["F20"]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/checkpoint-error-registry-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint Error Registry v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_id", "schema_version", "entries"],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.checkpoint-error-registry"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"entries": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["code", "phase", "category", "degraded_modes"],
|
||||
"properties": {
|
||||
"code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"},
|
||||
"phase": {"enum": ["classification", "policy", "source", "preflight", "reader", "protocol", "validation", "admission", "materialization", "promotion", "evidence", "cleanup"]},
|
||||
"category": {"enum": ["unsupported", "trust", "source", "runtime", "resource", "protocol", "validation", "output", "evidence", "cleanup", "concurrency"]},
|
||||
"degraded_modes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^F(?:0[1-9]|1[0-9]|20)$"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/checkpoint-trust-policy-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint Trust Policy v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_id", "schema_version", "policy_id", "operation_level", "requested_state_scope", "adapter", "source", "runtime", "isolation", "resources", "output", "authorization", "evidence"],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.checkpoint-trust-policy"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"policy_id": {"$ref": "#/$defs/id"},
|
||||
"operation_level": {"enum": ["trusted_metadata", "trusted_conversion"]},
|
||||
"requested_state_scope": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}},
|
||||
"adapter": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["capability_id", "adapter_id", "adapter_version", "adapter_digest"],
|
||||
"properties": {
|
||||
"capability_id": {"$ref": "#/$defs/id"},
|
||||
"adapter_id": {"$ref": "#/$defs/id"},
|
||||
"adapter_version": {"$ref": "#/$defs/id"},
|
||||
"adapter_digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["root_id", "inventory_digest", "files"],
|
||||
"properties": {
|
||||
"root_id": {"$ref": "#/$defs/id"},
|
||||
"inventory_digest": {"$ref": "#/$defs/digest"},
|
||||
"files": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 1000000,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file_id", "relative_path", "size_bytes", "sha256", "observation_id"],
|
||||
"properties": {
|
||||
"file_id": {"$ref": "#/$defs/id"},
|
||||
"relative_path": {"type": "string", "minLength": 1, "maxLength": 4096, "not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}},
|
||||
"size_bytes": {"$ref": "#/$defs/nonNegativeInt64"},
|
||||
"sha256": {"$ref": "#/$defs/digest"},
|
||||
"observation_id": {"$ref": "#/$defs/id"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"runtime": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["framework", "framework_version", "dependency_identity", "runtime_digest"],
|
||||
"properties": {
|
||||
"framework": {"$ref": "#/$defs/id"},
|
||||
"framework_version": {"$ref": "#/$defs/id"},
|
||||
"dependency_identity": {"$ref": "#/$defs/digest"},
|
||||
"runtime_digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
},
|
||||
"isolation": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["profile_id", "profile_version", "profile_digest", "required_capabilities"],
|
||||
"properties": {
|
||||
"profile_id": {"$ref": "#/$defs/id"},
|
||||
"profile_version": {"$ref": "#/$defs/id"},
|
||||
"profile_digest": {"$ref": "#/$defs/digest"},
|
||||
"required_capabilities": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {"enum": ["unprivileged", "no_new_privs", "read_only_source", "root_isolation", "private_scratch", "network_denied", "ipc_isolated", "environment_allowlist", "fd_allowlist", "syscall_policy", "resource_limits", "device_denied", "core_dumps_disabled", "bounded_cleanup"]}
|
||||
}
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["wall_time_seconds", "cpu_seconds", "memory_bytes", "processes", "threads", "open_files", "input_bytes", "header_bytes", "message_bytes", "scratch_bytes", "output_bytes"],
|
||||
"properties": {
|
||||
"wall_time_seconds": {"$ref": "#/$defs/positiveInt64"},
|
||||
"cpu_seconds": {"$ref": "#/$defs/positiveInt64"},
|
||||
"memory_bytes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"processes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"threads": {"$ref": "#/$defs/positiveInt64"},
|
||||
"open_files": {"$ref": "#/$defs/positiveInt64"},
|
||||
"input_bytes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"header_bytes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"message_bytes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"scratch_bytes": {"$ref": "#/$defs/positiveInt64"},
|
||||
"output_bytes": {"$ref": "#/$defs/nonNegativeInt64"}
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["root_id", "staging_policy"],
|
||||
"properties": {
|
||||
"root_id": {"type": ["string", "null"], "minLength": 1, "maxLength": 512},
|
||||
"staging_policy": {"enum": ["none", "private_sibling_atomic"]}
|
||||
}
|
||||
},
|
||||
"authorization": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["event_id", "actor_role", "created_at", "expires_at", "single_use_nonce", "single_use"],
|
||||
"properties": {
|
||||
"event_id": {"$ref": "#/$defs/id"},
|
||||
"actor_role": {"$ref": "#/$defs/id"},
|
||||
"created_at": {"type": "string", "format": "date-time"},
|
||||
"expires_at": {"type": "string", "format": "date-time"},
|
||||
"single_use_nonce": {"type": "string", "minLength": 32, "maxLength": 256},
|
||||
"single_use": {"const": true}
|
||||
}
|
||||
},
|
||||
"evidence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["candidate_commit", "policy_digest"],
|
||||
"properties": {
|
||||
"candidate_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"policy_digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"nonNegativeInt64": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807},
|
||||
"positiveInt64": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/conversion-manifest-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint Conversion Manifest v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_id",
|
||||
"schema_version",
|
||||
"manifest_id",
|
||||
"descriptor",
|
||||
"source_inventory_digest",
|
||||
"source_files",
|
||||
"adapter",
|
||||
"source_topology",
|
||||
"state",
|
||||
"dropped_scopes",
|
||||
"canonical_output",
|
||||
"resource_usage",
|
||||
"validation",
|
||||
"provenance",
|
||||
"publication"
|
||||
],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.conversion-manifest"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"manifest_id": {"$ref": "#/$defs/id"},
|
||||
"descriptor": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_version", "digest"],
|
||||
"properties": {
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
},
|
||||
"source_inventory_digest": {"$ref": "#/$defs/digest"},
|
||||
"source_files": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"$ref": "#/$defs/file"}
|
||||
},
|
||||
"adapter": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["id", "version", "capability_digest", "producer", "producer_version"],
|
||||
"properties": {
|
||||
"id": {"$ref": "#/$defs/id"},
|
||||
"version": {"$ref": "#/$defs/id"},
|
||||
"capability_digest": {"$ref": "#/$defs/digest"},
|
||||
"producer": {"$ref": "#/$defs/id"},
|
||||
"producer_version": {"$ref": "#/$defs/id"}
|
||||
}
|
||||
},
|
||||
"source_topology": {"type": "object"},
|
||||
"state": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["source_classification", "output_classification", "observed_scopes"],
|
||||
"properties": {
|
||||
"source_classification": {"enum": ["weights_only", "model_and_optimizer", "exact_resume", "unknown"]},
|
||||
"output_classification": {"const": "weights_only"},
|
||||
"observed_scopes": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}
|
||||
}
|
||||
},
|
||||
"dropped_scopes": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["scope", "reason"],
|
||||
"properties": {
|
||||
"scope": {"$ref": "#/$defs/id"},
|
||||
"reason": {"type": "string", "minLength": 1, "maxLength": 2048}
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonical_output": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["format", "dtype_policy", "files", "hf_index", "logical_tensor_count", "logical_bytes"],
|
||||
"properties": {
|
||||
"format": {"const": "hf_safetensors"},
|
||||
"dtype_policy": {"$ref": "#/$defs/id"},
|
||||
"files": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/file"}},
|
||||
"hf_index": {"type": ["string", "null"], "maxLength": 4096},
|
||||
"logical_tensor_count": {"$ref": "#/$defs/int64"},
|
||||
"logical_bytes": {"$ref": "#/$defs/int64"}
|
||||
}
|
||||
},
|
||||
"resource_usage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["estimated_peak_ram_bytes", "actual_peak_ram_bytes", "estimated_temp_bytes", "actual_temp_bytes"],
|
||||
"properties": {
|
||||
"estimated_peak_ram_bytes": {"$ref": "#/$defs/int64"},
|
||||
"actual_peak_ram_bytes": {"anyOf": [{"$ref": "#/$defs/int64"}, {"type": "null"}]},
|
||||
"estimated_temp_bytes": {"$ref": "#/$defs/int64"},
|
||||
"actual_temp_bytes": {"anyOf": [{"$ref": "#/$defs/int64"}, {"type": "null"}]}
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["coverage", "replicas", "ties", "hashes", "index", "safe_reload", "source_unchanged", "result"],
|
||||
"properties": {
|
||||
"coverage": {"type": "boolean"},
|
||||
"replicas": {"type": "boolean"},
|
||||
"ties": {"type": "boolean"},
|
||||
"hashes": {"type": "boolean"},
|
||||
"index": {"type": "boolean"},
|
||||
"safe_reload": {"type": "boolean"},
|
||||
"source_unchanged": {"type": "boolean"},
|
||||
"result": {"enum": ["passed", "failed"]}
|
||||
}
|
||||
},
|
||||
"provenance": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["obliteratus_commit", "configuration_digest", "tokenizer_digest", "base_model", "transformation_log", "unknowns"],
|
||||
"properties": {
|
||||
"obliteratus_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"configuration_digest": {"type": ["string", "null"], "pattern": "^(sha256:[0-9a-f]{64})?$"},
|
||||
"tokenizer_digest": {"type": ["string", "null"], "pattern": "^(sha256:[0-9a-f]{64})?$"},
|
||||
"base_model": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["identity", "revision", "digest"],
|
||||
"properties": {
|
||||
"identity": {"type": ["string", "null"], "maxLength": 512},
|
||||
"revision": {"type": ["string", "null"], "maxLength": 256},
|
||||
"digest": {"anyOf": [{"$ref": "#/$defs/digest"}, {"type": "null"}]}
|
||||
}
|
||||
},
|
||||
"transformation_log": {"type": "array", "items": {"type": "string", "maxLength": 2048}},
|
||||
"unknowns": {"type": "array", "items": {"type": "string", "maxLength": 2048}}
|
||||
}
|
||||
},
|
||||
"publication": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["staging_validated", "promoted", "atomic_strategy", "rollback_result"],
|
||||
"properties": {
|
||||
"staging_validated": {"type": "boolean"},
|
||||
"promoted": {"type": "boolean"},
|
||||
"atomic_strategy": {"$ref": "#/$defs/id"},
|
||||
"rollback_result": {"enum": ["not_required", "succeeded", "failed"]}
|
||||
}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"int64": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807},
|
||||
"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"file": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["relative_path", "size_bytes", "sha256"],
|
||||
"properties": {
|
||||
"relative_path": {"type": "string", "minLength": 1, "maxLength": 4096, "not": {"pattern": "(^/|(^|/)\\.\\.(/|$))"}},
|
||||
"size_bytes": {"$ref": "#/$defs/int64"},
|
||||
"sha256": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schema_id": "obliteratus.adapter-capability",
|
||||
"schema_version": "1.0.0",
|
||||
"adapter_id": "invalid-adapter",
|
||||
"adapter_version": "1.0.0",
|
||||
"contract_version": "1.0.0",
|
||||
"supported_producer_versions": [],
|
||||
"tested_producer_versions": [
|
||||
{"producer": "unknown", "version": "latest"}
|
||||
],
|
||||
"required_extras": [],
|
||||
"formats": ["pytorch_dcp"],
|
||||
"model_families": ["any"],
|
||||
"state_scopes": ["exact_resume"],
|
||||
"topology_capabilities": {"axes": [], "saved_to_canonical": true, "saved_to_changed_topology": true},
|
||||
"quantization_capabilities": [],
|
||||
"safe_inspection": true,
|
||||
"trusted_inspection": true,
|
||||
"conversion": true,
|
||||
"evidence": []
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"schema_id": "obliteratus.checkpoint-descriptor",
|
||||
"schema_version": "1.0.0",
|
||||
"descriptor_id": "invalid-generic-offset",
|
||||
"offset": [0, 4]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"schema_id": "obliteratus.checkpoint-trust-policy",
|
||||
"schema_version": "1.0.0",
|
||||
"environment": {"HOME": "/private/path", "TOKEN": "must-not-persist"}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"schema_id": "obliteratus.trusted-worker-message",
|
||||
"schema_version": "1.0.0",
|
||||
"message_type": "error",
|
||||
"sequence": 1,
|
||||
"source_inventory_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"adapter_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"policy_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
|
||||
"payload": {"code": "DCI_TRUSTED_READER_FAILED", "phase": "reader", "category": "runtime", "subject_ids": [], "limit": null, "actual": null, "raw_message": "private path or secret"}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"schema_id": "obliteratus.adapter-capability",
|
||||
"schema_version": "1.0.0",
|
||||
"adapter_id": "megatron-bridge-planned",
|
||||
"adapter_version": "0.1.0-planned",
|
||||
"contract_version": "1.0.0",
|
||||
"supported_producer_versions": [
|
||||
{
|
||||
"producer": "megatron-core",
|
||||
"specifier": "unselected",
|
||||
"rationale": "Version band remains blocked on exact fixture and adapter selection."
|
||||
}
|
||||
],
|
||||
"tested_producer_versions": [],
|
||||
"required_extras": ["checkpoint-megatron"],
|
||||
"formats": ["megatron_torch_dist"],
|
||||
"model_families": ["unselected"],
|
||||
"state_scopes": ["model_weights"],
|
||||
"topology_capabilities": {
|
||||
"axes": ["world", "tp", "pp", "dp"],
|
||||
"saved_to_canonical": true,
|
||||
"saved_to_changed_topology": false
|
||||
},
|
||||
"quantization_capabilities": [
|
||||
{"format": "native_distributed", "status": "refused", "evidence_refs": []}
|
||||
],
|
||||
"safe_inspection": false,
|
||||
"trusted_inspection": false,
|
||||
"conversion": false,
|
||||
"evidence": [
|
||||
{"kind": "primary_documentation", "reference": "R04,R07", "status": "present"},
|
||||
{"kind": "fixture", "reference": "exact-version-producer-fixture", "status": "missing"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"schema_id": "obliteratus.checkpoint-trust-policy",
|
||||
"schema_version": "1.0.0",
|
||||
"policy_id": "policy-fixture-1",
|
||||
"operation_level": "trusted_metadata",
|
||||
"requested_state_scope": ["model_weights"],
|
||||
"adapter": {"capability_id": "dcp-metadata", "adapter_id": "fixture-reader", "adapter_version": "1.0.0", "adapter_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
"source": {"root_id": "source-1", "inventory_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "files": [{"file_id": "file-1", "relative_path": ".metadata", "size_bytes": 32, "sha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "observation_id": "observation-1"}]},
|
||||
"runtime": {"framework": "pytorch", "framework_version": "exact-fixture-version", "dependency_identity": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "runtime_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"},
|
||||
"isolation": {"profile_id": "disposable-linux-fixture", "profile_version": "1", "profile_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "required_capabilities": ["unprivileged", "no_new_privs", "read_only_source", "network_denied", "resource_limits", "bounded_cleanup"]},
|
||||
"resources": {"wall_time_seconds": 30, "cpu_seconds": 20, "memory_bytes": 1073741824, "processes": 8, "threads": 32, "open_files": 128, "input_bytes": 1048576, "header_bytes": 65536, "message_bytes": 1048576, "scratch_bytes": 1048576, "output_bytes": 0},
|
||||
"output": {"root_id": null, "staging_policy": "none"},
|
||||
"authorization": {"event_id": "authorization-fixture-1", "actor_role": "checkpoint-security-operator", "created_at": "2026-09-02T12:00:00Z", "expires_at": "2026-09-02T12:05:00Z", "single_use_nonce": "0123456789abcdef0123456789abcdef", "single_use": true},
|
||||
"evidence": {"candidate_commit": "1111111111111111111111111111111111111111", "policy_digest": "sha256:9999999999999999999999999999999999999999999999999999999999999999"}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"schema_id": "obliteratus.trusted-worker-message",
|
||||
"schema_version": "1.0.0",
|
||||
"message_type": "complete",
|
||||
"sequence": 4,
|
||||
"source_inventory_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"adapter_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"policy_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
|
||||
"payload": {"record_count": 4, "encoded_bytes": 4096, "transcript_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"schema_id": "obliteratus.checkpoint-descriptor",
|
||||
"schema_version": "1.0.0",
|
||||
"descriptor_id": "fixture-typed-offset-namespaces",
|
||||
"primary_format": "megatron_torch_dist",
|
||||
"classification_confidence": "verified",
|
||||
"components": [
|
||||
{
|
||||
"component_id": "model",
|
||||
"kind": "model",
|
||||
"format": "megatron_torch_dist",
|
||||
"producer": {
|
||||
"name": "megatron-core",
|
||||
"version": "fixture-version",
|
||||
"format_version": "torch_dist",
|
||||
"evidence_refs": ["ev-producer"]
|
||||
},
|
||||
"state_scopes": ["model_weights"],
|
||||
"topology_ref": "saved-topology",
|
||||
"inventory_refs": ["shard-0"],
|
||||
"tensor_fragment_refs": ["fragment-0"]
|
||||
}
|
||||
],
|
||||
"producer": {
|
||||
"name": "megatron-core",
|
||||
"version": "fixture-version",
|
||||
"format_version": "torch_dist",
|
||||
"evidence_refs": ["ev-producer"]
|
||||
},
|
||||
"evidence": [
|
||||
{
|
||||
"evidence_id": "ev-producer",
|
||||
"subject": "/producer",
|
||||
"kind": "explicit_metadata",
|
||||
"file_ref": "shard-0",
|
||||
"location": "fixture metadata producer",
|
||||
"confidence": "verified",
|
||||
"verifier": "neutral-fixture-generator-v1"
|
||||
}
|
||||
],
|
||||
"source_inventory": {
|
||||
"inventory_id": "inventory-0",
|
||||
"files": [
|
||||
{
|
||||
"file_id": "shard-0",
|
||||
"relative_path": "rank-0/model.safetensors",
|
||||
"role": "model_fragment",
|
||||
"size_bytes": 16,
|
||||
"sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"regular_file": true,
|
||||
"observation_id": "dev-1:ino-1:size-16:mtime-0"
|
||||
}
|
||||
],
|
||||
"total_bytes": 16,
|
||||
"observation_complete": true
|
||||
},
|
||||
"safety": {
|
||||
"inspection_level": "trusted_metadata",
|
||||
"trust_required": true,
|
||||
"inventory_revalidated": true,
|
||||
"unsafe_serialization_findings": [],
|
||||
"violations": []
|
||||
},
|
||||
"state": {
|
||||
"observed_scopes": ["model_weights"],
|
||||
"classification": "weights_only"
|
||||
},
|
||||
"topologies": [
|
||||
{
|
||||
"topology_id": "saved-topology",
|
||||
"kind": "saved",
|
||||
"node_count": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"global_world_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"machine_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"global_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"local_rank": {"value": 0, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"tp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"pp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"dp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"cp_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"ep_size": {"value": 1, "provenance": "explicit", "evidence_refs": ["ev-producer"]},
|
||||
"etp_size": {"value": null, "provenance": "unknown", "evidence_refs": []},
|
||||
"zero_stage": {"value": null, "provenance": "unknown", "evidence_refs": []}
|
||||
}
|
||||
],
|
||||
"tensor_fragments": [
|
||||
{
|
||||
"fragment_id": "fragment-0",
|
||||
"component_id": "model",
|
||||
"fqn": "layer.weight",
|
||||
"role": "parameter",
|
||||
"dtype": "F32",
|
||||
"global_shape": [2, 2],
|
||||
"local_shape": [2, 2],
|
||||
"element_offset": [0, 0],
|
||||
"element_extent": [2, 2],
|
||||
"padding": {"before": [0, 0], "after": [0, 0], "semantic": "none"},
|
||||
"shard_file_id": "shard-0",
|
||||
"shard_digest_ref": "shard-0",
|
||||
"fragment_digest": null,
|
||||
"replica": {"group_id": null, "member_index": 0, "member_count": 1},
|
||||
"partition_axes": [],
|
||||
"logical_tensor_id": "layer.weight",
|
||||
"tie_group_id": null,
|
||||
"shared_storage_id": null,
|
||||
"storage_locations": [
|
||||
{"kind": "logical_element_range", "element_offset": [0, 0], "element_extent": [2, 2]},
|
||||
{"kind": "megatron_rank_offsets", "global_offset": [0, 0], "rank_offsets": [[0, 0, 1]]},
|
||||
{"kind": "safetensors_byte_range", "begin": 0, "end": 16},
|
||||
{"kind": "hf_weight_map", "tensor_name": "layer.weight", "file_id": "shard-0"}
|
||||
],
|
||||
"evidence_refs": ["ev-producer"]
|
||||
}
|
||||
],
|
||||
"adapter_resolution": {
|
||||
"status": "matched",
|
||||
"adapter_id": "fixture-megatron",
|
||||
"adapter_version": "0.0.0-fixture",
|
||||
"capability_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"reason": "Neutral schema fixture only; not a product support claim."
|
||||
},
|
||||
"conversion_plan": {
|
||||
"eligible": true,
|
||||
"target_format": "hf_safetensors",
|
||||
"state_scope": "weights_only",
|
||||
"dropped_scopes": []
|
||||
},
|
||||
"resource_estimate": {
|
||||
"source_bytes": 16,
|
||||
"logical_bytes": 16,
|
||||
"output_bytes": 16,
|
||||
"temporary_bytes": 32,
|
||||
"peak_ram_bytes": 16,
|
||||
"peak_vram_bytes": 0,
|
||||
"file_count": 1,
|
||||
"tensor_count": 1,
|
||||
"shard_count": 1,
|
||||
"assumptions": ["neutral fixture"],
|
||||
"confidence": "verified",
|
||||
"admission": "admitted"
|
||||
},
|
||||
"support_decision": "conversion_supported",
|
||||
"blockers": []
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_id": "obliteratus.conversion-manifest",
|
||||
"schema_version": "1.0.0",
|
||||
"manifest_id": "fixture-manifest-weights-only",
|
||||
"descriptor": {
|
||||
"schema_version": "1.0.0",
|
||||
"digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
},
|
||||
"source_inventory_digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333",
|
||||
"source_files": [
|
||||
{
|
||||
"relative_path": "rank-0/model.safetensors",
|
||||
"size_bytes": 16,
|
||||
"sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||
}
|
||||
],
|
||||
"adapter": {
|
||||
"id": "fixture-adapter",
|
||||
"version": "0.0.0-fixture",
|
||||
"capability_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"producer": "fixture",
|
||||
"producer_version": "0"
|
||||
},
|
||||
"source_topology": {"topology_id": "saved-topology"},
|
||||
"state": {
|
||||
"source_classification": "model_and_optimizer",
|
||||
"output_classification": "weights_only",
|
||||
"observed_scopes": ["model_weights", "optimizer"]
|
||||
},
|
||||
"dropped_scopes": [
|
||||
{"scope": "optimizer", "reason": "Version 1 canonical output is model weights only."}
|
||||
],
|
||||
"canonical_output": {
|
||||
"format": "hf_safetensors",
|
||||
"dtype_policy": "preserve",
|
||||
"files": [
|
||||
{
|
||||
"relative_path": "model.safetensors",
|
||||
"size_bytes": 16,
|
||||
"sha256": "sha256:4444444444444444444444444444444444444444444444444444444444444444"
|
||||
}
|
||||
],
|
||||
"hf_index": null,
|
||||
"logical_tensor_count": 1,
|
||||
"logical_bytes": 16
|
||||
},
|
||||
"resource_usage": {
|
||||
"estimated_peak_ram_bytes": 16,
|
||||
"actual_peak_ram_bytes": 16,
|
||||
"estimated_temp_bytes": 32,
|
||||
"actual_temp_bytes": 16
|
||||
},
|
||||
"validation": {
|
||||
"coverage": true,
|
||||
"replicas": true,
|
||||
"ties": true,
|
||||
"hashes": true,
|
||||
"index": true,
|
||||
"safe_reload": true,
|
||||
"source_unchanged": true,
|
||||
"result": "passed"
|
||||
},
|
||||
"provenance": {
|
||||
"obliteratus_commit": "e39f908832405ccad89cb2a5111e7c2576741d94",
|
||||
"configuration_digest": null,
|
||||
"tokenizer_digest": null,
|
||||
"base_model": {"identity": null, "revision": null, "digest": null},
|
||||
"transformation_log": ["neutral fixture canonicalization"],
|
||||
"unknowns": ["base model identity intentionally absent in neutral fixture"]
|
||||
},
|
||||
"publication": {
|
||||
"staging_validated": true,
|
||||
"promoted": true,
|
||||
"atomic_strategy": "sibling-staging-and-rename",
|
||||
"rollback_result": "not_required"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/peft-adapter-manifest-v1.schema.json",
|
||||
"title": "OBLITERATUS PEFT Adapter Manifest v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_id", "schema_version", "adapter_name", "adapter_type", "adapter_format_version", "base_model", "rank", "alpha", "scaling", "dropout", "bias", "modules_to_save", "target_modules", "tie_policy", "merged", "key_map_digest", "model_card_digest", "key_map"],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.peft-adapter-manifest"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"adapter_name": {"$ref": "#/$defs/id"},
|
||||
"adapter_type": {"const": "lora"},
|
||||
"adapter_format_version": {"const": "peft-lora-v1"},
|
||||
"base_model": {"$ref": "#/$defs/baseModel"},
|
||||
"rank": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807},
|
||||
"alpha": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807},
|
||||
"scaling": {"type": "number", "exclusiveMinimum": 0},
|
||||
"dropout": {"type": "number", "minimum": 0, "exclusiveMaximum": 1},
|
||||
"bias": {"const": "none"},
|
||||
"modules_to_save": {"type": "array", "maxItems": 0},
|
||||
"target_modules": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}},
|
||||
"tie_policy": {"const": "base_model_declared"},
|
||||
"merged": {"const": false},
|
||||
"key_map_digest": {"$ref": "#/$defs/digest"},
|
||||
"model_card_digest": {"$ref": "#/$defs/digest"},
|
||||
"key_map": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/keyMap"}}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"shape": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}},
|
||||
"baseModel": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repo_id", "revision", "weights_digest", "tokenizer_digest", "vocab_size", "architecture", "tied_embeddings"],
|
||||
"properties": {
|
||||
"repo_id": {"$ref": "#/$defs/id"},
|
||||
"revision": {"type": "string", "pattern": "^[0-9a-f]{40,64}$"},
|
||||
"weights_digest": {"$ref": "#/$defs/digest"},
|
||||
"tokenizer_digest": {"$ref": "#/$defs/digest"},
|
||||
"vocab_size": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807},
|
||||
"architecture": {"$ref": "#/$defs/id"},
|
||||
"tied_embeddings": {"type": "boolean"}
|
||||
}
|
||||
},
|
||||
"keyMap": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["module_name", "target_module", "lora_A_key", "lora_B_key", "lora_A_shape", "lora_B_shape", "rank"],
|
||||
"properties": {
|
||||
"module_name": {"$ref": "#/$defs/id"},
|
||||
"target_module": {"$ref": "#/$defs/id"},
|
||||
"lora_A_key": {"$ref": "#/$defs/id"},
|
||||
"lora_B_key": {"$ref": "#/$defs/id"},
|
||||
"lora_A_shape": {"$ref": "#/$defs/shape"},
|
||||
"lora_B_shape": {"$ref": "#/$defs/shape"},
|
||||
"rank": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/support-matrix-v1.schema.json",
|
||||
"title": "OBLITERATUS Checkpoint and Runtime Support Matrix v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_id", "schema_version", "generated_from", "status_vocabulary", "rows"],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.checkpoint-support-matrix"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"generated_from": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"status_vocabulary": {
|
||||
"type": "array",
|
||||
"const": ["supported", "conditional", "deferred", "out_of_scope"]
|
||||
},
|
||||
"rows": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"$ref": "#/$defs/row"}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"status": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["value", "basis"],
|
||||
"properties": {
|
||||
"value": {"enum": ["supported", "conditional", "deferred", "out_of_scope"]},
|
||||
"basis": {"type": "string", "minLength": 1, "maxLength": 2048}
|
||||
}
|
||||
},
|
||||
"evidence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["references", "candidate_commit", "fixture_digest", "environment", "topology", "retained_result"],
|
||||
"properties": {
|
||||
"references": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/id"}},
|
||||
"candidate_commit": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}$"},
|
||||
"fixture_digest": {"type": ["string", "null"], "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"environment": {"type": ["string", "null"], "maxLength": 1024},
|
||||
"topology": {"type": ["string", "null"], "maxLength": 1024},
|
||||
"retained_result": {"type": ["string", "null"], "maxLength": 2048}
|
||||
}
|
||||
},
|
||||
"row": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"id",
|
||||
"subject",
|
||||
"format",
|
||||
"producer_versions",
|
||||
"adapter",
|
||||
"model_mapping",
|
||||
"state_scopes",
|
||||
"safety_level",
|
||||
"optional_extras",
|
||||
"capabilities",
|
||||
"canonical_output",
|
||||
"evidence",
|
||||
"limits"
|
||||
],
|
||||
"properties": {
|
||||
"id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$"},
|
||||
"subject": {"$ref": "#/$defs/id"},
|
||||
"format": {"$ref": "#/$defs/id"},
|
||||
"producer_versions": {"type": "array", "items": {"$ref": "#/$defs/id"}},
|
||||
"adapter": {"type": ["string", "null"], "maxLength": 512},
|
||||
"model_mapping": {"type": "string", "maxLength": 2048},
|
||||
"state_scopes": {"type": "array", "items": {"$ref": "#/$defs/id"}},
|
||||
"safety_level": {"enum": ["ordinary_hf_load", "safe_structure", "trusted_metadata", "not_applicable"]},
|
||||
"optional_extras": {"type": "array", "items": {"$ref": "#/$defs/id"}},
|
||||
"capabilities": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["detect", "safe_inspect", "trusted_inspect", "weights_canonicalize", "topology_reshard", "surgery", "exact_resume", "live_multi_node"],
|
||||
"properties": {
|
||||
"detect": {"$ref": "#/$defs/status"},
|
||||
"safe_inspect": {"$ref": "#/$defs/status"},
|
||||
"trusted_inspect": {"$ref": "#/$defs/status"},
|
||||
"weights_canonicalize": {"$ref": "#/$defs/status"},
|
||||
"topology_reshard": {"$ref": "#/$defs/status"},
|
||||
"surgery": {"$ref": "#/$defs/status"},
|
||||
"exact_resume": {"$ref": "#/$defs/status"},
|
||||
"live_multi_node": {"$ref": "#/$defs/status"}
|
||||
}
|
||||
},
|
||||
"canonical_output": {"type": ["string", "null"], "maxLength": 512},
|
||||
"evidence": {"$ref": "#/$defs/evidence"},
|
||||
"limits": {"type": "array", "minItems": 1, "items": {"type": "string", "maxLength": 2048}}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://obliteratus.dev/schemas/trusted-worker-message-v1.schema.json",
|
||||
"title": "OBLITERATUS Trusted Worker Message v1",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_id", "schema_version", "message_type", "sequence", "source_inventory_digest", "adapter_digest", "policy_digest", "payload"],
|
||||
"properties": {
|
||||
"schema_id": {"const": "obliteratus.trusted-worker-message"},
|
||||
"schema_version": {"const": "1.0.0"},
|
||||
"message_type": {"enum": ["metadata_record", "tensor_fragment_record", "error", "complete"]},
|
||||
"sequence": {"type": "integer", "minimum": 0, "maximum": 100000000},
|
||||
"source_inventory_digest": {"$ref": "#/$defs/digest"},
|
||||
"adapter_digest": {"$ref": "#/$defs/digest"},
|
||||
"policy_digest": {"$ref": "#/$defs/digest"},
|
||||
"payload": {"type": "object"}
|
||||
},
|
||||
"allOf": [
|
||||
{"if": {"properties": {"message_type": {"const": "metadata_record"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/metadata"}}}},
|
||||
{"if": {"properties": {"message_type": {"const": "tensor_fragment_record"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/fragment"}}}},
|
||||
{"if": {"properties": {"message_type": {"const": "error"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/error"}}}},
|
||||
{"if": {"properties": {"message_type": {"const": "complete"}}}, "then": {"properties": {"payload": {"$ref": "#/$defs/complete"}}}}
|
||||
],
|
||||
"$defs": {
|
||||
"id": {"type": "string", "minLength": 1, "maxLength": 512},
|
||||
"digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["record_id", "record_type", "subject", "value", "evidence_refs"],
|
||||
"properties": {
|
||||
"record_id": {"$ref": "#/$defs/id"},
|
||||
"record_type": {"enum": ["producer", "topology", "state_scope", "tensor_metadata"]},
|
||||
"subject": {"type": "string", "pattern": "^/", "maxLength": 1024},
|
||||
"value": {"type": ["string", "integer", "boolean", "null"], "maxLength": 4096},
|
||||
"evidence_refs": {"type": "array", "maxItems": 1024, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}}
|
||||
}
|
||||
},
|
||||
"fragment": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["fragment_id", "tensor_name", "dtype", "shape", "source_file_id", "source_byte_offset", "source_byte_length", "source_digest", "logical_axis", "logical_start", "logical_end"],
|
||||
"properties": {
|
||||
"fragment_id": {"$ref": "#/$defs/id"},
|
||||
"tensor_name": {"$ref": "#/$defs/id"},
|
||||
"dtype": {"$ref": "#/$defs/id"},
|
||||
"shape": {"type": "array", "maxItems": 64, "items": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}},
|
||||
"source_file_id": {"$ref": "#/$defs/id"},
|
||||
"source_byte_offset": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807},
|
||||
"source_byte_length": {"type": "integer", "minimum": 1, "maximum": 9223372036854775807},
|
||||
"source_digest": {"$ref": "#/$defs/digest"},
|
||||
"logical_axis": {"type": ["integer", "null"], "minimum": 0, "maximum": 63},
|
||||
"logical_start": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807},
|
||||
"logical_end": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807}
|
||||
}
|
||||
},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["code", "phase", "category", "subject_ids", "limit", "actual"],
|
||||
"properties": {
|
||||
"code": {"type": "string", "pattern": "^DCI_[A-Z0-9_]+$"},
|
||||
"phase": {"enum": ["policy", "preflight", "reader", "protocol", "validation", "materialization", "promotion", "evidence", "cleanup"]},
|
||||
"category": {"enum": ["unsupported", "trust", "source", "runtime", "resource", "protocol", "validation", "output", "evidence", "cleanup"]},
|
||||
"subject_ids": {"type": "array", "maxItems": 128, "uniqueItems": true, "items": {"$ref": "#/$defs/id"}},
|
||||
"limit": {"type": ["integer", "null"], "minimum": 0},
|
||||
"actual": {"type": ["integer", "null"], "minimum": 0}
|
||||
}
|
||||
},
|
||||
"complete": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["record_count", "encoded_bytes", "transcript_digest"],
|
||||
"properties": {
|
||||
"record_count": {"type": "integer", "minimum": 0, "maximum": 100000000},
|
||||
"encoded_bytes": {"type": "integer", "minimum": 0, "maximum": 9223372036854775807},
|
||||
"transcript_digest": {"$ref": "#/$defs/digest"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user