fix: surface bounded GPU admission failures

This commit is contained in:
Joseph Magly
2026-08-28 10:42:41 -04:00
parent a920391ae2
commit 683b39ba53
6 changed files with 214 additions and 12 deletions
+50
View File
@@ -127,6 +127,56 @@ def test_admission_failures_never_enter_allocation(tmp_path, decision):
assert "allocation_started" not in events
def test_timeout_error_contains_safe_correlated_diagnostics(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path,
admission_timeout_seconds=0.02,
admission_poll_seconds=0.002,
run_id="diagnostic-run",
)
with pytest.raises(AdmissionError) as captured:
publisher.loading("org/model")
error = captured.value
assert error.reason == "ack_timeout"
assert error.run_id == "diagnostic-run"
assert error.request_event_id == "diagnostic-run:1"
assert error.elapsed_seconds is not None
assert error.timeout_seconds == 0.02
assert "No model weights were loaded or modified" in error.user_message()
assert "run_id=diagnostic-run" in error.diagnostic_message()
assert "lease" not in error.diagnostic_message()
def test_supervisor_denial_reason_is_preserved(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path,
admission_timeout_seconds=0.2,
admission_poll_seconds=0.002,
run_id="denied-run",
)
def deny():
while not (tmp_path / "current.json").exists():
time.sleep(0.001)
request = json.loads((tmp_path / "current.json").read_text())
(tmp_path / "ack.json").write_text(json.dumps({
"schema_version": 1,
"run_id": "denied-run",
"request_event_id": request["event_id"],
"decision": "deny",
"reason": "broker_admission_failed",
}))
threading.Thread(target=deny).start()
with pytest.raises(AdmissionError) as captured:
publisher.loading("org/model")
assert captured.value.reason == "broker_admission_failed"
assert "could not reserve" in captured.value.user_message()
def test_delayed_ack_blocks_until_granted(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path, admission_timeout_seconds=1, admission_poll_seconds=0.005, run_id="run",