fix: surface bounded GPU admission failures

This commit is contained in:
Joseph Magly
2026-08-28 10:42:41 -04:00
parent a920391ae2
commit 683b39ba53
6 changed files with 214 additions and 12 deletions
@@ -0,0 +1,18 @@
"""Static UI contracts for actionable GPU admission failures."""
from pathlib import Path
def test_obliterate_catches_admission_before_cuda_cleanup():
source = Path("app.py").read_text(encoding="utf-8")
obliterate = source.index("def obliterate(")
loading = source.index("_gpu_lifecycle.loading(model_id)", obliterate)
handler = source.index("except AdmissionError as exc:", loading)
cleanup = source.index("_clear_gpu(release_lifecycle=False)", handler)
assert obliterate < loading < handler < cleanup
block = source[loading:cleanup]
assert "exc.user_message()" in block
assert "exc.diagnostic_message()" in block
assert '_state["status"] = "idle"' in block
assert "return" in block
+50
View File
@@ -127,6 +127,56 @@ def test_admission_failures_never_enter_allocation(tmp_path, decision):
assert "allocation_started" not in events
def test_timeout_error_contains_safe_correlated_diagnostics(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path,
admission_timeout_seconds=0.02,
admission_poll_seconds=0.002,
run_id="diagnostic-run",
)
with pytest.raises(AdmissionError) as captured:
publisher.loading("org/model")
error = captured.value
assert error.reason == "ack_timeout"
assert error.run_id == "diagnostic-run"
assert error.request_event_id == "diagnostic-run:1"
assert error.elapsed_seconds is not None
assert error.timeout_seconds == 0.02
assert "No model weights were loaded or modified" in error.user_message()
assert "run_id=diagnostic-run" in error.diagnostic_message()
assert "lease" not in error.diagnostic_message()
def test_supervisor_denial_reason_is_preserved(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path,
admission_timeout_seconds=0.2,
admission_poll_seconds=0.002,
run_id="denied-run",
)
def deny():
while not (tmp_path / "current.json").exists():
time.sleep(0.001)
request = json.loads((tmp_path / "current.json").read_text())
(tmp_path / "ack.json").write_text(json.dumps({
"schema_version": 1,
"run_id": "denied-run",
"request_event_id": request["event_id"],
"decision": "deny",
"reason": "broker_admission_failed",
}))
threading.Thread(target=deny).start()
with pytest.raises(AdmissionError) as captured:
publisher.loading("org/model")
assert captured.value.reason == "broker_admission_failed"
assert "could not reserve" in captured.value.user_message()
def test_delayed_ack_blocks_until_granted(tmp_path):
publisher = GpuLifecyclePublisher(
tmp_path, admission_timeout_seconds=1, admission_poll_seconds=0.005, run_id="run",