mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-30 06:30:37 +02:00
fix: surface bounded GPU admission failures
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
"""Static UI contracts for actionable GPU admission failures."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_obliterate_catches_admission_before_cuda_cleanup():
|
||||
source = Path("app.py").read_text(encoding="utf-8")
|
||||
obliterate = source.index("def obliterate(")
|
||||
loading = source.index("_gpu_lifecycle.loading(model_id)", obliterate)
|
||||
handler = source.index("except AdmissionError as exc:", loading)
|
||||
cleanup = source.index("_clear_gpu(release_lifecycle=False)", handler)
|
||||
|
||||
assert obliterate < loading < handler < cleanup
|
||||
block = source[loading:cleanup]
|
||||
assert "exc.user_message()" in block
|
||||
assert "exc.diagnostic_message()" in block
|
||||
assert '_state["status"] = "idle"' in block
|
||||
assert "return" in block
|
||||
@@ -127,6 +127,56 @@ def test_admission_failures_never_enter_allocation(tmp_path, decision):
|
||||
assert "allocation_started" not in events
|
||||
|
||||
|
||||
def test_timeout_error_contains_safe_correlated_diagnostics(tmp_path):
|
||||
publisher = GpuLifecyclePublisher(
|
||||
tmp_path,
|
||||
admission_timeout_seconds=0.02,
|
||||
admission_poll_seconds=0.002,
|
||||
run_id="diagnostic-run",
|
||||
)
|
||||
|
||||
with pytest.raises(AdmissionError) as captured:
|
||||
publisher.loading("org/model")
|
||||
|
||||
error = captured.value
|
||||
assert error.reason == "ack_timeout"
|
||||
assert error.run_id == "diagnostic-run"
|
||||
assert error.request_event_id == "diagnostic-run:1"
|
||||
assert error.elapsed_seconds is not None
|
||||
assert error.timeout_seconds == 0.02
|
||||
assert "No model weights were loaded or modified" in error.user_message()
|
||||
assert "run_id=diagnostic-run" in error.diagnostic_message()
|
||||
assert "lease" not in error.diagnostic_message()
|
||||
|
||||
|
||||
def test_supervisor_denial_reason_is_preserved(tmp_path):
|
||||
publisher = GpuLifecyclePublisher(
|
||||
tmp_path,
|
||||
admission_timeout_seconds=0.2,
|
||||
admission_poll_seconds=0.002,
|
||||
run_id="denied-run",
|
||||
)
|
||||
|
||||
def deny():
|
||||
while not (tmp_path / "current.json").exists():
|
||||
time.sleep(0.001)
|
||||
request = json.loads((tmp_path / "current.json").read_text())
|
||||
(tmp_path / "ack.json").write_text(json.dumps({
|
||||
"schema_version": 1,
|
||||
"run_id": "denied-run",
|
||||
"request_event_id": request["event_id"],
|
||||
"decision": "deny",
|
||||
"reason": "broker_admission_failed",
|
||||
}))
|
||||
|
||||
threading.Thread(target=deny).start()
|
||||
with pytest.raises(AdmissionError) as captured:
|
||||
publisher.loading("org/model")
|
||||
|
||||
assert captured.value.reason == "broker_admission_failed"
|
||||
assert "could not reserve" in captured.value.user_message()
|
||||
|
||||
|
||||
def test_delayed_ack_blocks_until_granted(tmp_path):
|
||||
publisher = GpuLifecyclePublisher(
|
||||
tmp_path, admission_timeout_seconds=1, admission_poll_seconds=0.005, run_id="run",
|
||||
|
||||
Reference in New Issue
Block a user