fix: surface bounded GPU admission failures

This commit is contained in:
Joseph Magly
2026-08-28 10:42:41 -04:00
parent a920391ae2
commit 683b39ba53
6 changed files with 214 additions and 12 deletions
+21 -2
View File
@@ -58,7 +58,7 @@ if "HF_HOME" not in os.environ:
import gradio as gr
import torch
from obliteratus import device as dev
from obliteratus.gpu_lifecycle import from_environment, measure_torch_memory
from obliteratus.gpu_lifecycle import AdmissionError, from_environment, measure_torch_memory
from obliteratus.benchmark_lifecycle import (
admit_benchmark,
mark_benchmark_ready,
@@ -2225,7 +2225,26 @@ def obliterate(model_choice: str, method_choice: str,
# Admission must be granted before cleanup can touch the CUDA runtime or
# the worker can enter any model-loading path.
_gpu_lifecycle.loading(model_id)
try:
_gpu_lifecycle.loading(model_id)
except AdmissionError as exc:
with _lock:
_state["status"] = "idle"
_state["model"] = None
_state["tokenizer"] = None
_state["log"] = [
f"GPU ADMISSION FAILED: {exc.user_message()}",
f"Diagnostic: {exc.diagnostic_message()}",
]
yield (
f"**GPU admission failed:** {exc.user_message()}",
"\n".join(_state["log"]),
get_chat_header(),
gr.update(),
gr.update(),
gr.update(),
)
return
_clear_gpu(release_lifecycle=False)
with _lock: