Merge pull request #181 from jmagly/fix/180-bounded-gpu-admission-errors

fix: surface bounded GPU admission failures

app.py

@@ -58,7 +58,7 @@ if "HF_HOME" not in os.environ:
 import gradio as gr
 import torch
 from obliteratus import device as dev
-from obliteratus.gpu_lifecycle import from_environment, measure_torch_memory
+from obliteratus.gpu_lifecycle import AdmissionError, from_environment, measure_torch_memory
 from obliteratus.benchmark_lifecycle import (
     admit_benchmark,
     mark_benchmark_ready,
@@ -2225,7 +2225,26 @@ def obliterate(model_choice: str, method_choice: str,
 
     # Admission must be granted before cleanup can touch the CUDA runtime or
     # the worker can enter any model-loading path.
-    _gpu_lifecycle.loading(model_id)
+    try:
+        _gpu_lifecycle.loading(model_id)
+    except AdmissionError as exc:
+        with _lock:
+            _state["status"] = "idle"
+            _state["model"] = None
+            _state["tokenizer"] = None
+            _state["log"] = [
+                f"GPU ADMISSION FAILED: {exc.user_message()}",
+                f"Diagnostic: {exc.diagnostic_message()}",
+            ]
+        yield (
+            f"**GPU admission failed:** {exc.user_message()}",
+            "\n".join(_state["log"]),
+            get_chat_header(),
+            gr.update(),
+            gr.update(),
+            gr.update(),
+        )
+        return
     _clear_gpu(release_lifecycle=False)
 
     with _lock:

ci/test-risk-map.json

@@ -34,6 +34,7 @@
       ],
       "required_tests": [
         "tests/test_app_vram.py",
+        "tests/test_app_admission_error_contract.py",
         "tests/test_app_model_lifecycle.py",
         "tests/test_app_model_load_settings.py",
         "tests/test_model_load_settings.py",

docs/deployment/shared-gpu-host.md

@@ -64,6 +64,12 @@ identity, or an invalid grant fails closed. Subsequent events include `resize`,
 `ready`, `heartbeat`, and `release`, with run/model identity, a monotonic
 sequence, process ID, timestamp, and measured allocator bytes.
 
+The host supervisor must finish its broker operation before the application's
+admission deadline. Reserve a safety margin for publishing a correlated denial
+and cleaning up a broker request that times out. Admission errors shown in the
+UI include safe run/event identifiers and deadline details; the service journal
+records the matching request, decision, and duration without lease credentials.
+
 The supervisor acknowledgment schema is:
 
 ```json
@@ -89,7 +95,10 @@ new admission.
 Readiness occurs after model allocation, never at UI startup. Set
 `OBLITERATUS_GPU_HEARTBEAT_SECONDS` to change the default 15-second heartbeat,
 and `OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS` to change the default 30-second
-fail-closed acknowledgment timeout.
+fail-closed acknowledgment timeout. Shared hosts that must drain another model
+server should use a larger measured deadline (120 seconds in the example below)
+and require the supervisor's broker timeout plus heartbeat handshake to finish
+several seconds before it.
 
 A minimal systemd setup lets the supervisor create the local boundary without
 giving the application scheduling authority:
@@ -102,7 +111,7 @@ RuntimeDirectory=obliteratus-gpu-lifecycle
 RuntimeDirectoryMode=0770
 Environment=OBLITERATUS_GPU_LIFECYCLE_DIR=/run/obliteratus-gpu-lifecycle
 Environment=OBLITERATUS_GPU_HEARTBEAT_SECONDS=15
-Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=30
+Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=120
 ExecStart=/srv/obliteratus/current/.venv/bin/obliteratus ui --host 127.0.0.1
 ```
 

obliteratus/gpu_lifecycle.py

@@ -6,19 +6,67 @@ import atexit
 from dataclasses import dataclass
 from datetime import datetime, timezone
 import json
+import logging
 import os
 from pathlib import Path
 import threading
+import time
 import uuid
 from typing import Callable
 
 
 SCHEMA_VERSION = 1
+logger = logging.getLogger(__name__)
 
 
 class AdmissionError(RuntimeError):
     """The local supervisor did not grant correlated GPU admission."""
 
+    def __init__(
+        self,
+        message: str,
+        *,
+        reason: str = "admission_failed",
+        run_id: str | None = None,
+        request_event_id: str | None = None,
+        elapsed_seconds: float | None = None,
+        timeout_seconds: float | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.reason = reason
+        self.run_id = run_id
+        self.request_event_id = request_event_id
+        self.elapsed_seconds = elapsed_seconds
+        self.timeout_seconds = timeout_seconds
+
+    def diagnostic_message(self) -> str:
+        """Return operator-safe correlated details without lease credentials."""
+        fields = [f"reason={self.reason}"]
+        if self.run_id:
+            fields.append(f"run_id={self.run_id}")
+        if self.request_event_id:
+            fields.append(f"request_event_id={self.request_event_id}")
+        if self.elapsed_seconds is not None:
+            fields.append(f"elapsed={self.elapsed_seconds:.1f}s")
+        if self.timeout_seconds is not None:
+            fields.append(f"deadline={self.timeout_seconds:.1f}s")
+        return ", ".join(fields)
+
+    def user_message(self) -> str:
+        """Return an actionable UI message for a safe admission failure."""
+        if self.reason == "ack_timeout":
+            return (
+                "GPU admission timed out before the host supervisor responded. "
+                "No model weights were loaded or modified. The GPU broker may still "
+                "be draining another workload; retry after it becomes available."
+            )
+        if self.reason.startswith("broker_admission_"):
+            return (
+                "The host GPU supervisor denied this run because it could not reserve "
+                "the requested GPU capacity. No model weights were loaded or modified."
+            )
+        return f"GPU admission failed: {self}"
+
 
 @dataclass(frozen=True)
 class MemoryUsage:
@@ -84,10 +132,7 @@ class GpuLifecyclePublisher:
 
     def resize(self, memory: MemoryUsage) -> dict | None:
         with self._lock:
-            if (
-                self.enabled
-                and memory.reserved_bytes > self._granted_vram_bytes
-            ):
+            if self.enabled and memory.reserved_bytes > self._granted_vram_bytes:
                 self._phase = "admission_lost"
                 self._publish("admission_lost", reason="reservation_exceeded")
                 raise AdmissionError("measured VRAM exceeds the supervisor grant")
@@ -126,6 +171,13 @@ class GpuLifecyclePublisher:
         acknowledgement = self._dir / "ack.json"  # type: ignore[operator]
         deadline = threading.Event()
         remaining = self._admission_timeout_seconds
+        started_at = time.monotonic()
+        logger.info(
+            "GPU admission requested run_id=%s request_event_id=%s timeout=%.1fs",
+            self._run_id,
+            request["event_id"],
+            self._admission_timeout_seconds,
+        )
         while remaining > 0:
             started = datetime.now(timezone.utc)
             try:
@@ -134,19 +186,65 @@ class GpuLifecyclePublisher:
                 payload = None
             if self._valid_ack(payload, request):
                 if payload["decision"] == "deny":
+                    reason = str(payload.get("reason", "denied"))
                     self._phase = "admission_denied"
-                    self._publish("admission_denied", reason=str(payload.get("reason", "denied")))
-                    raise AdmissionError("GPU admission denied by local supervisor")
+                    self._publish("admission_denied", reason=reason)
+                    elapsed = time.monotonic() - started_at
+                    logger.warning(
+                        "GPU admission denied run_id=%s request_event_id=%s "
+                        "reason=%s elapsed=%.1fs",
+                        self._run_id,
+                        request["event_id"],
+                        reason,
+                        elapsed,
+                    )
+                    raise AdmissionError(
+                        "GPU admission denied by local supervisor",
+                        reason=reason,
+                        run_id=self._run_id,
+                        request_event_id=request["event_id"],
+                        elapsed_seconds=elapsed,
+                        timeout_seconds=self._admission_timeout_seconds,
+                    )
                 lease_id = payload.get("lease_id")
                 granted = payload.get("granted_vram_bytes")
                 if not isinstance(lease_id, str) or not lease_id:
-                    raise AdmissionError("GPU admission ACK has no lease identity")
+                    raise AdmissionError(
+                        "GPU admission ACK has no lease identity",
+                        reason="invalid_lease_identity",
+                        run_id=self._run_id,
+                        request_event_id=request["event_id"],
+                        elapsed_seconds=time.monotonic() - started_at,
+                        timeout_seconds=self._admission_timeout_seconds,
+                    )
                 if self._lease_id is not None and lease_id != self._lease_id:
-                    raise AdmissionError("GPU admission ownership changed")
+                    raise AdmissionError(
+                        "GPU admission ownership changed",
+                        reason="lease_ownership_changed",
+                        run_id=self._run_id,
+                        request_event_id=request["event_id"],
+                        elapsed_seconds=time.monotonic() - started_at,
+                        timeout_seconds=self._admission_timeout_seconds,
+                    )
                 if isinstance(granted, bool) or not isinstance(granted, int) or granted < 0:
-                    raise AdmissionError("GPU admission ACK has an invalid VRAM grant")
+                    raise AdmissionError(
+                        "GPU admission ACK has an invalid VRAM grant",
+                        reason="invalid_vram_grant",
+                        run_id=self._run_id,
+                        request_event_id=request["event_id"],
+                        elapsed_seconds=time.monotonic() - started_at,
+                        timeout_seconds=self._admission_timeout_seconds,
+                    )
                 self._lease_id = lease_id
                 self._granted_vram_bytes = granted
+                logger.info(
+                    "GPU admission granted run_id=%s request_event_id=%s "
+                    "elapsed=%.1fs granted_vram_bytes=%d",
+                    self._run_id,
+                    request["event_id"],
+                    time.monotonic() - started_at,
+                    granted,
+                )
                 return
             waited = (datetime.now(timezone.utc) - started).total_seconds()
             pause = min(self._admission_poll_seconds, remaining)
@@ -154,7 +252,17 @@ class GpuLifecyclePublisher:
             remaining -= max(pause, waited)
         self._phase = "admission_denied"
         self._publish("admission_denied", reason="ack_timeout")
-        raise AdmissionError("timed out waiting for GPU admission ACK")
+        elapsed = time.monotonic() - started_at
+        error = AdmissionError(
+            "timed out waiting for GPU admission ACK",
+            reason="ack_timeout",
+            run_id=self._run_id,
+            request_event_id=request["event_id"],
+            elapsed_seconds=elapsed,
+            timeout_seconds=self._admission_timeout_seconds,
+        )
+        logger.error("GPU admission timeout %s", error.diagnostic_message())
+        raise error
 
     def _valid_ack(self, payload: object, request: dict) -> bool:
         return bool(

tests/test_app_admission_error_contract.py

@@ -0,0 +1,18 @@
+"""Static UI contracts for actionable GPU admission failures."""
+
+from pathlib import Path
+
+
+def test_obliterate_catches_admission_before_cuda_cleanup():
+    source = Path("app.py").read_text(encoding="utf-8")
+    obliterate = source.index("def obliterate(")
+    loading = source.index("_gpu_lifecycle.loading(model_id)", obliterate)
+    handler = source.index("except AdmissionError as exc:", loading)
+    cleanup = source.index("_clear_gpu(release_lifecycle=False)", handler)
+
+    assert obliterate < loading < handler < cleanup
+    block = source[loading:cleanup]
+    assert "exc.user_message()" in block
+    assert "exc.diagnostic_message()" in block
+    assert '_state["status"] = "idle"' in block
+    assert "return" in block

tests/test_gpu_lifecycle.py

@@ -127,6 +127,56 @@ def test_admission_failures_never_enter_allocation(tmp_path, decision):
     assert "allocation_started" not in events
 
 
+def test_timeout_error_contains_safe_correlated_diagnostics(tmp_path):
+    publisher = GpuLifecyclePublisher(
+        tmp_path,
+        admission_timeout_seconds=0.02,
+        admission_poll_seconds=0.002,
+        run_id="diagnostic-run",
+    )
+
+    with pytest.raises(AdmissionError) as captured:
+        publisher.loading("org/model")
+
+    error = captured.value
+    assert error.reason == "ack_timeout"
+    assert error.run_id == "diagnostic-run"
+    assert error.request_event_id == "diagnostic-run:1"
+    assert error.elapsed_seconds is not None
+    assert error.timeout_seconds == 0.02
+    assert "No model weights were loaded or modified" in error.user_message()
+    assert "run_id=diagnostic-run" in error.diagnostic_message()
+    assert "lease" not in error.diagnostic_message()
+
+
+def test_supervisor_denial_reason_is_preserved(tmp_path):
+    publisher = GpuLifecyclePublisher(
+        tmp_path,
+        admission_timeout_seconds=0.2,
+        admission_poll_seconds=0.002,
+        run_id="denied-run",
+    )
+
+    def deny():
+        while not (tmp_path / "current.json").exists():
+            time.sleep(0.001)
+        request = json.loads((tmp_path / "current.json").read_text())
+        (tmp_path / "ack.json").write_text(json.dumps({
+            "schema_version": 1,
+            "run_id": "denied-run",
+            "request_event_id": request["event_id"],
+            "decision": "deny",
+            "reason": "broker_admission_failed",
+        }))
+
+    threading.Thread(target=deny).start()
+    with pytest.raises(AdmissionError) as captured:
+        publisher.loading("org/model")
+
+    assert captured.value.reason == "broker_admission_failed"
+    assert "could not reserve" in captured.value.user_message()
+
+
 def test_delayed_ack_blocks_until_granted(tmp_path):
     publisher = GpuLifecyclePublisher(
         tmp_path, admission_timeout_seconds=1, admission_poll_seconds=0.005, run_id="run",