From 683b39ba5333b3d731f5c93a4c759d38cb6c2b76 Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:42:41 -0400 Subject: [PATCH 1/2] fix: surface bounded GPU admission failures --- app.py | 23 +++- ci/test-risk-map.json | 1 + docs/deployment/shared-gpu-host.md | 6 + obliteratus/gpu_lifecycle.py | 128 +++++++++++++++++++-- tests/test_app_admission_error_contract.py | 18 +++ tests/test_gpu_lifecycle.py | 50 ++++++++ 6 files changed, 214 insertions(+), 12 deletions(-) create mode 100644 tests/test_app_admission_error_contract.py diff --git a/app.py b/app.py index d6e8d6c..46cfa9e 100644 --- a/app.py +++ b/app.py @@ -58,7 +58,7 @@ if "HF_HOME" not in os.environ: import gradio as gr import torch from obliteratus import device as dev -from obliteratus.gpu_lifecycle import from_environment, measure_torch_memory +from obliteratus.gpu_lifecycle import AdmissionError, from_environment, measure_torch_memory from obliteratus.benchmark_lifecycle import ( admit_benchmark, mark_benchmark_ready, @@ -2225,7 +2225,26 @@ def obliterate(model_choice: str, method_choice: str, # Admission must be granted before cleanup can touch the CUDA runtime or # the worker can enter any model-loading path. - _gpu_lifecycle.loading(model_id) + try: + _gpu_lifecycle.loading(model_id) + except AdmissionError as exc: + with _lock: + _state["status"] = "idle" + _state["model"] = None + _state["tokenizer"] = None + _state["log"] = [ + f"GPU ADMISSION FAILED: {exc.user_message()}", + f"Diagnostic: {exc.diagnostic_message()}", + ] + yield ( + f"**GPU admission failed:** {exc.user_message()}", + "\n".join(_state["log"]), + get_chat_header(), + gr.update(), + gr.update(), + gr.update(), + ) + return _clear_gpu(release_lifecycle=False) with _lock: diff --git a/ci/test-risk-map.json b/ci/test-risk-map.json index 1661207..c3c485f 100644 --- a/ci/test-risk-map.json +++ b/ci/test-risk-map.json @@ -34,6 +34,7 @@ ], "required_tests": [ "tests/test_app_vram.py", + "tests/test_app_admission_error_contract.py", "tests/test_app_model_lifecycle.py", "tests/test_app_model_load_settings.py", "tests/test_model_load_settings.py", diff --git a/docs/deployment/shared-gpu-host.md b/docs/deployment/shared-gpu-host.md index 3ff2b5f..63f740d 100644 --- a/docs/deployment/shared-gpu-host.md +++ b/docs/deployment/shared-gpu-host.md @@ -64,6 +64,12 @@ identity, or an invalid grant fails closed. Subsequent events include `resize`, `ready`, `heartbeat`, and `release`, with run/model identity, a monotonic sequence, process ID, timestamp, and measured allocator bytes. +The host supervisor must finish its broker operation before the application's +admission deadline. Reserve a safety margin for publishing a correlated denial +and cleaning up a broker request that times out. Admission errors shown in the +UI include safe run/event identifiers and deadline details; the service journal +records the matching request, decision, and duration without lease credentials. + The supervisor acknowledgment schema is: ```json diff --git a/obliteratus/gpu_lifecycle.py b/obliteratus/gpu_lifecycle.py index e6b3bcd..6b0ee91 100644 --- a/obliteratus/gpu_lifecycle.py +++ b/obliteratus/gpu_lifecycle.py @@ -6,19 +6,67 @@ import atexit from dataclasses import dataclass from datetime import datetime, timezone import json +import logging import os from pathlib import Path import threading +import time import uuid from typing import Callable SCHEMA_VERSION = 1 +logger = logging.getLogger(__name__) class AdmissionError(RuntimeError): """The local supervisor did not grant correlated GPU admission.""" + def __init__( + self, + message: str, + *, + reason: str = "admission_failed", + run_id: str | None = None, + request_event_id: str | None = None, + elapsed_seconds: float | None = None, + timeout_seconds: float | None = None, + ) -> None: + super().__init__(message) + self.reason = reason + self.run_id = run_id + self.request_event_id = request_event_id + self.elapsed_seconds = elapsed_seconds + self.timeout_seconds = timeout_seconds + + def diagnostic_message(self) -> str: + """Return operator-safe correlated details without lease credentials.""" + fields = [f"reason={self.reason}"] + if self.run_id: + fields.append(f"run_id={self.run_id}") + if self.request_event_id: + fields.append(f"request_event_id={self.request_event_id}") + if self.elapsed_seconds is not None: + fields.append(f"elapsed={self.elapsed_seconds:.1f}s") + if self.timeout_seconds is not None: + fields.append(f"deadline={self.timeout_seconds:.1f}s") + return ", ".join(fields) + + def user_message(self) -> str: + """Return an actionable UI message for a safe admission failure.""" + if self.reason == "ack_timeout": + return ( + "GPU admission timed out before the host supervisor responded. " + "No model weights were loaded or modified. The GPU broker may still " + "be draining another workload; retry after it becomes available." + ) + if self.reason.startswith("broker_admission_"): + return ( + "The host GPU supervisor denied this run because it could not reserve " + "the requested GPU capacity. No model weights were loaded or modified." + ) + return f"GPU admission failed: {self}" + @dataclass(frozen=True) class MemoryUsage: @@ -84,10 +132,7 @@ class GpuLifecyclePublisher: def resize(self, memory: MemoryUsage) -> dict | None: with self._lock: - if ( - self.enabled - and memory.reserved_bytes > self._granted_vram_bytes - ): + if self.enabled and memory.reserved_bytes > self._granted_vram_bytes: self._phase = "admission_lost" self._publish("admission_lost", reason="reservation_exceeded") raise AdmissionError("measured VRAM exceeds the supervisor grant") @@ -126,6 +171,13 @@ class GpuLifecyclePublisher: acknowledgement = self._dir / "ack.json" # type: ignore[operator] deadline = threading.Event() remaining = self._admission_timeout_seconds + started_at = time.monotonic() + logger.info( + "GPU admission requested run_id=%s request_event_id=%s timeout=%.1fs", + self._run_id, + request["event_id"], + self._admission_timeout_seconds, + ) while remaining > 0: started = datetime.now(timezone.utc) try: @@ -134,19 +186,65 @@ class GpuLifecyclePublisher: payload = None if self._valid_ack(payload, request): if payload["decision"] == "deny": + reason = str(payload.get("reason", "denied")) self._phase = "admission_denied" - self._publish("admission_denied", reason=str(payload.get("reason", "denied"))) - raise AdmissionError("GPU admission denied by local supervisor") + self._publish("admission_denied", reason=reason) + elapsed = time.monotonic() - started_at + logger.warning( + "GPU admission denied run_id=%s request_event_id=%s " + "reason=%s elapsed=%.1fs", + self._run_id, + request["event_id"], + reason, + elapsed, + ) + raise AdmissionError( + "GPU admission denied by local supervisor", + reason=reason, + run_id=self._run_id, + request_event_id=request["event_id"], + elapsed_seconds=elapsed, + timeout_seconds=self._admission_timeout_seconds, + ) lease_id = payload.get("lease_id") granted = payload.get("granted_vram_bytes") if not isinstance(lease_id, str) or not lease_id: - raise AdmissionError("GPU admission ACK has no lease identity") + raise AdmissionError( + "GPU admission ACK has no lease identity", + reason="invalid_lease_identity", + run_id=self._run_id, + request_event_id=request["event_id"], + elapsed_seconds=time.monotonic() - started_at, + timeout_seconds=self._admission_timeout_seconds, + ) if self._lease_id is not None and lease_id != self._lease_id: - raise AdmissionError("GPU admission ownership changed") + raise AdmissionError( + "GPU admission ownership changed", + reason="lease_ownership_changed", + run_id=self._run_id, + request_event_id=request["event_id"], + elapsed_seconds=time.monotonic() - started_at, + timeout_seconds=self._admission_timeout_seconds, + ) if isinstance(granted, bool) or not isinstance(granted, int) or granted < 0: - raise AdmissionError("GPU admission ACK has an invalid VRAM grant") + raise AdmissionError( + "GPU admission ACK has an invalid VRAM grant", + reason="invalid_vram_grant", + run_id=self._run_id, + request_event_id=request["event_id"], + elapsed_seconds=time.monotonic() - started_at, + timeout_seconds=self._admission_timeout_seconds, + ) self._lease_id = lease_id self._granted_vram_bytes = granted + logger.info( + "GPU admission granted run_id=%s request_event_id=%s " + "elapsed=%.1fs granted_vram_bytes=%d", + self._run_id, + request["event_id"], + time.monotonic() - started_at, + granted, + ) return waited = (datetime.now(timezone.utc) - started).total_seconds() pause = min(self._admission_poll_seconds, remaining) @@ -154,7 +252,17 @@ class GpuLifecyclePublisher: remaining -= max(pause, waited) self._phase = "admission_denied" self._publish("admission_denied", reason="ack_timeout") - raise AdmissionError("timed out waiting for GPU admission ACK") + elapsed = time.monotonic() - started_at + error = AdmissionError( + "timed out waiting for GPU admission ACK", + reason="ack_timeout", + run_id=self._run_id, + request_event_id=request["event_id"], + elapsed_seconds=elapsed, + timeout_seconds=self._admission_timeout_seconds, + ) + logger.error("GPU admission timeout %s", error.diagnostic_message()) + raise error def _valid_ack(self, payload: object, request: dict) -> bool: return bool( diff --git a/tests/test_app_admission_error_contract.py b/tests/test_app_admission_error_contract.py new file mode 100644 index 0000000..3cf3f25 --- /dev/null +++ b/tests/test_app_admission_error_contract.py @@ -0,0 +1,18 @@ +"""Static UI contracts for actionable GPU admission failures.""" + +from pathlib import Path + + +def test_obliterate_catches_admission_before_cuda_cleanup(): + source = Path("app.py").read_text(encoding="utf-8") + obliterate = source.index("def obliterate(") + loading = source.index("_gpu_lifecycle.loading(model_id)", obliterate) + handler = source.index("except AdmissionError as exc:", loading) + cleanup = source.index("_clear_gpu(release_lifecycle=False)", handler) + + assert obliterate < loading < handler < cleanup + block = source[loading:cleanup] + assert "exc.user_message()" in block + assert "exc.diagnostic_message()" in block + assert '_state["status"] = "idle"' in block + assert "return" in block diff --git a/tests/test_gpu_lifecycle.py b/tests/test_gpu_lifecycle.py index 8a5343b..2216012 100644 --- a/tests/test_gpu_lifecycle.py +++ b/tests/test_gpu_lifecycle.py @@ -127,6 +127,56 @@ def test_admission_failures_never_enter_allocation(tmp_path, decision): assert "allocation_started" not in events +def test_timeout_error_contains_safe_correlated_diagnostics(tmp_path): + publisher = GpuLifecyclePublisher( + tmp_path, + admission_timeout_seconds=0.02, + admission_poll_seconds=0.002, + run_id="diagnostic-run", + ) + + with pytest.raises(AdmissionError) as captured: + publisher.loading("org/model") + + error = captured.value + assert error.reason == "ack_timeout" + assert error.run_id == "diagnostic-run" + assert error.request_event_id == "diagnostic-run:1" + assert error.elapsed_seconds is not None + assert error.timeout_seconds == 0.02 + assert "No model weights were loaded or modified" in error.user_message() + assert "run_id=diagnostic-run" in error.diagnostic_message() + assert "lease" not in error.diagnostic_message() + + +def test_supervisor_denial_reason_is_preserved(tmp_path): + publisher = GpuLifecyclePublisher( + tmp_path, + admission_timeout_seconds=0.2, + admission_poll_seconds=0.002, + run_id="denied-run", + ) + + def deny(): + while not (tmp_path / "current.json").exists(): + time.sleep(0.001) + request = json.loads((tmp_path / "current.json").read_text()) + (tmp_path / "ack.json").write_text(json.dumps({ + "schema_version": 1, + "run_id": "denied-run", + "request_event_id": request["event_id"], + "decision": "deny", + "reason": "broker_admission_failed", + })) + + threading.Thread(target=deny).start() + with pytest.raises(AdmissionError) as captured: + publisher.loading("org/model") + + assert captured.value.reason == "broker_admission_failed" + assert "could not reserve" in captured.value.user_message() + + def test_delayed_ack_blocks_until_granted(tmp_path): publisher = GpuLifecyclePublisher( tmp_path, admission_timeout_seconds=1, admission_poll_seconds=0.005, run_id="run", From 089c6eb5c02b9dc03479fce3893ba95e566a3b4f Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:46:24 -0400 Subject: [PATCH 2/2] docs: clarify shared-host admission budget --- docs/deployment/shared-gpu-host.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/deployment/shared-gpu-host.md b/docs/deployment/shared-gpu-host.md index 63f740d..0202db2 100644 --- a/docs/deployment/shared-gpu-host.md +++ b/docs/deployment/shared-gpu-host.md @@ -95,7 +95,10 @@ new admission. Readiness occurs after model allocation, never at UI startup. Set `OBLITERATUS_GPU_HEARTBEAT_SECONDS` to change the default 15-second heartbeat, and `OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS` to change the default 30-second -fail-closed acknowledgment timeout. +fail-closed acknowledgment timeout. Shared hosts that must drain another model +server should use a larger measured deadline (120 seconds in the example below) +and require the supervisor's broker timeout plus heartbeat handshake to finish +several seconds before it. A minimal systemd setup lets the supervisor create the local boundary without giving the application scheduling authority: @@ -108,7 +111,7 @@ RuntimeDirectory=obliteratus-gpu-lifecycle RuntimeDirectoryMode=0770 Environment=OBLITERATUS_GPU_LIFECYCLE_DIR=/run/obliteratus-gpu-lifecycle Environment=OBLITERATUS_GPU_HEARTBEAT_SECONDS=15 -Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=30 +Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=120 ExecStart=/srv/obliteratus/current/.venv/bin/obliteratus ui --host 127.0.0.1 ```