diff --git a/app.py b/app.py index ef87722..de6ce08 100644 --- a/app.py +++ b/app.py @@ -262,6 +262,23 @@ def _cancel_pipeline_worker( return not worker.is_alive() +def _pipeline_timeout_seconds() -> int: + """Return the validated whole-pipeline timeout for supervised runs.""" + + raw = os.environ.get("OBLITERATUS_PIPELINE_TIMEOUT_SECONDS", "2700") + try: + value = int(raw) + except ValueError as exc: + raise ValueError( + "OBLITERATUS_PIPELINE_TIMEOUT_SECONDS must be an integer" + ) from exc + if not 60 <= value <= 86_400: + raise ValueError( + "OBLITERATUS_PIPELINE_TIMEOUT_SECONDS must be between 60 and 86400" + ) + return value + + def _cleanup_failed_pipeline(pipeline, output_dir: str) -> None: """Release failed model state and remove only this UI run's final artifact.""" if pipeline is not None: @@ -2552,8 +2569,11 @@ def obliterate(model_choice: str, method_choice: str, worker = threading.Thread(target=run_pipeline, daemon=True) worker.start() - # Stream log updates while pipeline runs (max 45 minutes to prevent indefinite hang) - _max_pipeline_secs = 45 * 60 + # Stream updates with a bounded, operator-configurable whole-pipeline + # timeout. Large checkpoints can spend most of the default merely loading + # when another host workload saturates storage. + _max_pipeline_secs = _pipeline_timeout_seconds() + _max_pipeline_minutes = _max_pipeline_secs / 60 _pipeline_start = time.time() timed_out = False status_msg = "**Obliterating\u2026** (0s)" @@ -2565,7 +2585,9 @@ def obliterate(model_choice: str, method_choice: str, else: yield status_msg, "\n".join(log_lines), gr.update(), gr.update(), gr.update(), gr.update() if time.time() - _pipeline_start > _max_pipeline_secs: - log_lines.append("\nTIMEOUT: Pipeline exceeded 45-minute limit.") + log_lines.append( + f"\nTIMEOUT: Pipeline exceeded {_max_pipeline_minutes:g}-minute limit." + ) cancellation.set() timed_out = True break @@ -2594,7 +2616,9 @@ def obliterate(model_choice: str, method_choice: str, log_lines.append(detail) run_archive.fail( run_id, - TimeoutError("pipeline exceeded the 45-minute limit"), + TimeoutError( + f"pipeline exceeded the {_max_pipeline_minutes:g}-minute limit" + ), phase="timeout", ) _state["log"] = log_lines diff --git a/deployment/systemd/obliteratus.service.d/70-pipeline-timeout.conf b/deployment/systemd/obliteratus.service.d/70-pipeline-timeout.conf new file mode 100644 index 0000000..c3f494b --- /dev/null +++ b/deployment/systemd/obliteratus.service.d/70-pipeline-timeout.conf @@ -0,0 +1,4 @@ +[Service] +# Whole-pipeline budget for large checkpoints on the shared A100 host. +# Resource admission, memory limits, and GPU lifecycle timeouts remain separate. +Environment=OBLITERATUS_PIPELINE_TIMEOUT_SECONDS=7200 diff --git a/docs/deployment/shared-gpu-host.md b/docs/deployment/shared-gpu-host.md index 39df30a..3de0d46 100644 --- a/docs/deployment/shared-gpu-host.md +++ b/docs/deployment/shared-gpu-host.md @@ -112,6 +112,7 @@ RuntimeDirectoryMode=0770 Environment=OBLITERATUS_GPU_LIFECYCLE_DIR=/run/obliteratus-gpu-lifecycle Environment=OBLITERATUS_GPU_HEARTBEAT_SECONDS=15 Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=120 +Environment=OBLITERATUS_PIPELINE_TIMEOUT_SECONDS=7200 Environment=OBLITERATUS_RUN_ARCHIVE=/srv/obliteratus/run-archive ExecStart=/srv/obliteratus/current/.venv/bin/obliteratus ui --host 127.0.0.1 ``` diff --git a/tests/test_app_model_lifecycle.py b/tests/test_app_model_lifecycle.py index 408a103..117e790 100644 --- a/tests/test_app_model_lifecycle.py +++ b/tests/test_app_model_lifecycle.py @@ -13,10 +13,23 @@ def test_active_checkpoint_survives_cleanup_and_remains_chat_recoverable(tmp_pat """Exercise app state in isolation from Gradio's import-time worker sockets.""" script = r''' import pathlib +import os import sys import app +assert app._pipeline_timeout_seconds() == 2700 +os.environ["OBLITERATUS_PIPELINE_TIMEOUT_SECONDS"] = "7200" +assert app._pipeline_timeout_seconds() == 7200 +os.environ["OBLITERATUS_PIPELINE_TIMEOUT_SECONDS"] = "invalid" +try: + app._pipeline_timeout_seconds() +except ValueError as error: + assert "must be an integer" in str(error) +else: + raise AssertionError("invalid pipeline timeout was accepted") +os.environ.pop("OBLITERATUS_PIPELINE_TIMEOUT_SECONDS") + theme = app.THEME.to_dict()["theme"] assert theme["body_background_fill"] != theme["body_background_fill_dark"] assert theme["body_text_color"] != theme["body_text_color_dark"]