fix: make supervised pipeline timeout configurable (#185)

app.py

@@ -262,6 +262,23 @@ def _cancel_pipeline_worker(
     return not worker.is_alive()
 
 
+def _pipeline_timeout_seconds() -> int:
+    """Return the validated whole-pipeline timeout for supervised runs."""
+
+    raw = os.environ.get("OBLITERATUS_PIPELINE_TIMEOUT_SECONDS", "2700")
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise ValueError(
+            "OBLITERATUS_PIPELINE_TIMEOUT_SECONDS must be an integer"
+        ) from exc
+    if not 60 <= value <= 86_400:
+        raise ValueError(
+            "OBLITERATUS_PIPELINE_TIMEOUT_SECONDS must be between 60 and 86400"
+        )
+    return value
+
+
 def _cleanup_failed_pipeline(pipeline, output_dir: str) -> None:
     """Release failed model state and remove only this UI run's final artifact."""
     if pipeline is not None:
@@ -2552,8 +2569,11 @@ def obliterate(model_choice: str, method_choice: str,
     worker = threading.Thread(target=run_pipeline, daemon=True)
     worker.start()
 
-    # Stream log updates while pipeline runs (max 45 minutes to prevent indefinite hang)
-    _max_pipeline_secs = 45 * 60
+    # Stream updates with a bounded, operator-configurable whole-pipeline
+    # timeout. Large checkpoints can spend most of the default merely loading
+    # when another host workload saturates storage.
+    _max_pipeline_secs = _pipeline_timeout_seconds()
+    _max_pipeline_minutes = _max_pipeline_secs / 60
     _pipeline_start = time.time()
     timed_out = False
     status_msg = "**Obliterating\u2026** (0s)"
@@ -2565,7 +2585,9 @@ def obliterate(model_choice: str, method_choice: str,
         else:
             yield status_msg, "\n".join(log_lines), gr.update(), gr.update(), gr.update(), gr.update()
         if time.time() - _pipeline_start > _max_pipeline_secs:
-            log_lines.append("\nTIMEOUT: Pipeline exceeded 45-minute limit.")
+            log_lines.append(
+                f"\nTIMEOUT: Pipeline exceeded {_max_pipeline_minutes:g}-minute limit."
+            )
             cancellation.set()
             timed_out = True
             break
@@ -2594,7 +2616,9 @@ def obliterate(model_choice: str, method_choice: str,
         log_lines.append(detail)
         run_archive.fail(
             run_id,
-            TimeoutError("pipeline exceeded the 45-minute limit"),
+            TimeoutError(
+                f"pipeline exceeded the {_max_pipeline_minutes:g}-minute limit"
+            ),
             phase="timeout",
         )
         _state["log"] = log_lines

deployment/systemd/obliteratus.service.d/70-pipeline-timeout.conf

@@ -0,0 +1,4 @@
+[Service]
+# Whole-pipeline budget for large checkpoints on the shared A100 host.
+# Resource admission, memory limits, and GPU lifecycle timeouts remain separate.
+Environment=OBLITERATUS_PIPELINE_TIMEOUT_SECONDS=7200

docs/deployment/shared-gpu-host.md

@@ -112,6 +112,7 @@ RuntimeDirectoryMode=0770
 Environment=OBLITERATUS_GPU_LIFECYCLE_DIR=/run/obliteratus-gpu-lifecycle
 Environment=OBLITERATUS_GPU_HEARTBEAT_SECONDS=15
 Environment=OBLITERATUS_GPU_ADMISSION_TIMEOUT_SECONDS=120
+Environment=OBLITERATUS_PIPELINE_TIMEOUT_SECONDS=7200
 Environment=OBLITERATUS_RUN_ARCHIVE=/srv/obliteratus/run-archive
 ExecStart=/srv/obliteratus/current/.venv/bin/obliteratus ui --host 127.0.0.1
 ```

tests/test_app_model_lifecycle.py

@@ -13,10 +13,23 @@ def test_active_checkpoint_survives_cleanup_and_remains_chat_recoverable(tmp_pat
     """Exercise app state in isolation from Gradio's import-time worker sockets."""
     script = r'''
 import pathlib
+import os
 import sys
 
 import app
 
+assert app._pipeline_timeout_seconds() == 2700
+os.environ["OBLITERATUS_PIPELINE_TIMEOUT_SECONDS"] = "7200"
+assert app._pipeline_timeout_seconds() == 7200
+os.environ["OBLITERATUS_PIPELINE_TIMEOUT_SECONDS"] = "invalid"
+try:
+    app._pipeline_timeout_seconds()
+except ValueError as error:
+    assert "must be an integer" in str(error)
+else:
+    raise AssertionError("invalid pipeline timeout was accepted")
+os.environ.pop("OBLITERATUS_PIPELINE_TIMEOUT_SECONDS")
+
 theme = app.THEME.to_dict()["theme"]
 assert theme["body_background_fill"] != theme["body_background_fill_dark"]
 assert theme["body_text_color"] != theme["body_text_color_dark"]