feat: add durable reconnect-safe experiment runs (#184)

app.py

@@ -18,6 +18,7 @@ ZeroGPU Support:
 from __future__ import annotations
 
 import gc
+import hashlib
 import json as _json
 import os
 import re
@@ -379,48 +380,55 @@ def _persist_session_meta(output_dir: str, label: str, meta: dict) -> None:
 
 
 def _recover_sessions_from_disk() -> None:
-    """Scan /tmp for obliterated checkpoints and repopulate _session_models.
+    """Scan temporary and durable checkpoints and repopulate session models.
 
     Called on startup and when a stale dropdown value is detected.  Skips
     directories that are already registered.
     """
     global _last_obliterated_label, _obliterate_counter
     found_any = False
+    candidates = []
     for pattern in ("obliterated_*", "obliterated", "bench_*", "obliteratus_tourney/r*"):
-        for p in Path("/tmp").glob(pattern):
-            if not p.is_dir():
-                continue
-            meta_file = p / _SESSION_META_FILE
-            if not meta_file.exists():
-                continue
+        candidates.extend(Path("/tmp").glob(pattern))
+    try:
+        from obliteratus.run_archive import default_archive_root
+        candidates.extend(default_archive_root().glob("run-*/checkpoint"))
+    except (OSError, ValueError):
+        pass
+    for p in candidates:
+        if not p.is_dir():
+            continue
+        meta_file = p / _SESSION_META_FILE
+        if not meta_file.exists():
+            continue
+        try:
+            data = _json.loads(meta_file.read_text())
+        except Exception:
+            continue
+        label = data.get("label", p.name)
+        if label in _session_models:
+            continue  # already registered
+        _session_models[label] = {
+            "model_id": data.get("model_id", ""),
+            "model_choice": data.get("model_choice", data.get("model_id", "")),
+            "method": data.get("method", "unknown"),
+            "dataset_key": data.get("dataset_key", ""),
+            "prompt_volume": data.get("prompt_volume", 0),
+            "output_dir": str(p),
+            "source": data.get("source", "recovered"),
+            "load_settings": data.get("load_settings", {}),
+        }
+        found_any = True
+        # Track the latest for auto-select
+        _last_obliterated_label = label
+        # Keep counter above any existing numbered dirs
+        if p.name.startswith("obliterated_"):
             try:
-                data = _json.loads(meta_file.read_text())
-            except Exception:
+                idx = int(p.name.split("_", 1)[1])
+                if idx >= _obliterate_counter:
+                    _obliterate_counter = idx + 1
+            except (ValueError, IndexError):
                 continue
-            label = data.get("label", p.name)
-            if label in _session_models:
-                continue  # already registered
-            _session_models[label] = {
-                "model_id": data.get("model_id", ""),
-                "model_choice": data.get("model_choice", data.get("model_id", "")),
-                "method": data.get("method", "unknown"),
-                "dataset_key": data.get("dataset_key", ""),
-                "prompt_volume": data.get("prompt_volume", 0),
-                "output_dir": str(p),
-                "source": data.get("source", "recovered"),
-                "load_settings": data.get("load_settings", {}),
-            }
-            found_any = True
-            # Track the latest for auto-select
-            _last_obliterated_label = label
-            # Keep counter above any existing numbered dirs
-            if p.name.startswith("obliterated_"):
-                try:
-                    idx = int(p.name.split("_", 1)[1])
-                    if idx >= _obliterate_counter:
-                        _obliterate_counter = idx + 1
-                except (ValueError, IndexError):
-                    pass
     # If we recovered sessions but _state has no output_dir, set it to the
     # most recent checkpoint so chat_respond can reload from disk.
     if found_any and not _state.get("output_dir"):
@@ -2236,11 +2244,52 @@ def obliterate(model_choice: str, method_choice: str,
         _state["model_name"] = model_choice
         _state["method"] = method
 
+    from obliteratus.run_archive import RunArchive
+
+    run_config = {
+        key: value
+        for key, value in locals().copy().items()
+        if key.startswith("adv_")
+        or key in {
+            "model_choice", "model_id", "method_choice", "method",
+            "prompt_volume_choice", "prompt_volume", "dataset_source_choice",
+            "quantization_choice", "dtype_choice",
+        }
+    }
+    run_config["load_settings"] = load_settings.metadata()
+    run_config["custom_prompt_counts"] = {
+        "harmful_chars": len(custom_harmful or ""),
+        "harmless_chars": len(custom_harmless or ""),
+    }
+    run_config["custom_prompt_sha256"] = {
+        "harmful": hashlib.sha256((custom_harmful or "").encode()).hexdigest(),
+        "harmless": hashlib.sha256((custom_harmless or "").encode()).hexdigest(),
+    }
+    try:
+        run_archive = RunArchive()
+        run_id = run_archive.begin(
+            [model_id],
+            notes="Launched from the OBLITERATUS operator UI.",
+            metadata={"source": "operator_ui", "configuration": run_config},
+        )
+        save_dir = str(run_archive._run_dir(run_id) / "checkpoint")
+    except Exception as exc:
+        with _lock:
+            _state["status"] = "idle"
+        detail = f"{type(exc).__name__}: {exc}"
+        yield (
+            f"**Run manifest failed:** {detail}",
+            f"RUN MANIFEST FAILED before GPU allocation: {detail}",
+            get_chat_header(), gr.update(), gr.update(), gr.update(),
+        )
+        return
+
     # Admission must be granted before cleanup can touch the CUDA runtime or
     # the worker can enter any model-loading path.
     try:
         _gpu_lifecycle.loading(model_id)
     except AdmissionError as exc:
+        run_archive.fail(run_id, exc, phase="admission")
         with _lock:
             _state["status"] = "idle"
             _state["model"] = None
@@ -2250,7 +2299,9 @@ def obliterate(model_choice: str, method_choice: str,
                 f"Diagnostic: {exc.diagnostic_message()}",
             ]
         yield (
-            f"**GPU admission failed:** {exc.user_message()}",
+            f"**GPU admission failed ({type(exc).__name__}, phase admission):** "
+            f"{exc.user_message()} (run `{run_id}`; "
+            f"log `{run_archive._run_dir(run_id) / 'run.log'}`)",
             "\n".join(_state["log"]),
             get_chat_header(),
             gr.update(),
@@ -2260,11 +2311,6 @@ def obliterate(model_choice: str, method_choice: str,
         return
     _clear_gpu(release_lifecycle=False)
 
-    with _lock:
-        global _obliterate_counter
-        _obliterate_counter += 1
-        save_dir = f"/tmp/obliterated_{_obliterate_counter}"
-
     log_lines = []
     last_yielded = [0]
     pipeline_ref = [None]
@@ -2278,6 +2324,7 @@ def obliterate(model_choice: str, method_choice: str,
 
     def on_log(msg):
         log_lines.append(msg)
+        run_archive.append_log(run_id, msg)
 
     def on_stage(result):
         stage_key = result.stage
@@ -2285,6 +2332,12 @@ def obliterate(model_choice: str, method_choice: str,
                 "excise": "\u2702\ufe0f", "verify": "\u2705", "rebirth": "\u2b50"}.get(stage_key, "\u25b6")
         if result.status == "running":
             log_lines.append(f"\n{icon} {stage_key.upper()} \u2014 {result.message}")
+            run_archive.append_log(run_id, f"{stage_key.upper()} — {result.message}")
+            run_archive.mark_running(run_id, phase=stage_key)
+        elif stage_key == "summon" and result.status == "done":
+            memory = measure_torch_memory(torch)
+            _gpu_lifecycle.resize(memory)
+            _gpu_lifecycle.ready(memory)
         stage_order = {"summon": 0, "probe": 1, "distill": 2,
                        "excise": 3, "verify": 4, "rebirth": 5}
         idx = stage_order.get(stage_key, 0)
@@ -2311,6 +2364,12 @@ def obliterate(model_choice: str, method_choice: str,
                 n = min(prompt_volume, len(harmful_all), len(harmless_all))
             else:
                 n = min(len(harmful_all), len(harmless_all))
+            run_archive.record_dataset(
+                run_id,
+                identifier="custom" if use_custom else dataset_key,
+                harmful=harmful_all[:n],
+                harmless=harmless_all[:n],
+            )
 
             if method == "informed":
                 # Use the analysis-guided InformedAbliterationPipeline
@@ -2395,6 +2454,8 @@ def obliterate(model_choice: str, method_choice: str,
         source_info = DATASET_SOURCES.get(dataset_key)
         source_label = source_info.label if source_info else dataset_key
     log_lines.append(f"Target: {model_id}")
+    log_lines.append(f"Run ID: {run_id}")
+    log_lines.append(f"Durable log: {run_archive._run_dir(run_id) / 'run.log'}")
     log_lines.append(f"Method: {method}")
     if _adaptive_info:
         log_lines.append(_adaptive_info)
@@ -2403,6 +2464,8 @@ def obliterate(model_choice: str, method_choice: str,
     log_lines.append(f"Prompt volume: {vol_label} pairs")
     log_lines.append(f"Resolved load mode: {load_settings.summary}")
     log_lines.append("")
+    for header_line in log_lines:
+        run_archive.append_log(run_id, header_line)
 
     worker = threading.Thread(target=run_pipeline, daemon=True)
     worker.start()
@@ -2447,9 +2510,15 @@ def obliterate(model_choice: str, method_choice: str,
             _state["model"] = None
             _state["tokenizer"] = None
         log_lines.append(detail)
+        run_archive.fail(
+            run_id,
+            TimeoutError("pipeline exceeded the 45-minute limit"),
+            phase="timeout",
+        )
         _state["log"] = log_lines
         yield (
-            "**Timed out:** Pipeline cancelled; no successful artifact was registered.",
+            f"**Timed out (TimeoutError, phase timeout):** Pipeline cancelled (run `{run_id}`; "
+            f"log `{run_archive._run_dir(run_id) / 'run.log'}`).",
             "\n".join(log_lines),
             get_chat_header(),
             gr.update(),
@@ -2467,9 +2536,15 @@ def obliterate(model_choice: str, method_choice: str,
         with _lock:
             _state["status"] = "idle"
         err_msg = str(error_ref[0]) or repr(error_ref[0])
+        run_archive.fail(run_id, error_ref[0], phase="pipeline")
         log_lines.append(f"\nERROR: {err_msg}")
         _state["log"] = log_lines
-        yield f"**Error:** {err_msg}", "\n".join(log_lines), get_chat_header(), gr.update(), gr.update(), gr.update()
+        yield (
+            f"**Error ({type(error_ref[0]).__name__}, phase pipeline):** {err_msg} "
+            f"(run `{run_id}`; "
+            f"log `{run_archive._run_dir(run_id) / 'run.log'}`)",
+            "\n".join(log_lines), get_chat_header(), gr.update(), gr.update(), gr.update(),
+        )
         return
 
     # Success — keep model in memory for chat.
@@ -2480,6 +2555,18 @@ def obliterate(model_choice: str, method_choice: str,
         _gpu_lifecycle.resize(memory)
         _gpu_lifecycle.ready(memory)
         save_dir = str(_resolve_local_checkpoint(save_dir))
+        model_config = getattr(pipeline.handle.model, "config", None)
+        tokenizer_kwargs = getattr(pipeline.handle.tokenizer, "init_kwargs", {}) or {}
+        run_archive.record_revisions(
+            run_id,
+            model_revision=getattr(model_config, "_commit_hash", None),
+            tokenizer_revision=tokenizer_kwargs.get("_commit_hash"),
+        )
+        run_archive.complete(
+            run_id,
+            checkpoint=save_dir,
+            metrics=dict(pipeline._quality_metrics),
+        )
         can_generate = pipeline._quality_metrics.get("coherence") is not None
 
         # ── Telemetry: log single obliteration to community leaderboard ──
@@ -2724,13 +2811,22 @@ def obliterate(model_choice: str, method_choice: str,
     except Exception as e:
         # Ensure status never gets stuck on "obliterating"
         _gpu_lifecycle.release(reason="post_load_failed")
-        _cleanup_failed_pipeline(pipeline_ref[0], save_dir)
+        completed = run_archive.status(run_id).get("status") == "succeeded"
+        if not completed:
+            _cleanup_failed_pipeline(pipeline_ref[0], save_dir)
+            run_archive.fail(run_id, e, phase="post_pipeline")
         with _lock:
             _state["status"] = "idle"
         err_msg = str(e) or repr(e)
         log_lines.append(f"\nERROR (post-pipeline): {err_msg}")
+        run_archive.append_log(run_id, f"ERROR (post-pipeline): {err_msg}")
         _state["log"] = log_lines
-        yield f"**Error:** {err_msg}", "\n".join(log_lines), get_chat_header(), gr.update(), gr.update(), gr.update()
+        yield (
+            f"**Error ({type(e).__name__}, phase post-pipeline):** {err_msg} "
+            f"(run `{run_id}`; "
+            f"log `{run_archive._run_dir(run_id) / 'run.log'}`)",
+            "\n".join(log_lines), get_chat_header(), gr.update(), gr.update(), gr.update(),
+        )
 
 
 # ---------------------------------------------------------------------------