Fix output preservation, research validation, and documentation drift

Repair image and metadata publication contracts, detection routing, research
artifact validation, release verification, and the published agent skill.
Refresh the dependency lock and reconcile documentation with verified behavior.

Add regression coverage and record the completed audit and corrected VideoSeal
measurements. Preserve the existing Accelerate and Lightning versions.

Validation: Ruff and Pyright pass; 2358 tests pass with 3 skipped. The maintenance
gate exits 2 on the documented Accelerate and Lightning security findings.

scripts/_isolated_image_workers.py

@@ -0,0 +1,65 @@
+"""Bound native image work in fresh interpreters, with no parent-process fallback."""
+
+from __future__ import annotations
+
+import json
+import runpy
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+
+def _run_one(script: Path, path: str, timeout: float) -> dict[str, Any]:
+    command = [sys.executable, str(Path(__file__).resolve()), str(script), path]
+    try:
+        result = subprocess.run(  # noqa: S603 -- fixed interpreter and local worker entry point
+            command,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return {"path": path, "keys": [], "status": "timeout"}
+    except OSError as error:
+        return {"path": path, "keys": [], "status": f"worker-error:{type(error).__name__}"}
+    if result.returncode != 0:
+        return {"path": path, "keys": [], "status": "crashed", "returncode": result.returncode}
+    try:
+        row = json.loads(result.stdout)
+        if not isinstance(row, dict) or row.get("status") is None:
+            raise ValueError("worker returned an invalid record")
+    except ValueError:
+        return {"path": path, "keys": [], "status": "invalid-worker-result"}
+    return row
+
+
+def run_batch(script: Path, paths: list[str], *, jobs: int, timeout: float) -> list[dict[str, Any]]:
+    """Run at most jobs child processes; each child gets its own timeout.
+
+    Threads only supervise subprocesses. subprocess.run kills and reaps a timed-out
+    child before returning, so executor shutdown never waits on unbounded native work.
+    No fork or multiprocessing spawn imports the parent's OpenCV/Pillow state.
+    """
+    if jobs < 1 or timeout <= 0:
+        raise ValueError("jobs and timeout must be positive")
+    with ThreadPoolExecutor(max_workers=jobs) as executor:
+        return list(executor.map(lambda path: _run_one(script, path, timeout), paths))
+
+
+def main() -> None:
+    script = Path(sys.argv[1]).resolve()
+    sys.path.insert(0, str(script.parent.parent / "src"))
+    # stdout is the worker protocol, diagnostics from the audit go to stderr.
+    from contextlib import redirect_stdout
+
+    with redirect_stdout(sys.stderr):
+        namespace = runpy.run_path(str(script))
+        row = namespace["_one"](sys.argv[2])
+    sys.stdout.write(json.dumps(row))
+
+
+if __name__ == "__main__":
+    main()

scripts/_text_eval.py

@@ -11,7 +11,7 @@ def normalize_text(text: str) -> str:
 
 
 def levenshtein_normalized(left: str, right: str) -> float:
-    """Return normalized Levenshtein distance without changing either input."""
+    """Return Levenshtein distance divided by the longer input length (not CER)."""
     if not left and not right:
         return 0.0
     previous = list(range(len(right) + 1))

scripts/audioseal_experiment.py

@@ -62,6 +62,7 @@ FRAME_RATE = 12.0
 FRAME_WIDTH = 840
 FRAME_HEIGHT = 480
 CARRIER_SEED = 20260907
+NOISE_ATTACK_SEED = CARRIER_SEED + 1
 MESSAGE_SEED = 7
 
 
@@ -103,12 +104,12 @@ def carrier_seed(name: str) -> int:
     return CARRIER_SEED + zlib.crc32(name.encode()) % 1000
 
 
-def synth_carrier(name: str, seconds: float) -> np.ndarray:
+def synth_carrier(name: str, seconds: float, *, seed: int | None = None) -> np.ndarray:
     """Synthesize one deterministic carrier by registered name."""
     factory = CARRIERS.get(name)
     if factory is None:
         raise KeyError(f"unknown carrier {name!r}; expected one of {sorted(CARRIERS)}")
-    master = np.random.default_rng(carrier_seed(name))
+    master = np.random.default_rng(carrier_seed(name) if seed is None else seed)
     return np.asarray(factory(int(seconds * SAMPLE_RATE), master), dtype=np.float32)
 
 
@@ -383,7 +384,7 @@ def embed_audio(generator: object, samples: np.ndarray, message: list[int]) -> n
 
 
 def numpy_attacks(samples: np.ndarray) -> dict[str, np.ndarray]:
-    rng = np.random.default_rng(CARRIER_SEED + 1)
+    rng = np.random.default_rng(NOISE_ATTACK_SEED)
     out: dict[str, np.ndarray] = {}
     power = float(np.sum(samples.astype(np.float64) ** 2))
     noise = rng.normal(0.0, 1.0, samples.shape[0]).astype(np.float32)

scripts/fidelity_metrics.py

@@ -32,10 +32,10 @@ Two subcommands:
               Run this on the ORIGINALS, hand-verify/correct the file, and it
               becomes the ground truth for ``compare --ground-truth`` -- the clean
               way to score text, since OCR-vs-OCR is doubly noisy (errors on both
-              images + reading-order differences inflate CER even on identical text).
+              images + reading-order differences inflate NED even on identical text).
 
   compare  -- Score each VARIANT against the ORIGINAL across four groups:
-              1. Text  -- character error rate (CER) of the variant's OCR vs the
+              1. Text  -- normalized edit distance (NED) of the variant's OCR vs the
                  verified ground truth (or the original's OCR if no --ground-truth).
               2. Face identity -- insightface (buffalo_l) ArcFace cosine similarity.
               3. Face texture  -- LPIPS + Laplacian-variance ratio on face crops
@@ -88,7 +88,7 @@ def _match_size(variant: np.ndarray, ref: np.ndarray) -> np.ndarray:
 
 
 def _norm(text: str) -> str:
-    """Normalize for CER: NFC + drop ALL whitespace (segmentation-order agnostic)."""
+    """Normalize for NED: NFC + drop ALL whitespace (segmentation-order agnostic)."""
     return "".join(unicodedata.normalize("NFC", text).split())
 
 
@@ -167,7 +167,8 @@ def _ocr_lines(bgr: np.ndarray, langs: list[str], min_score: float = 0.5) -> lis
     return [t for _, t in kept]
 
 
-def _cer(ref: str, hyp: str) -> float:
+def _text_ned(ref: str, hyp: str) -> float:
+    """Case-sensitive edit distance divided by the longer normalized string."""
     return levenshtein_normalized(_norm(ref), _norm(hyp))
 
 
@@ -334,7 +335,7 @@ def compare(original: str, variants: tuple[str, ...], ocr_langs: str, ground_tru
     lp = _lpips_model()  # AlexNet LPIPS, loaded once and reused for face crops + whole image
 
     # ── text ──
-    ocr_cer: dict[str, float | None] = {label: None for label, _ in parsed}
+    ocr_ned: dict[str, float | None] = {label: None for label, _ in parsed}
     if langs:
         ref_text: str | None = None
         if ground_truth:
@@ -348,7 +349,7 @@ def compare(original: str, variants: tuple[str, ...], ocr_langs: str, ground_tru
         if ref_text:
             console.print(f"  OCR variants ({','.join(langs)})...")
             for label, img in parsed:
-                ocr_cer[label] = _cer(ref_text, "\n".join(_ocr_lines(img, langs)))
+                ocr_ned[label] = _text_ned(ref_text, "\n".join(_ocr_lines(img, langs)))
 
     # ── faces ──
     face_stats: dict[str, FaceStats] = {label: FaceStats() for label, _ in parsed}
@@ -392,14 +393,14 @@ def compare(original: str, variants: tuple[str, ...], ocr_langs: str, ground_tru
 
     # ── report ──
     table = Table(title=f"Fidelity vs {Path(original).name} (reference)")
-    for col in ("variant", "text CER↓", "faces", "ID cos↑", "face LPIPS↓", "lapvar↑", "img LPIPS↓", "SSIM↑", "PSNR↑"):
+    for col in ("variant", "text NED↓", "faces", "ID cos↑", "face LPIPS↓", "lapvar↑", "img LPIPS↓", "SSIM↑", "PSNR↑"):
         table.add_column(col)
     for label, _ in parsed:
         st = face_stats[label]
         wl, ws, wp = whole[label]
         table.add_row(
             label,
-            _fmt(ocr_cer[label]),
+            _fmt(ocr_ned[label]),
             str(st.n_faces),
             _fmt(_mean(st.identity)),
             _fmt(_mean(st.lpips)),
@@ -410,7 +411,7 @@ def compare(original: str, variants: tuple[str, ...], ocr_langs: str, ground_tru
         )
     console.print(table)
     console.print(
-        "  Legend: CER lower=better; ID cos higher=better; face LPIPS lower=better; "
+        "  Legend: NED lower=better; ID cos higher=better; face LPIPS lower=better; "
         "lapvar ratio ~1=detail kept, <1=smoothed/plastic; img LPIPS lower=better; SSIM/PSNR higher=closer."
     )
 

scripts/pill_gate_audit.py

@@ -32,12 +32,12 @@ import json
 import math
 import os
 import sys
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from concurrent.futures import TimeoutError as FutureTimeout
-from concurrent.futures.process import BrokenProcessPool
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
+sys.path.insert(0, str(Path(__file__).parent))
+
+from _isolated_image_workers import run_batch
 
 REPO = Path(__file__).resolve().parents[1]
 POSITIVES = REPO / ".local-eval" / "visible-positives.jsonl"
@@ -95,18 +95,8 @@ def _one(path_str: str) -> dict[str, object]:
 
 
 def _batch(paths: list[str], jobs: int, timeout: int) -> list[dict[str, object]]:
-    try:
-        with ProcessPoolExecutor(max_workers=jobs) as ex:
-            futs = [ex.submit(_one, p) for p in paths]
-            return [f.result() for f in as_completed(futs, timeout=timeout)]
-    except (FutureTimeout, BrokenProcessPool, OSError, RuntimeError):
-        out = []
-        for p in paths:
-            try:
-                out.append(_one(p))
-            except BaseException:  # a native crash costs one file, not the sweep
-                out.append({"path": Path(p).name, "status": "crashed"})
-        return out
+    """Bound each image in an independent interpreter, including native failures."""
+    return run_batch(Path(__file__), paths, jobs=jobs, timeout=timeout)
 
 
 def report(rows: list[dict[str, object]]) -> None:

scripts/provider_oracle_web.py

@@ -14,7 +14,7 @@ from urllib.parse import unquote, urlsplit
 import provider_oracles as oracles
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping
+    from collections.abc import Iterator, Mapping
     from pathlib import Path
 
     from playwright.sync_api import Browser, BrowserContext, BrowserType, Page, ProxySettings
@@ -241,11 +241,10 @@ def _submit_batch(
     *,
     headed: bool,
     timeout_seconds: float,
-) -> list[WebVerdict]:
+) -> Iterator[WebVerdict]:
     """Submit every row in one isolated session without retries or route changes."""
     from playwright.sync_api import sync_playwright
 
-    verdicts: list[WebVerdict] = []
     with sync_playwright() as playwright:
         browser = _launch_browser(playwright.chromium, proxy, headed=headed)
         context = _new_context(browser, proxy)
@@ -256,13 +255,12 @@ def _submit_batch(
                     log.info("Submitting %s to %s", upload.name, oracles.SURFACES[surface].url)
                     verdict = _check_page(page, surface, upload, timeout_seconds)
                     log.info("Provider response for %s: %s", upload.name, verdict.raw_response)
-                    verdicts.append(verdict)
+                    yield verdict
                 finally:
                     page.close()
         finally:
             context.close()
             browser.close()
-    return verdicts
 
 
 def run_web_batch(
@@ -305,10 +303,8 @@ def run_web_batch(
         headed=headed,
         timeout_seconds=timeout_seconds,
     )
-    if len(verdicts) != len(pending):
-        raise RuntimeError(f"Playwright returned {len(verdicts)} verdicts for {len(pending)} uploads")
-    checked_at = datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z")
     for (manifest_row, _result), verdict in zip(pending, verdicts, strict=True):
+        checked_at = datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
         oracles.record_result(
             manifest_path,
             artifact_id=manifest_row["artifact_id"],

scripts/robustness_suite.py

@@ -95,6 +95,9 @@ def graceful(res: Results, case: str, cmd: str, args: list[str], timeout: int =
     if timed_out:
         res.add(case, cmd, False, f"HUNG (> {timeout}s)", out)
         return
+    if code not in (0, 1, 2):
+        res.add(case, cmd, False, f"unexpected process exit {code}", out)
+        return
     crashed = next((m for m in _CRASH_MARKERS if m in out), None)
     if crashed:
         res.add(case, cmd, False, f"unhandled {crashed} (exit {code})", out)

scripts/smoke_matrix.py

@@ -49,6 +49,8 @@ import tempfile
 from dataclasses import dataclass, field
 from pathlib import Path
 
+from robustness_suite import _CRASH_MARKERS
+
 REPO = Path(__file__).resolve().parents[1]
 SAMPLES = REPO / "data" / "fixtures" / "provenance"
 CORPUS = REPO / ".local-eval" / "originals"
@@ -61,6 +63,8 @@ def _capture(args: list[str]) -> str:
     p = subprocess.run(  # noqa: S603
         [exe, "run", "remove-ai-watermarks", *args], capture_output=True, text=True, cwd=REPO, check=False
     )
+    if p.returncode != 0:
+        raise RuntimeError(f"CLI capture failed (exit {p.returncode}): {p.stderr}")
     return p.stdout
 
 
@@ -77,7 +81,7 @@ class Runner:
     tmp: Path
     results: list[Result] = field(default_factory=list)
 
-    def run(self, name: str, args: list[str], *, expect_exit: int | None = 0, timeout: int = 180) -> Result:
+    def run(self, name: str, args: list[str], *, expect_exit: int | tuple[int, ...] = 0, timeout: int = 180) -> Result:
         exe = shutil.which("uv") or "uv"
         cmd = [exe, "run", "remove-ai-watermarks", *args]
         try:
@@ -88,7 +92,10 @@ class Runner:
             r = Result(name, "FAIL", f"timeout after {timeout}s", " ".join(args))
             self.results.append(r)
             return r
-        ok = expect_exit is None or p.returncode == expect_exit
+        acceptable = (expect_exit,) if isinstance(expect_exit, int) else expect_exit
+        output = p.stdout + p.stderr
+        crashed = any(marker in output for marker in _CRASH_MARKERS)
+        ok = p.returncode in acceptable and not crashed
         detail = "" if ok else f"exit {p.returncode} (want {expect_exit}): {(p.stderr or p.stdout).strip()[-200:]}"
         r = Result(name, "pass" if ok else "FAIL", detail, " ".join(args))
         self.results.append(r)
@@ -293,7 +300,7 @@ def main() -> None:
         r.run(
             f"batch --mode {mode}",
             ["batch", str(bd), "--mode", mode, "-o", str(tmp / f"batch_{mode}")],
-            expect_exit=None,
+            expect_exit=0,
         )
 
     # ---- diffusion: argument handling always; the model body under --diffusion ----
@@ -319,8 +326,8 @@ def main() -> None:
             r.run(
                 f"visible auto real {label}",
                 ["visible", str(p), "-o", str(tmp / f"r_{label}_{p.stem[:8]}.png")],
-                expect_exit=None,
-            )  # 0 or 2 are both correct; a CRASH is not
+                expect_exit=(0, EXIT_NO_VISIBLE_MARK),
+            )  # Successful removal or the documented no-mark outcome.
 
         # unicode + misnamed extension + truncated: the documented real-world traps
         if picks:
@@ -333,7 +340,7 @@ def main() -> None:
             r.run("misnamed extension", ["identify", str(mis), "--json"])
             trunc = tmp / "truncated.png"
             trunc.write_bytes((SAMPLES / "chatgpt-1.png").read_bytes()[:4096])
-            r.run("truncated file does not crash", ["identify", str(trunc), "--json"], expect_exit=None)
+            r.run("truncated file does not crash", ["identify", str(trunc), "--json"], expect_exit=(0, 1))
 
     # ---- report --------------------------------------------------------------
     bad = [x for x in r.results if x.status == "FAIL"]
@@ -470,9 +477,12 @@ def _media_rows(r: Runner, tmp: Path) -> None:
             r.skip(f"{name} metadata strip", "ffmpeg could not synthesize the fixture")
             continue
         out = tmp / f"media_clean.{name}"
-        r.run(f"{name} metadata strip runs", ["metadata", str(src), "--remove", "-o", str(out)], expect_exit=None)
-        if out.exists():
-            r.check(f"{name} strip produced a non-empty file", out.stat().st_size > 0, "empty output")
+        r.run(f"{name} metadata strip runs", ["metadata", str(src), "--remove", "-o", str(out)], expect_exit=0)
+        r.check(
+            f"{name} strip produced a non-empty file",
+            out.is_file() and out.stat().st_size > 0,
+            "missing or empty output",
+        )
 
 
 def _sdxl_watermark_bits(img: object) -> float:
@@ -557,7 +567,7 @@ def _diffusion_rows(r: Runner, tmp: Path, doubao: Path) -> None:
     res = r.run(
         "batch --mode invisible runs (512px)",
         ["batch", str(bd), "--mode", "invisible", "-o", str(bout), *small],
-        expect_exit=None,
+        expect_exit=0,
         timeout=3600,
     )
     if res.status == "pass":
@@ -569,7 +579,7 @@ def _diffusion_rows(r: Runner, tmp: Path, doubao: Path) -> None:
     r.run(
         "batch --mode all runs (512px)",
         ["batch", str(bd), "--mode", "all", "-o", str(aout), *small],
-        expect_exit=None,
+        expect_exit=0,
         timeout=3600,
     )
 

scripts/synthid_affine_lattice_probe.py

@@ -1003,7 +1003,7 @@ def score_lattice(
         for index in rerank_candidates
     ]
     rerank_scores = [
-        min(*whitened_scores[index], amplitude.selection, amplitude.confirmation)
+        min(whitened_scores[index][0], amplitude.selection)
         for index, amplitude in zip(rerank_candidates, rerank_amplitudes, strict=True)
     ]
     rerank_winner = int(np.argmax(rerank_scores))
@@ -1603,7 +1603,8 @@ def main(
     report_out.write_text(
         json.dumps(
             {
-                "schema_version": 13,
+                "schema_version": 14,
+                "period_selection": "selection-only-whitened-amplitude-v1",
                 "template": str(template_path),
                 "periods": [float(value) for value in periods],
                 "rotations_degrees": [float(value) for value in rotations],

scripts/synthid_conformal_cascade.py

@@ -284,7 +284,7 @@ def load_config(path: Path) -> CascadeConfig:
 def load_observation_records(path: Path) -> list[tuple[str, tuple[ExpertObservation, ...]]]:
     """Load named score records with explicit support for every expert."""
     payload = _mapping(json.loads(path.read_text(encoding="utf-8")), "observation manifest")
-    if payload.get("schema_version") != 1:
+    if payload.get("schema_version") not in {1, 2}:
         raise ValueError("unsupported observation manifest schema")
     records: list[tuple[str, tuple[ExpertObservation, ...]]] = []
     for record_index, raw_record in enumerate(_sequence(payload.get("records"), "records")):
@@ -311,6 +311,11 @@ def load_observation_records(path: Path) -> list[tuple[str, tuple[ExpertObservat
                     else _number(raw_score, f"records[{record_index}].observations[{observation_index}].score"),
                 )
             )
+        if payload.get("schema_version") == 1:
+            from synthid_runtime.synthid_detector import REGISTERED_DETECTOR_ID
+
+            if any(observation.name == REGISTERED_DETECTOR_ID for observation in observations):
+                raise ValueError("legacy runtime observations require rescoring to preserve fallback expert identity")
         records.append((record_id, tuple(observations)))
     return records
 

scripts/synthid_periodic_tile_probe.py

@@ -120,8 +120,8 @@ def score_pixels(
         path=path,
         score=score,
         active_support=min(norm / (model.expected_norm + 1e-12), 1.0),
-        row_shift=row_shift,
-        column_shift=column_shift,
+        row_shift=int(row_shift),
+        column_shift=int(column_shift),
         repeat_count=(model.height // model.tile_height) * (model.width // model.tile_width),
     )
 
@@ -172,8 +172,8 @@ def load_model(path: Path) -> PeriodicTileModel:
         )
     if model.height < 1 or model.width < 1 or model.tile_height < 1 or model.tile_width < 1:
         raise ValueError("invalid periodic-tile geometry")
-    if model.height % model.tile_height or model.width % model.tile_width:
-        raise ValueError("image geometry is not divisible by periodic-tile geometry")
+    if model.height < model.tile_height or model.width < model.tile_width:
+        raise ValueError("image geometry must be at least as large as the tile geometry")
     if model.template.shape != (model.tile_height, model.tile_width, 3):
         raise ValueError("invalid periodic-tile template shape")
     if not np.all(np.isfinite(model.template)) or not np.isclose(np.linalg.norm(model.template), 1.0):

scripts/synthid_pixel_probe.py

@@ -62,7 +62,9 @@ def carrier(gray: NDArray[np.float64]) -> NDArray[np.float64]:
 
 def ncc(a: NDArray[np.float64], b: NDArray[np.float64]) -> float:
     """Normalized cross-correlation of two carriers (unit-norm zero-mean vectors)."""
-    if a.shape != b.shape or a.size == 0:
+    if a.shape != b.shape:
+        raise ValueError("carrier geometry does not match")
+    if a.size == 0:
         return 0.0
     return float(np.dot(a, b))
 
@@ -88,14 +90,13 @@ def random_baseline(shape: tuple[int, ...], n: int, *, seed: int = 0) -> float:
 
 
 def _load_carriers(paths: tuple[str, ...]) -> list[NDArray[np.float64]]:
-    """Load carriers for same-shaped images; warn and skip mismatched shapes."""
+    """Load carriers only when every image has the same comparable geometry."""
     grays = [(p, load_gray(p)) for p in paths]
     shape = grays[0][1].shape
     carriers: list[NDArray[np.float64]] = []
     for p, g in grays:
         if g.shape != shape:
-            console.print(f"  [yellow]skip[/] {p}: shape {g.shape} != {shape}")
-            continue
+            raise click.ClickException(f"{p}: geometry {g.shape} does not match {shape}")
         carriers.append(carrier(g))
     return carriers
 
@@ -129,8 +130,9 @@ def consistency(images: tuple[str, ...]) -> None:
 )
 def removal(pos: tuple[str, ...], cleaned: tuple[str, ...]) -> None:
     """Does the pipeline drop the carrier correlation toward the random baseline?"""
-    pos_carriers = _load_carriers(pos)
-    cleaned_carriers = _load_carriers(cleaned)
+    carriers = _load_carriers(pos + cleaned)
+    pos_carriers = carriers[: len(pos)]
+    cleaned_carriers = carriers[len(pos) :]
     if not pos_carriers or not cleaned_carriers:
         console.print("[red]Need at least one positive and one cleaned fill of matching shape.[/]")
         raise SystemExit(1)

scripts/synthid_research_manifest.py

@@ -208,6 +208,13 @@ def _row_errors(row: dict[str, str], index: int) -> list[str]:
         errors.append(f"{prefix}: source-evidence cannot establish a same-provider negative")
     if verifier == "source-evidence" and not row.get("evidence_reference", "").strip():
         errors.append(f"{prefix}: source-evidence requires evidence_reference")
+    if (
+        synthid == "not_detected"
+        and source != target
+        and verifier not in matching
+        and not row.get("evidence_reference", "").strip()
+    ):
+        errors.append(f"{prefix}: an external negative without a matching verifier requires evidence_reference")
     if verifier in matching and not row.get("oracle_session", "").strip():
         errors.append(f"{prefix}: provider verification requires oracle_session")
     if oracle_role == "source_control" and synthid != "detected":

scripts/synthid_routed_detector.py

@@ -53,7 +53,7 @@ def main(images: tuple[Path, ...], report_out: Path) -> None:
     report_out.write_text(
         json.dumps(
             {
-                "schema_version": 1,
+                "schema_version": 2,
                 "counts": {"detected": detected, "abstain": len(records) - detected},
                 "records": records,
             },

scripts/synthid_routed_expert_bank.py

@@ -27,6 +27,7 @@ from synthid_conformal_cascade import (  # noqa: E402
 )
 from synthid_research_manifest import artifact_sha256  # noqa: E402
 from synthid_runtime import synthid_detector  # noqa: E402
+from synthid_runtime_expert_scores import EXPERT_NAMES  # noqa: E402
 
 log = logging.getLogger(__name__)
 
@@ -46,18 +47,18 @@ class RoutedBankResult:
     registered_score: float | None
     large_supported: bool
     large_score: float | None
+    opponent_supported: bool
+    opponent_score: float | None
+    fine_opponent_supported: bool
+    fine_opponent_score: float | None
 
 
 def classify_routed(observations: tuple[ExpertObservation, ...]) -> RoutedBankResult:
-    """Route explicit fixed, registered, and large observations without an OR rule."""
+    """Route fixed, large, and the active registered expert without an OR rule."""
     by_name = {observation.name: observation for observation in observations}
     if len(by_name) != len(observations):
         raise ValueError("observation expert names must be unique")
-    expected = {
-        synthid_detector.DETECTOR_ID,
-        synthid_detector.REGISTERED_DETECTOR_ID,
-        synthid_detector.LARGE_DETECTOR_ID,
-    }
+    expected = set(EXPERT_NAMES)
     if by_name.keys() != expected:
         missing = sorted(expected - by_name.keys())
         unknown = sorted(by_name.keys() - expected)
@@ -66,6 +67,17 @@ def classify_routed(observations: tuple[ExpertObservation, ...]) -> RoutedBankRe
     fixed = by_name[synthid_detector.DETECTOR_ID]
     registered = by_name[synthid_detector.REGISTERED_DETECTOR_ID]
     large = by_name[synthid_detector.LARGE_DETECTOR_ID]
+    opponent = by_name[synthid_detector.OPPONENT_REGISTERED_DETECTOR_ID]
+    fine_opponent = by_name[synthid_detector.FINE_OPPONENT_REGISTERED_DETECTOR_ID]
+    active_registered = [item for item in (registered, opponent, fine_opponent) if item.supported]
+    if len(active_registered) > 1:
+        raise ValueError("registered route must contain at most one active expert")
+    routed_registered = active_registered[0] if active_registered else registered
+    registered_thresholds = {
+        registered.name: synthid_detector.REGISTERED_THRESHOLD,
+        opponent.name: synthid_detector.OPPONENT_REGISTERED_THRESHOLD,
+        fine_opponent.name: synthid_detector.FINE_OPPONENT_REGISTERED_THRESHOLD,
+    }
     if large.supported:
         if large.score is None:
             raise RuntimeError("validated large observation lost its score")
@@ -77,13 +89,13 @@ def classify_routed(observations: tuple[ExpertObservation, ...]) -> RoutedBankRe
             verdict = "abstain"
             reason = "large_below_threshold"
             selected_expert = None
-    elif registered.supported:
-        if registered.score is None:
+    elif routed_registered.supported:
+        if routed_registered.score is None:
             raise RuntimeError("validated registered observation lost its score")
-        if registered.score >= synthid_detector.REGISTERED_THRESHOLD:
+        if routed_registered.score >= registered_thresholds[routed_registered.name]:
             verdict = "detected"
             reason = "registered_threshold_crossed"
-            selected_expert = registered.name
+            selected_expert = routed_registered.name
         else:
             verdict = "abstain"
             reason = (
@@ -115,6 +127,10 @@ def classify_routed(observations: tuple[ExpertObservation, ...]) -> RoutedBankRe
         registered_score=registered.score,
         large_supported=large.supported,
         large_score=large.score,
+        opponent_supported=opponent.supported,
+        opponent_score=opponent.score,
+        fine_opponent_supported=fine_opponent.supported,
+        fine_opponent_score=fine_opponent.score,
     )
 
 
@@ -122,7 +138,7 @@ def classify_routed(observations: tuple[ExpertObservation, ...]) -> RoutedBankRe
 @click.argument("observation_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
 @click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True)
 def main(observation_path: Path, report_out: Path) -> None:
-    """Route a three-expert pixel score manifest from OBSERVATION_PATH."""
+    """Route a versioned runtime expert score manifest from OBSERVATION_PATH."""
     logging.basicConfig(level=logging.INFO, format="%(message)s")
     counts: dict[RoutedVerdict, int] = {"detected": 0, "abstain": 0}
     rows: list[dict[str, object]] = []
@@ -134,7 +150,7 @@ def main(observation_path: Path, report_out: Path) -> None:
     report_out.write_text(
         json.dumps(
             {
-                "schema_version": 1,
+                "schema_version": 2,
                 "observation_sha256": artifact_sha256(observation_path),
                 "counts": counts,
                 "records": rows,

scripts/synthid_runtime_expert_scores.py

@@ -1,4 +1,4 @@
-"""Export fixed, large, and scale-registered SynthID observations for images.
+"""Export correctly named fixed, large, and routed registered observations.
 
 The output is an input manifest for ``synthid_conformal_cascade.py``. All
 experts consume decoded RGB pixels only. Unsupported geometry is recorded
@@ -32,6 +32,15 @@ log = logging.getLogger(__name__)
 FIXED_EXPERT_NAME = synthid_detector.DETECTOR_ID
 REGISTERED_EXPERT_NAME = synthid_detector.REGISTERED_DETECTOR_ID
 LARGE_EXPERT_NAME = synthid_detector.LARGE_DETECTOR_ID
+OPPONENT_EXPERT_NAME = synthid_detector.OPPONENT_REGISTERED_DETECTOR_ID
+FINE_OPPONENT_EXPERT_NAME = synthid_detector.FINE_OPPONENT_REGISTERED_DETECTOR_ID
+EXPERT_NAMES = (
+    FIXED_EXPERT_NAME,
+    REGISTERED_EXPERT_NAME,
+    LARGE_EXPERT_NAME,
+    OPPONENT_EXPERT_NAME,
+    FINE_OPPONENT_EXPERT_NAME,
+)
 
 
 class ExpertScore(TypedDict):
@@ -57,26 +66,21 @@ def _observation(name: str, supported: bool, score: float | None) -> ExpertScore
 
 
 def score_pixels(pixels: NDArray[np.uint8]) -> list[ExpertScore]:
-    """Return explicit fixed, registered, and large observations for RGB PIXELS."""
+    """Return five named slots; inactive registered fallbacks remain unsupported."""
     if pixels.ndim != 3 or pixels.shape[2] != 3 or pixels.dtype != np.uint8:
         raise ValueError("pixels must be an RGB uint8 array")
     bgr_pixels = np.ascontiguousarray(pixels[:, :, ::-1])
     native = synthid_detector.detect_synthid("decoded-image", image=bgr_pixels, register_scale=False)
     registered = synthid_detector.detect_synthid("decoded-image", image=bgr_pixels, register_scale=True)
-    fixed = _observation(FIXED_EXPERT_NAME, False, None)
-    large = _observation(LARGE_EXPERT_NAME, False, None)
-    native_observation = _observation(native.detector, native.status != "unsupported", native.score)
-    if native.detector == FIXED_EXPERT_NAME:
-        fixed = native_observation
-    elif native.detector == LARGE_EXPERT_NAME:
-        large = native_observation
-    else:
-        raise RuntimeError(f"unexpected default SynthID expert: {native.detector}")
-    return [
-        fixed,
-        _observation(REGISTERED_EXPERT_NAME, registered.status != "unsupported", registered.score),
-        large,
-    ]
+    observations = {name: _observation(name, False, None) for name in EXPERT_NAMES}
+    for result, expected in (
+        (native, {FIXED_EXPERT_NAME, LARGE_EXPERT_NAME}),
+        (registered, {REGISTERED_EXPERT_NAME, OPPONENT_EXPERT_NAME, FINE_OPPONENT_EXPERT_NAME}),
+    ):
+        if result.detector not in expected:
+            raise RuntimeError(f"unexpected SynthID expert: {result.detector}")
+        observations[result.detector] = _observation(result.detector, result.status != "unsupported", result.score)
+    return list(observations.values())
 
 
 def score_path(path: Path) -> ScoredImage:
@@ -103,8 +107,8 @@ def main(images: tuple[Path, ...], report_out: Path) -> None:
     report_out.write_text(
         json.dumps(
             {
-                "schema_version": 1,
-                "experts": [FIXED_EXPERT_NAME, REGISTERED_EXPERT_NAME, LARGE_EXPERT_NAME],
+                "schema_version": 2,
+                "experts": list(EXPERT_NAMES),
                 "records": records,
             },
             indent=2,
@@ -112,7 +116,7 @@ def main(images: tuple[Path, ...], report_out: Path) -> None:
         + "\n",
         encoding="utf-8",
     )
-    log.info("Wrote %d three-expert score records: %s", len(records), report_out)
+    log.info("Wrote %d runtime expert score records: %s", len(records), report_out)
 
 
 if __name__ == "__main__":

scripts/videoseal_temporal_study.py

@@ -39,7 +39,7 @@ sys.path.insert(0, str(REPO / "scripts"))
 import videoseal_oracle  # noqa: E402
 from audioseal_experiment import ffmpeg_version  # noqa: E402
 from videoseal_oracle import read, read_aggregation_matrix  # noqa: E402
-from watermark_benchmark import _decode_video, sha256_file  # noqa: E402
+from watermark_benchmark import _decode_video, decode_video_artifact, sha256_file  # noqa: E402
 from watermark_benchmark_video_cohort import (  # noqa: E402
     FRAME_RATE,
     encode_clip,
@@ -205,21 +205,25 @@ def main() -> int:
         workdir = out_dir / name
         workdir.mkdir()
         clean_path = encode_clip(ffmpeg, workdir / f"{name}-clean.mp4", clean_frames)
+        clean_frames, clean_digest = decode_video_artifact(clean_path)
         record(
             "carrier_clean",
             carrier=name,
-            sha256=sha256_file(clean_path),
+            sha256=clean_digest,
+            measurement_domain="decoded_artifact",
             aggregation_matrix=read_aggregation_matrix(model, clean_frames),
             **provenance,
         )
 
         marked_frames = np.asarray(videoseal_oracle.embed(model, clean_frames, message), dtype=np.float32)
         marked_path = encode_clip(ffmpeg, workdir / f"{name}-marked.mp4", marked_frames)
+        marked_frames, marked_digest = decode_video_artifact(marked_path)
         marked_read = read(model, marked_frames)
         record(
             "carrier_marked",
             carrier=name,
-            sha256=sha256_file(marked_path),
+            sha256=marked_digest,
+            measurement_domain="decoded_artifact",
             bit_accuracy_avg=marked_read.bit_accuracy,
             aggregation_matrix=read_aggregation_matrix(model, marked_frames),
             per_frame_bit_accuracy=[round(v, 4) for v in marked_read.per_frame_bit_accuracy],
@@ -243,15 +247,14 @@ def main() -> int:
             ("fps_half", apply_fps_half(ffmpeg, marked_path, workdir)),
         ]
         for attack, path in attacked:
-            frames = _decode_video(path)
-            if frames is None:
-                raise SystemExit(f"attack artifact failed to decode: {path}")
+            frames, digest = decode_video_artifact(path)
             reading = read(model, frames)
             record(
                 "attack",
                 carrier=name,
                 attack=attack,
-                sha256=sha256_file(path),
+                sha256=digest,
+                measurement_domain="decoded_artifact",
                 frames=frames.shape[0],
                 bit_accuracy_avg=reading.bit_accuracy,
                 aggregation_matrix=read_aggregation_matrix(model, frames),

scripts/visible_positives.py

@@ -13,8 +13,8 @@ CRASH TOLERANCE IS NOT OPTIONAL AT THIS SCALE
   because results were only written at the end). So this script:
     * writes every result to JSONL as it arrives -- a kill never costs more than a batch;
     * is resumable -- an interrupted run skips what it already recorded;
-    * runs a FRESH pool per batch with a timeout, so one poisoned file costs one batch,
-      and that batch is retried serially to find and record the offender.
+    * runs each image in a fresh interpreter with a per-image timeout and bounded
+      concurrency; a poisoned file is recorded without retrying it in the parent.
 
 Treat input datasets as sensitive and read-only, and keep output gitignored.
 
@@ -29,12 +29,12 @@ import glob
 import json
 import os
 import sys
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from concurrent.futures import TimeoutError as FutureTimeout
-from concurrent.futures.process import BrokenProcessPool
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
+sys.path.insert(0, str(Path(__file__).parent))
+
+from _isolated_image_workers import run_batch
 
 # The package's own format set. An inlined copy here silently skipped .heif, which
 # CLAUDE.md documents as supported.
@@ -60,20 +60,8 @@ def _one(path: str) -> dict[str, object]:
 
 
 def _run_batch(batch: list[str], jobs: int, timeout: int) -> list[dict[str, object]]:
-    """One batch in a fresh pool. On a native worker crash or timeout, retry serially so
-    the poisoned file is identified and recorded instead of stalling the whole sweep."""
-    try:
-        with ProcessPoolExecutor(max_workers=jobs) as ex:
-            futs = {ex.submit(_one, p): p for p in batch}
-            return [f.result() for f in as_completed(futs, timeout=timeout)]
-    except (FutureTimeout, BrokenProcessPool, OSError, RuntimeError):
-        out: list[dict[str, object]] = []
-        for p in batch:
-            try:
-                out.append(_one(p))
-            except BaseException:  # a native crash here kills only this file, not the sweep
-                out.append({"path": p, "keys": [], "status": "crashed"})
-        return out
+    """Bound each image in an independent interpreter, including native failures."""
+    return run_batch(Path(__file__), batch, jobs=jobs, timeout=timeout)
 
 
 def main() -> None:
@@ -81,7 +69,7 @@ def main() -> None:
     ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2))
     ap.add_argument("--limit", type=int, default=0)
     ap.add_argument("--batch", type=int, default=200)
-    ap.add_argument("--timeout", type=int, default=600, help="seconds per batch before falling back to serial")
+    ap.add_argument("--timeout", type=int, default=600, help="seconds per image before terminating its isolated worker")
     ap.add_argument("--out", type=Path, default=OUT)
     ap.add_argument("--restart", action="store_true")
     a = ap.parse_args()

scripts/visible_recall_sample.py

@@ -24,6 +24,7 @@ from __future__ import annotations
 
 import argparse
 import csv
+import hashlib
 import json
 import random
 from pathlib import Path
@@ -64,6 +65,16 @@ def corner_strip(img: NDArray[Any]) -> NDArray[Any] | None:
     return strip
 
 
+def unique_content_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Keep one row per file digest, independently of detector observations."""
+    unique: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        with Path(row["path"]).open("rb") as source:
+            digest = hashlib.file_digest(source, "sha256").hexdigest()
+        unique.setdefault(digest, row)
+    return list(unique.values())
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("scan", type=Path, help="JSONL corpus scan produced by the visible evaluation harness")
@@ -78,10 +89,7 @@ def main() -> None:
     out.mkdir(parents=True, exist_ok=True)
 
     recs = [json.loads(line) for line in scan.open() if '"marks"' in line]
-    seen: dict[tuple, dict] = {}
-    for r in recs:  # exact-duplicate uploads share the whole NCC vector
-        seen.setdefault(tuple(r.get("shape", ())) + tuple(sorted((k, m["conf"]) for k, m in r["marks"].items())), r)
-    uniq = list(seen.values())
+    uniq = unique_content_rows(recs)
 
     tc = [r for r in uniq if r["cls"] == "tc260"]
     goog = [r for r in uniq if r["cls"] == "neg" and "Google" in r.get("platform", "")]

scripts/watermark_benchmark.py

@@ -523,7 +523,7 @@ def _decode_video(path: Path) -> NDArray[Any] | None:
     if probe.returncode != 0:
         return None
     try:
-        height, width = (int(value) for value in probe.stdout.strip().split(","))
+        width, height = (int(value) for value in probe.stdout.strip().split(","))
     except ValueError:
         return None
     result = subprocess.run(  # noqa: S603 - resolved ffmpeg with fixed arguments
@@ -546,6 +546,8 @@ def _decode_video(path: Path) -> NDArray[Any] | None:
     )
     if result.returncode != 0 or not result.stdout:
         return None
+    if height <= 0 or width <= 0:
+        return None
     frame_size = height * width * 3
     if len(result.stdout) % frame_size != 0:
         return None
@@ -553,6 +555,17 @@ def _decode_video(path: Path) -> NDArray[Any] | None:
     return frames.astype(np.float32) / 255.0
 
 
+def decode_video_artifact(path: Path) -> tuple[NDArray[Any], str]:
+    """Decode saved bytes and bind the measurement to their unchanged digest."""
+    digest = sha256_file(path)
+    frames = _decode_video(path)
+    if frames is None:
+        raise ValueError(f"video artifact failed to decode: {path}")
+    if sha256_file(path) != digest:
+        raise ValueError(f"video artifact changed during decoding: {path}")
+    return frames, digest
+
+
 def _detect_audioseal(path: Path, samples: NDArray[Any]) -> str | None:
     """Read the pinned AudioSeal oracle; the label is the decoded message."""
     from audioseal_oracle import detect_sample_array

scripts/watermark_benchmark_audio_cohort.py

@@ -37,7 +37,10 @@ sys.path.insert(0, str(REPO / "scripts"))
 import audioseal_oracle  # noqa: E402
 from audioseal_experiment import (  # noqa: E402
     FFMPEG_ATTACKS,
+    MESSAGE_SEED,
+    NOISE_ATTACK_SEED,
     SAMPLE_RATE,
+    carrier_seed,
     decode_audio_f32,
     ffmpeg_attack,
     ffmpeg_version,
@@ -52,6 +55,7 @@ log = logging.getLogger("watermark_benchmark_audio_cohort")
 
 RECIPE_VERSION = "watermark-benchmark-audio-cohort-v2"
 FORGED_MESSAGE_SEED = 8
+HARD_NEGATIVE_SEED = 20260907 + 991
 # Carriers the 10 dB additive-noise transform measurably takes below the
 # audioseal decision rule; the synthetic noise carriers survive it.
 REMOVED_BY_NOISE: tuple[str, ...] = ("tone_stack",)
@@ -102,7 +106,7 @@ def benchmark_row(
     transform_name: str,
     transform_revision: str,
     parameters: dict[str, object],
-    seed: int,
+    seed: int | None,
     expected: str,
 ) -> dict[str, object]:
     """Build one strict benchmark-manifest row for a generated artifact."""
@@ -176,7 +180,8 @@ def build_cohort(
     rows: list[dict[str, object]] = []
 
     for name in carriers:
-        clean = synth_carrier(name, duration_s)
+        executed_seed = carrier_seed(name)
+        clean = synth_carrier(name, duration_s, seed=executed_seed)
         clean_path = artifacts / f"{name}-clean.wav"
         clean_path.write_bytes(wav_pcm16_bytes(clean, SAMPLE_RATE))
 
@@ -201,7 +206,7 @@ def build_cohort(
                     "sample_rate": SAMPLE_RATE,
                     "dependencies": dependencies,
                 },
-                seed=20260907,
+                seed=executed_seed,
                 expected="not_detected",
             )
         )
@@ -222,7 +227,7 @@ def build_cohort(
                     "alpha": 1.0,
                     "dependencies": dependencies,
                 },
-                seed=7,
+                seed=MESSAGE_SEED,
                 expected="detected",
             )
         )
@@ -241,7 +246,7 @@ def build_cohort(
                     transform_name=f"attack-{attack}",
                     transform_revision=RECIPE_VERSION,
                     parameters=attack_parameters(attack, dependencies),
-                    seed=7,
+                    seed=NOISE_ATTACK_SEED if attack == "noise_snr10" else None,
                     expected="unresolved",
                 )
             )
@@ -262,14 +267,13 @@ def build_cohort(
                     transform_name="remove-noise-snr10",
                     transform_revision=RECIPE_VERSION,
                     parameters=attack_parameters("noise_snr10", dependencies),
-                    seed=7,
+                    seed=NOISE_ATTACK_SEED,
                     expected="not_detected",
                 )
             )
 
         # The forged row embeds a different message into the same clean
-        # carrier: a watermark is present, just not the oracle's, and the
-        # matched verifier's correct answer is not_detected.
+        # carrier: a watermark is present, with a foreign decoded payload.
         forged_message = message_bits(FORGED_MESSAGE_SEED)
         forged_samples = np.asarray(audioseal_oracle.embed(generator, clean, forged_message), dtype=np.float32)
         forged_path = artifacts / f"{name}-forged.wav"
@@ -321,7 +325,7 @@ def build_cohort(
                 "sample_rate": SAMPLE_RATE,
                 "dependencies": dependencies,
             },
-            seed=20260907 + 991,
+            seed=HARD_NEGATIVE_SEED,
             expected="not_detected",
         )
     )
@@ -390,7 +394,7 @@ def _append_speech_rows(
                 transform_name="synthesize-speech",
                 transform_revision=RECIPE_VERSION,
                 parameters={"tts": tts, "sample_rate": SAMPLE_RATE, "dependencies": dependencies},
-                seed=20260907,
+                seed=None,
                 expected="not_detected",
             )
         )
@@ -412,7 +416,7 @@ def _append_speech_rows(
                     "tts": tts,
                     "dependencies": dependencies,
                 },
-                seed=7,
+                seed=MESSAGE_SEED,
                 expected="detected",
             )
         )
@@ -430,7 +434,7 @@ def _append_speech_rows(
                     transform_name=f"attack-{attack}",
                     transform_revision=RECIPE_VERSION,
                     parameters={**attack_parameters(attack, dependencies), "tts": tts},
-                    seed=7,
+                    seed=NOISE_ATTACK_SEED if attack == "noise_snr10" else None,
                     expected="unresolved",
                 )
             )
@@ -438,7 +442,7 @@ def _append_speech_rows(
 
 def synth_carrier_hard_negative(duration_s: float) -> np.ndarray:
     """A never-embedded white-noise carrier keyed off a distinct seed."""
-    rng = np.random.default_rng(20260907 + 991)
+    rng = np.random.default_rng(HARD_NEGATIVE_SEED)
     n = int(duration_s * SAMPLE_RATE)
     return (rng.normal(0.0, 0.1, n)).astype(np.float32)
 

scripts/watermark_benchmark_report.py

@@ -411,6 +411,7 @@ def _fidelity_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]:
         counts = {"measured": 0, "incomparable": 0, "not_measured": 0, "inconsistent": 0}
         identical = 0
         psnr_values: list[float] = []
+        mean_psnr_values: list[float] = []
         snr_values: list[float] = []
         for key, (_, value) in pairs.items():
             if key in inconsistent_pairs:
@@ -426,6 +427,9 @@ def _fidelity_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]:
                 psnr = value.get("psnr_db")
                 if isinstance(psnr, (int, float)) and not isinstance(psnr, bool) and math.isfinite(psnr):
                     psnr_values.append(float(psnr))
+                mean_psnr = value.get("mean_psnr_db")
+                if isinstance(mean_psnr, (int, float)) and not isinstance(mean_psnr, bool) and math.isfinite(mean_psnr):
+                    mean_psnr_values.append(float(mean_psnr))
                 snr = value.get("snr_db")
                 if isinstance(snr, (int, float)) and not isinstance(snr, bool) and math.isfinite(snr):
                     snr_values.append(float(snr))
@@ -438,6 +442,8 @@ def _fidelity_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]:
                 "identical": identical,
                 "finite_psnr": len(psnr_values),
                 "p50_psnr_db": nearest_rank(sorted(psnr_values), 0.50) if psnr_values else None,
+                "finite_mean_psnr": len(mean_psnr_values),
+                "p50_mean_psnr_db": nearest_rank(sorted(mean_psnr_values), 0.50) if mean_psnr_values else None,
                 "finite_snr": len(snr_values),
                 "p50_snr_db": nearest_rank(sorted(snr_values), 0.50) if snr_values else None,
             }
@@ -743,6 +749,8 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench
                     ("inconsistent", "Inconsistent"),
                     ("finite_psnr", "Finite PSNR"),
                     ("p50_psnr_db", "p50 PSNR dB"),
+                    ("finite_mean_psnr", "Finite video mean PSNR"),
+                    ("p50_mean_psnr_db", "p50 video mean PSNR dB"),
                     ("finite_snr", "Finite SNR"),
                     ("p50_snr_db", "p50 SNR dB"),
                 ),

scripts/watermark_benchmark_video_cohort.py

@@ -89,9 +89,9 @@ def _carrier_seed(name: str) -> int:
     return CARRIER_SEED + zlib.crc32(name.encode()) % 1000
 
 
-def synth_carrier(name: str, *, seed_offset: int = 0) -> np.ndarray:
+def synth_carrier(name: str, *, seed_offset: int = 0, seed: int | None = None) -> np.ndarray:
     """Render one deterministic clip as float RGB frames in [0, 1]."""
-    rng = np.random.default_rng(_carrier_seed(name) + seed_offset)
+    rng = np.random.default_rng(_carrier_seed(name) + seed_offset if seed is None else seed)
     frames = np.empty((FRAME_COUNT, HEIGHT, WIDTH, 3), dtype=np.float32)
     axis = np.arange(WIDTH, dtype=np.float32)[None, :]
     rows = np.arange(HEIGHT, dtype=np.float32)[:, None]
@@ -192,7 +192,7 @@ def benchmark_row(
     transform_name: str,
     transform_revision: str,
     parameters: dict[str, object],
-    seed: int,
+    seed: int | None,
     expected: str,
 ) -> dict[str, object]:
     return {
@@ -242,7 +242,8 @@ def build_cohort(
     rows: list[dict[str, object]] = []
 
     for name in carriers:
-        clean = synth_carrier(name)
+        executed_seed = _carrier_seed(name)
+        clean = synth_carrier(name, seed=executed_seed)
         clean_path = encode_clip(ffmpeg, artifacts / f"{name}-clean.mp4", clean)
         marked_frames = np.asarray(videoseal_oracle.embed(model, clean, message), dtype=np.float32)
         marked_path = encode_clip(ffmpeg, artifacts / f"{name}-marked.mp4", marked_frames)
@@ -260,7 +261,7 @@ def build_cohort(
                 transform_name="synthesize-clip",
                 transform_revision=RECIPE_VERSION,
                 parameters={"carrier": name, **geometry, "dependencies": dependencies},
-                seed=_carrier_seed(name),
+                seed=executed_seed,
                 expected="not_detected",
             )
         )
@@ -281,7 +282,7 @@ def build_cohort(
                     **geometry,
                     "dependencies": dependencies,
                 },
-                seed=7,
+                seed=videoseal_oracle.MESSAGE_SEED,
                 expected="detected",
             )
         )
@@ -299,7 +300,7 @@ def build_cohort(
                     transform_name=f"attack-{attack}",
                     transform_revision=RECIPE_VERSION,
                     parameters=attack_parameters(attack, dependencies),
-                    seed=7,
+                    seed=None,
                     expected="unresolved",
                 )
             )
@@ -320,7 +321,7 @@ def build_cohort(
                     transform_name="remove-h264-crf23",
                     transform_revision=RECIPE_VERSION,
                     parameters=attack_parameters("h264_crf23", dependencies),
-                    seed=7,
+                    seed=None,
                     expected="not_detected",
                 )
             )
@@ -351,7 +352,8 @@ def build_cohort(
             )
         )
 
-    hard = synth_carrier("moving_texture", seed_offset=991)
+    hard_seed = _carrier_seed("moving_texture") + 991
+    hard = synth_carrier("moving_texture", seed=hard_seed)
     hard_path = encode_clip(ffmpeg, artifacts / f"{HARD_NEGATIVE_CARRIER}.mp4", hard)
     rows.append(
         benchmark_row(
@@ -365,7 +367,7 @@ def build_cohort(
             transform_name="synthesize-clip",
             transform_revision=RECIPE_VERSION,
             parameters={"carrier": "moving_texture", "seed_offset": 991, "dependencies": dependencies},
-            seed=_carrier_seed(HARD_NEGATIVE_CARRIER) + 991,
+            seed=hard_seed,
             expected="not_detected",
         )
     )

scripts/watermark_forgery_study.py

@@ -134,6 +134,7 @@ def audio_cells(out_dir: Path) -> list[dict[str, object]]:
 
 
 def video_cells(out_dir: Path, ffmpeg: str) -> list[dict[str, object]]:
+    from watermark_benchmark import decode_video_artifact
     from watermark_benchmark_video_cohort import encode_clip
 
     model = videoseal_oracle.load_model()
@@ -143,7 +144,8 @@ def video_cells(out_dir: Path, ffmpeg: str) -> list[dict[str, object]]:
     workdir = out_dir / "videoseal"
     workdir.mkdir()
 
-    def measure(label: str, frames: np.ndarray, path: Path) -> dict[str, object]:
+    def measure(label: str, path: Path) -> np.ndarray:
+        frames, digest = decode_video_artifact(path)
         reading = videoseal_oracle.read(model, frames, message=message_a)
         row: dict[str, object] = {
             "scheme": "videoseal",
@@ -151,11 +153,12 @@ def video_cells(out_dir: Path, ffmpeg: str) -> list[dict[str, object]]:
             "bit_accuracy_vs_a": round(reading.bit_accuracy, 6),
             "bit_accuracy_vs_b": round(bit_accuracy(reading.decoded_bits, message_b), 6),
             "detected_by_matched_rule": reading.detected,
-            "sha256": sha256_file(path),
+            "sha256": digest,
+            "measurement_domain": "decoded_artifact",
         }
         rows.append(row)
         log.info("%s", json.dumps(row, sort_keys=True))
-        return row
+        return frames
 
     for carrier_name, clean in (
         ("moving_gradient", synth_video_carrier("moving_gradient")[:CLIP_FRAMES]),
@@ -163,32 +166,27 @@ def video_cells(out_dir: Path, ffmpeg: str) -> list[dict[str, object]]:
     ):
         prefix = f"{carrier_name}-"
         clean_path = encode_clip(ffmpeg, workdir / f"{prefix}clean.mp4", clean)
-        measure(f"{carrier_name}/clean", clean, clean_path)
+        clean = measure(f"{carrier_name}/clean", clean_path)
 
         marked_a = np.asarray(videoseal_oracle.embed(model, clean, message_a), dtype=np.float32)
         marked_a_path = encode_clip(ffmpeg, workdir / f"{prefix}marked_a.mp4", marked_a)
-        measure(f"{carrier_name}/marked_a", marked_a, marked_a_path)
+        marked_a = measure(f"{carrier_name}/marked_a", marked_a_path)
 
         removed_path = apply_crf(ffmpeg, marked_a_path, 23, workdir)
-        from watermark_benchmark import _decode_video
-
-        removed_frames = _decode_video(removed_path)
-        if removed_frames is None:
-            raise SystemExit(f"removed artifact failed to decode: {removed_path}")
-        measure(f"{carrier_name}/removed_crf23", removed_frames, removed_path)
+        removed_frames = measure(f"{carrier_name}/removed_crf23", removed_path)
 
         forged_on_clean = np.asarray(videoseal_oracle.embed(model, clean, message_b), dtype=np.float32)
         path = encode_clip(ffmpeg, workdir / f"{prefix}forged_b_on_clean.mp4", forged_on_clean)
-        measure(f"{carrier_name}/forged_b_on_clean", forged_on_clean, path)
+        measure(f"{carrier_name}/forged_b_on_clean", path)
 
         forged_on_marked = np.asarray(videoseal_oracle.embed(model, marked_a, message_b), dtype=np.float32)
         path = encode_clip(ffmpeg, workdir / f"{prefix}forged_b_on_marked.mp4", forged_on_marked)
-        measure(f"{carrier_name}/forged_b_on_marked_a", forged_on_marked, path)
+        measure(f"{carrier_name}/forged_b_on_marked_a", path)
 
         if removed_frames.shape[0] == marked_a.shape[0]:
             forged_on_removed = np.asarray(videoseal_oracle.embed(model, removed_frames, message_b), dtype=np.float32)
             path = encode_clip(ffmpeg, workdir / f"{prefix}forged_b_on_removed.mp4", forged_on_removed)
-            measure(f"{carrier_name}/forged_b_on_removed_a", forged_on_removed, path)
+            measure(f"{carrier_name}/forged_b_on_removed_a", path)
 
     rows.append(
         {