Add audio and video watermark benchmark oracles, cohorts, and research studies

Extend the development-only benchmark family from images to audio and
video, and close four research directions from the watermark-landscape
map with local, revision-pinned evidence.

Benchmark schema: media_type audio and video rows, audioseal and
videoseal adapters with media-checked adapter binding, sample-domain
audio fidelity (SNR with unbounded_identical), frame-domain video
fidelity (mean PSNR, changed-frame fraction), and per-frame temporal
evidence in videoseal detection records. The report accepts all three
media types and carries SNR columns.

Audio: shared pinned AudioSeal oracle, synthetic and system-voice
speech cohorts with matched negatives, codec, noise, removed, and
forged arms, and the provenance experiment proving visual-only video
cleaning leaves the audio bitstream and watermark verdict invariant.

Video: TorchScript-only VideoSeal oracle (the pip package is unusable
on macOS arm64 and its loader resolves paths relative to cwd), video
cohort with crf/scale/fps attacks and four-state arms, temporal
evaluation measuring the crf decay curve, the frame-aggregation matrix
(computed locally after finding the TorchScript build ignores the
aggregation argument), and real provider clips as carriers.

Forgery: double-embedding study showing the last writer wins completely
under both oracles, scoped remover-trace probe, and the forged/
removed cohort arms encoding each scheme's correct expected verdict
(presence rule for audioseal, message rule for videoseal).

C2PA resolution: measured the five vendors' resolver contracts and the
C2PA 2.2 18.10.5 privacy gradient; product decision recorded that this
tool never resolves over the network. A registry guard pins the
published resolution-API set.

Gate: ruff and pyright src/ clean, 2114 passed / 3 skipped; maintain.sh
stops only on the known unfixed lightning 2.6.5 CVEs, pre-existing.

tests/test_audioseal_experiment.py

@@ -0,0 +1,151 @@
+"""Tests for the pure helpers of the AudioSeal experiment script.
+
+The oracle-dependent stages (embedding, detection, ffmpeg arms) stay
+development-only and are exercised by the recorded local runs; these tests
+cover the deterministic helpers that must not drift.
+"""
+
+from __future__ import annotations
+
+import shutil
+import struct
+import sys
+import wave
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO / "scripts"))
+
+from audioseal_experiment import (  # noqa: E402
+    CARRIERS,
+    SAMPLE_RATE,
+    SPEECH_VOICES,
+    assert_distinct_voices,
+    message_bits,
+    moving_background,
+    partial_noise_attack,
+    snr_db,
+    sora_like_mark,
+    stamped_frame,
+    synthesize_speech,
+    tone_stack_carrier,
+    wav_pcm16_bytes,
+)
+
+
+class TestCarriers:
+    def test_carriers_are_deterministic_per_seed(self) -> None:
+        n = 4000
+        first = tone_stack_carrier(n, np.random.default_rng(11))
+        second = tone_stack_carrier(n, np.random.default_rng(11))
+        np.testing.assert_array_equal(first, second)
+
+    def test_different_seeds_produce_different_carriers(self) -> None:
+        n = 4000
+        first = tone_stack_carrier(n, np.random.default_rng(11))
+        second = tone_stack_carrier(n, np.random.default_rng(12))
+        assert not np.array_equal(first, second)
+
+    def test_registered_carriers_return_expected_lengths(self) -> None:
+        n = 1000
+        for name, factory in CARRIERS.items():
+            samples = np.asarray(factory(n, np.random.default_rng(3)), dtype=np.float32)
+            assert samples.shape == (n,), name
+            assert np.isfinite(samples).all(), name
+
+
+class TestMessage:
+    def test_message_is_stable_and_binary(self) -> None:
+        bits = message_bits()
+        assert bits == message_bits()
+        assert len(bits) == 16
+        assert set(bits) <= {0, 1}
+
+
+class TestWavWriter:
+    def test_wav_bytes_round_trip_through_wave_module(self, tmp_path: Path) -> None:
+        samples = np.linspace(-0.9, 0.9, 256, dtype=np.float32)
+        path = tmp_path / "out.wav"
+        path.write_bytes(wav_pcm16_bytes(samples, SAMPLE_RATE))
+        with wave.open(str(path)) as reader:
+            assert reader.getnchannels() == 1
+            assert reader.getsampwidth() == 2
+            assert reader.getframerate() == SAMPLE_RATE
+            decoded = np.frombuffer(reader.readframes(reader.getnframes()), dtype="<i2")
+        np.testing.assert_allclose(decoded.astype(np.float32) / 32767.0, samples, atol=1 / 32767.0)
+
+    def test_wav_header_declares_exact_payload_length(self) -> None:
+        samples = np.zeros(10, dtype=np.float32)
+        data = wav_pcm16_bytes(samples, SAMPLE_RATE)
+        assert data[:4] == b"RIFF"
+        assert struct.unpack("<I", data[4:8])[0] == 36 + 20
+        assert data[36:40] == b"data"
+        assert struct.unpack("<I", data[40:44])[0] == 20
+
+
+class TestSnr:
+    def test_zero_residual_is_infinite_headroom(self) -> None:
+        signal = np.ones(100, dtype=np.float32)
+        assert snr_db(signal, np.zeros(100, dtype=np.float32)) > 100.0
+
+    def test_known_ratio(self) -> None:
+        signal = np.ones(1000, dtype=np.float32)
+        residual = np.full(1000, 0.1, dtype=np.float32)
+        assert snr_db(signal, residual) == pytest.approx(20.0, abs=0.01)
+
+
+class TestVideoFrames:
+    def test_mark_stamps_only_its_region(self) -> None:
+        clean = moving_background(5)
+        mark, mark_x, mark_y = sora_like_mark()
+        stamped = stamped_frame(5, mark, mark_x, mark_y)
+        height, width = mark.shape
+        outside = (
+            slice(0, mark_y),
+            slice(0, mark_x),
+        )
+        np.testing.assert_array_equal(stamped[outside], clean[outside])
+        assert not np.array_equal(
+            stamped[mark_y : mark_y + height, mark_x : mark_x + width],
+            clean[mark_y : mark_y + height, mark_x : mark_x + width],
+        )
+
+    def test_mark_geometry_matches_the_test_clip_layout(self) -> None:
+        mark, mark_x, mark_y = sora_like_mark()
+        assert mark.shape == (44, 124)
+        assert mark_x + 124 <= 840
+        assert mark_y + 44 <= 480
+
+
+class TestSpeechCarriers:
+    def test_assert_distinct_voices_rejects_aliased_bytes(self) -> None:
+        with pytest.raises(SystemExit, match="byte-identical"):
+            assert_distinct_voices({"Samantha": b"abc", "Alex": b"abc"})
+
+    def test_assert_distinct_voices_accepts_distinct_bytes(self) -> None:
+        assert_distinct_voices({"Samantha": b"abc", "Daniel": b"abd"})
+
+    def test_partial_noise_attack_touches_only_its_window(self) -> None:
+        samples = np.ones(4 * SAMPLE_RATE, dtype=np.float32)
+        attacked = partial_noise_attack(samples, 1.0, 2.0)
+
+        np.testing.assert_array_equal(attacked[:SAMPLE_RATE], samples[:SAMPLE_RATE])
+        np.testing.assert_array_equal(attacked[2 * SAMPLE_RATE :], samples[2 * SAMPLE_RATE :])
+        assert not np.array_equal(attacked[SAMPLE_RATE : 2 * SAMPLE_RATE], samples[SAMPLE_RATE : 2 * SAMPLE_RATE])
+
+    @pytest.mark.skipif(shutil.which("say") is None, reason="requires the macOS say tool")
+    def test_system_speech_is_deterministic_per_voice(self, tmp_path: Path) -> None:
+        first = synthesize_speech("Samantha", tmp_path)
+        (tmp_path / "again").mkdir()
+        synthesize_speech("Samantha", tmp_path / "again")
+
+        assert first.read_bytes() == (tmp_path / "again" / "speech-Samantha.aiff").read_bytes()
+
+    @pytest.mark.skipif(shutil.which("say") is None, reason="requires the macOS say tool")
+    def test_configured_voices_render_distinct_bytes(self, tmp_path: Path) -> None:
+        rendered = {voice: synthesize_speech(voice, tmp_path).read_bytes() for voice in SPEECH_VOICES}
+
+        assert_distinct_voices(rendered)

tests/test_audioseal_oracle.py

@@ -0,0 +1,41 @@
+"""Tests for the shared AudioSeal oracle's pure helpers."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+from audioseal_oracle import SAMPLE_RATE, window_bounds  # noqa: E402
+
+
+class TestWindowBounds:
+    def test_exact_multiple_covers_every_sample_once(self) -> None:
+        bounds = window_bounds(32_000, 1.0, SAMPLE_RATE)
+
+        assert bounds == [(0, 16_000), (16_000, 32_000)]
+
+    def test_trailing_remainder_keeps_its_own_window(self) -> None:
+        bounds = window_bounds(32_500, 1.0, SAMPLE_RATE)
+
+        assert bounds[-1] == (32_000, 32_500)
+        assert len(bounds) == 3
+
+    def test_short_signal_is_one_partial_window(self) -> None:
+        assert window_bounds(100, 1.0, SAMPLE_RATE) == [(0, 100)]
+
+    def test_empty_signal_has_no_windows(self) -> None:
+        assert window_bounds(0, 1.0, SAMPLE_RATE) == []
+
+    @pytest.mark.parametrize(
+        ("total", "window_s", "rate"),
+        [(-1, 1.0, 16_000), (100, 0.0, 16_000), (100, -1.0, 16_000), (100, 1.0, 0), (100, 0.00001, 16_000)],
+    )
+    def test_rejects_degenerate_arguments(self, total: int, window_s: float, rate: int) -> None:
+        with pytest.raises(ValueError, match=r"total_samples|window_s|sample_rate"):
+            window_bounds(total, window_s, rate)

tests/test_c2pa_registry_sync.py

@@ -78,3 +78,37 @@ def test_official_source_is_fetched_through_its_resolved_revision(monkeypatch: p
     assert payload == b"[]"
     assert resolved == revision
     assert requested == [sync.PINNED_SOURCE_URL.format(revision=revision)]
+
+
+def test_packaged_snapshot_preserves_published_resolution_apis():
+    """Guard the resolution-API subset the runtime deliberately never calls.
+
+    Eight algorithms publish ``softBindingResolutionApis`` today. The runtime
+    resolves nothing over the network by design (see
+    docs/c2pa-resolution-research.md for the contracts and the product
+    decision); this guard keeps the packaged snapshot honest about what the
+    ecosystem publishes, so an upstream change surfaces here consciously
+    instead of passing silently through a registry regeneration.
+    """
+    documented = {
+        "ai.trufo.pawprint.watermark": ("https://c2pa.trufo.ai/v1",),
+        "ai.trufo.pawprint.fingerprint": ("https://c2pa.trufo.ai/v1",),
+        "com.aiwatermark.videoseal.1": ("https://aiwatermark.com/api/v1",),
+        "com.aiwatermark.pixelseal.1": ("https://aiwatermark.com/api/v1",),
+        "com.aiwatermark.audioseal.1": ("https://aiwatermark.com/api/v1",),
+        "me.deepmark.audio.vigil.128": ("https://resolution-api.deepmark.me",),
+        "io.blockfact.audio.watermark.32": ("https://api.blockfact.io/api/soft-binding/v1",),
+        "com.joinmonolith.sha256": ("https://api.joinmonolith.com/api/c2pa",),
+    }
+    packaged = {entry.algorithm: entry.resolution_apis for entry in C2PA_SOFT_BINDING_REGISTRY}
+
+    for algorithm, apis in documented.items():
+        assert packaged.get(algorithm) == apis, (
+            f"{algorithm} resolution APIs changed: {packaged.get(algorithm)} != {apis}; "
+            "update docs/c2pa-resolution-research.md with the new contract"
+        )
+    with_apis = {algorithm for algorithm, apis in packaged.items() if apis}
+    assert with_apis == set(documented), (
+        f"upstream softbinding-algorithm-list resolution-API set changed: {sorted(with_apis)}; "
+        "revisit docs/c2pa-resolution-research.md before regenerating"
+    )

tests/test_videoseal_oracle.py

@@ -0,0 +1,97 @@
+"""Tests for the pinned VideoSeal TorchScript oracle's pure helpers."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+import videoseal_oracle  # noqa: E402
+
+
+class TestMessage:
+    def test_message_is_deterministic_and_256_bits(self) -> None:
+        first = videoseal_oracle.message_bits()
+        second = videoseal_oracle.message_bits()
+
+        assert first == second
+        assert len(first) == 256
+        assert set(first) <= {0, 1}
+
+    def test_seed_changes_the_message(self) -> None:
+        assert videoseal_oracle.message_bits(seed=8) != videoseal_oracle.message_bits(seed=7)
+
+
+class TestLabel:
+    def test_label_is_hex_of_the_decoded_bits(self) -> None:
+        bits = tuple([1] * 8 + [0] * 8)
+        reading = videoseal_oracle.VideoSealReading(
+            bit_accuracy=1.0,
+            decoded_bits=bits,
+            per_frame_bit_accuracy=(1.0,),
+        )
+
+        assert reading.label == "ff00"
+
+    def test_detection_rule_is_the_documented_threshold(self) -> None:
+        below = videoseal_oracle.VideoSealReading(0.89, (0,), ())
+        exact = videoseal_oracle.VideoSealReading(0.9, (0,), ())
+
+        assert below.detected is False
+        assert exact.detected is True
+
+
+class TestPins:
+    def test_checkpoint_pin_is_a_sha256_digest(self) -> None:
+        digest = videoseal_oracle.JIT_SHA256
+
+        assert len(digest) == 64
+        assert all(char in "0123456789abcdef" for char in digest)
+
+    def test_source_commit_is_pinned(self) -> None:
+        assert len(videoseal_oracle.SOURCE_COMMIT) == 40
+
+
+class TestCacheOverride:
+    def test_cache_dir_env_override_is_honored(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+        monkeypatch.setenv("VIDEOSEAL_CACHE_DIR", str(tmp_path / "cache"))
+
+        import importlib
+
+        importlib.reload(videoseal_oracle)
+
+        try:
+            assert tmp_path / "cache" == videoseal_oracle.CACHE_DIR
+        finally:
+            importlib.reload(videoseal_oracle)
+
+
+class TestAggregationFormulas:
+    def test_aggregations_match_the_upstream_formulas(self) -> None:
+        torch = pytest.importorskip("torch")
+        # Two frames, four bits; hand-computed from the upstream extract_message
+        # formulas so a silent mutation of the port fails here.
+        bit_preds = torch.tensor([[2.0, -1.0, 0.5, -3.0], [0.1, 0.2, -0.4, 0.3]])
+
+        avg = videoseal_oracle._aggregate_bit_preds(bit_preds, "avg")
+        squared = videoseal_oracle._aggregate_bit_preds(bit_preds, "squared_avg")
+        l1 = videoseal_oracle._aggregate_bit_preds(bit_preds, "l1norm_avg")
+        l2 = videoseal_oracle._aggregate_bit_preds(bit_preds, "l2norm_avg")
+
+        torch.testing.assert_close(avg, bit_preds.mean(dim=0))
+        torch.testing.assert_close(squared, (bit_preds * bit_preds.abs()).mean(dim=0))
+        w1 = torch.norm(bit_preds, p=1, dim=1).unsqueeze(1)
+        torch.testing.assert_close(l1, (bit_preds * w1).mean(dim=0))
+        w2 = torch.norm(bit_preds, p=2, dim=1).unsqueeze(1)
+        torch.testing.assert_close(l2, (bit_preds * w2).mean(dim=0))
+
+    def test_unknown_aggregation_is_rejected(self) -> None:
+        torch = pytest.importorskip("torch")
+
+        with pytest.raises(ValueError, match="unknown aggregation"):
+            videoseal_oracle._aggregate_bit_preds(torch.zeros(2, 4), "median")

tests/test_watermark_benchmark.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import hashlib
 import importlib.util
 import json
+import shutil
 import sys
 from pathlib import Path
 from typing import Any
@@ -392,9 +393,11 @@ def test_jsonl_writer_refuses_overwrite_and_emits_strict_json(tmp_path: Path) ->
 def test_builtin_adapters_name_their_exact_source_files() -> None:
     adapters = MODULE.default_adapters()
 
-    assert set(adapters) == {"dwt-dct", "trustmark"}
+    assert set(adapters) == {"dwt-dct", "trustmark", "audioseal", "videoseal"}
     assert adapters["dwt-dct"].source_file.name == "invisible_watermark.py"
     assert adapters["trustmark"].source_file.name == "trustmark_detector.py"
+    assert adapters["audioseal"].source_file.name == "audioseal_oracle.py"
+    assert adapters["videoseal"].source_file.name == "videoseal_oracle.py"
 
 
 def test_run_benchmark_caches_only_recurring_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -446,3 +449,314 @@ def test_run_benchmark_writes_official_trustmark_result(tmp_path: Path) -> None:
     assert record["detection"]["status"] == "detected"
     assert record["detection"]["label"] == "Adobe TrustMark (variant P, schema 1)"
     assert record["case_id"] == "case-1"
+
+
+# --- Audio rows (schema v1, media_type "audio", adapter "audioseal") --------
+
+
+def _wav(path: Path, samples: np.ndarray) -> Path:
+    import sys
+
+    scripts = Path(__file__).resolve().parents[1] / "scripts"
+    if str(scripts) not in sys.path:
+        sys.path.insert(0, str(scripts))
+    from audioseal_experiment import wav_pcm16_bytes
+
+    path.write_bytes(wav_pcm16_bytes(samples, 16_000))
+    return path
+
+
+def _audio_row(artifact: Path, *, reference: Path | None = None, adapter: str = "audioseal") -> dict[str, Any]:
+    return {
+        "schema_version": 1,
+        "case_id": "audio-case-1",
+        "pair_id": "audio-pair-1",
+        "media_type": "audio",
+        "adapter": adapter,
+        "arm": "positive",
+        "state": "marked",
+        "path": artifact.name,
+        "sha256": _sha256(artifact),
+        "reference_path": reference.name if reference is not None else None,
+        "reference_sha256": _sha256(reference) if reference is not None else None,
+        "source_revision": "fixture-set@abc123",
+        "transform": {"name": "audioseal-embed", "revision": "builtin-v1", "parameters": {}},
+        "seed": 7,
+        "expected": "detected",
+    }
+
+
+def test_manifest_accepts_audio_rows(tmp_path: Path) -> None:
+    artifact = _wav(tmp_path / "marked.wav", np.zeros(1600, dtype=np.float32))
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [_audio_row(artifact)])
+
+    row = MODULE.load_manifest(manifest)[0]
+
+    assert row.media_type == "audio"
+    assert row.adapter == "audioseal"
+
+
+def test_manifest_rejects_unknown_media_type(tmp_path: Path) -> None:
+    artifact = _image(tmp_path / "marked.png", 100)
+    row = _row(artifact)
+    row["media_type"] = "document"
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [row])
+
+    with pytest.raises(ValueError, match="media_type must be one of image, audio, video"):
+        MODULE.load_manifest(manifest)
+
+
+def test_manifest_rejects_adapter_measuring_another_media_type(tmp_path: Path) -> None:
+    artifact = _wav(tmp_path / "marked.wav", np.zeros(1600, dtype=np.float32))
+    row = _audio_row(artifact)
+    row["adapter"] = "dwt-dct"
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [row])
+
+    with pytest.raises(ValueError, match="'dwt-dct' measures image artifacts, not audio"):
+        MODULE.load_manifest(manifest)
+
+
+def test_manifest_still_accepts_custom_adapter_names_on_audio_rows(tmp_path: Path) -> None:
+    artifact = _wav(tmp_path / "marked.wav", np.zeros(1600, dtype=np.float32))
+    row = _audio_row(artifact, adapter="custom-audio-detector")
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [row])
+
+    assert MODULE.load_manifest(manifest)[0].adapter == "custom-audio-detector"
+
+
+def test_audio_fidelity_reports_unbounded_snr_for_identical_samples(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    samples = np.linspace(-0.5, 0.5, 4096, dtype=np.float32)
+    artifact = _wav(tmp_path / "marked.wav", samples)
+    reference = _wav(tmp_path / "clean.wav", samples.copy())
+    row = _audio_row(artifact, reference=reference, adapter="fake-audio")
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [row])
+    case = MODULE.load_manifest(manifest)[0]
+    decode_calls: list[Path] = []
+
+    def decode(path: Path) -> Any:
+        decode_calls.append(path)
+        return samples
+
+    fidelity = MODULE._audio_fidelity(case, samples, decode)
+
+    assert fidelity["status"] == "measured"
+    assert fidelity["identical"] is True
+    assert fidelity["snr_db"] is None
+    assert fidelity["snr_status"] == "unbounded_identical"
+    assert decode_calls == [reference.resolve()]
+
+
+def test_audio_fidelity_marks_length_mismatch_incomparable(tmp_path: Path) -> None:
+    short = np.zeros(1000, dtype=np.float32)
+    long = np.zeros(2000, dtype=np.float32)
+    artifact = _wav(tmp_path / "marked.wav", short)
+    reference = _wav(tmp_path / "clean.wav", long)
+    row = _audio_row(artifact, reference=reference, adapter="fake-audio")
+    case = MODULE.load_manifest(_write_manifest(tmp_path / "manifest.jsonl", [row]))[0]
+
+    fidelity = MODULE._audio_fidelity(case, short, lambda _path: long)
+
+    assert fidelity["status"] == "incomparable"
+    assert fidelity["reason"] == "length_mismatch"
+    assert fidelity["artifact_samples"] == 1000
+    assert fidelity["reference_samples"] == 2000
+
+
+def test_run_benchmark_evaluates_audio_rows_end_to_end(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("audio rows decode through the system ffmpeg")
+    samples = np.linspace(-0.5, 0.5, 16000, dtype=np.float32)
+    artifact = _wav(tmp_path / "marked.wav", samples)
+    reference = _wav(tmp_path / "clean.wav", samples * 0.99)
+    manifest = _write_manifest(
+        tmp_path / "manifest.jsonl",
+        [_audio_row(artifact, reference=reference)],
+    )
+    adapter = _FakeAdapter(status="detected", label="1111111000011001")
+    monkeypatch.setattr(MODULE, "default_adapters", lambda: {"audioseal": adapter})
+    output = tmp_path / "result.jsonl"
+
+    counts = MODULE.run_benchmark(manifest, output)
+
+    record = json.loads(output.read_text(encoding="utf-8"))
+    assert counts == {"detected": 1}
+    assert record["media_type"] == "audio"
+    assert record["detection"]["label"] == "1111111000011001"
+    assert record["fidelity"]["status"] == "measured"
+    assert record["fidelity"]["snr_status"] == "measured"
+    assert record["fidelity"]["snr_db"] > 30.0
+    assert adapter.calls == [artifact.resolve()]
+
+
+# --- Video rows (schema v1, media_type "video", adapter "videoseal") --------
+
+
+def _video(path: Path, frames: int = 4, width: int = 64, height: int = 48) -> Path:
+    import subprocess
+
+    payload = b"".join(bytes([frame_index * 12 % 256]) * (height * width * 3) for frame_index in range(frames))
+    subprocess.run(  # noqa: S603 - resolved ffmpeg with fixed arguments
+        [
+            shutil.which("ffmpeg"),
+            "-y",
+            "-loglevel",
+            "error",
+            "-f",
+            "rawvideo",
+            "-pix_fmt",
+            "rgb24",
+            "-s:v",
+            f"{width}x{height}",
+            "-r",
+            "12",
+            "-i",
+            "pipe:0",
+            "-c:v",
+            "libx264",
+            "-preset",
+            "ultrafast",
+            "-crf",
+            "8",
+            "-pix_fmt",
+            "yuv420p",
+            str(path),
+        ],
+        input=payload,
+        check=True,
+    )
+    return path
+
+
+def _video_row(artifact: Path, *, reference: Path | None = None, adapter: str = "videoseal") -> dict[str, Any]:
+    return {
+        "schema_version": 1,
+        "case_id": "video-case-1",
+        "pair_id": "video-pair-1",
+        "media_type": "video",
+        "adapter": adapter,
+        "arm": "positive",
+        "state": "marked",
+        "path": artifact.name,
+        "sha256": _sha256(artifact),
+        "reference_path": reference.name if reference is not None else None,
+        "reference_sha256": _sha256(reference) if reference is not None else None,
+        "source_revision": "fixture-set@abc123",
+        "transform": {"name": "videoseal-embed", "revision": "builtin-v1", "parameters": {}},
+        "seed": 7,
+        "expected": "detected",
+    }
+
+
+def test_manifest_accepts_video_rows(tmp_path: Path) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    artifact = _video(tmp_path / "marked.mp4")
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [_video_row(artifact)])
+
+    row = MODULE.load_manifest(manifest)[0]
+
+    assert row.media_type == "video"
+    assert row.adapter == "videoseal"
+
+
+def test_manifest_rejects_image_adapter_on_video_rows(tmp_path: Path) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    artifact = _video(tmp_path / "marked.mp4")
+    row = _video_row(artifact)
+    row["adapter"] = "trustmark"
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [row])
+
+    with pytest.raises(ValueError, match="'trustmark' measures image artifacts, not video"):
+        MODULE.load_manifest(manifest)
+
+
+def test_video_fidelity_marks_shape_mismatch_incomparable(tmp_path: Path) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    artifact = _video(tmp_path / "marked.mp4", frames=4)
+    reference = _video(tmp_path / "clean.mp4", frames=3)
+    row = _video_row(artifact, reference=reference, adapter="fake-video")
+    case = MODULE.load_manifest(_write_manifest(tmp_path / "manifest.jsonl", [row]))[0]
+
+    fidelity = MODULE._video_fidelity(case, np.zeros((4, 48, 64, 3)), lambda _path: np.zeros((3, 48, 64, 3)))
+
+    assert fidelity["status"] == "incomparable"
+    assert fidelity["reason"] == "shape_mismatch"
+
+
+def test_video_fidelity_reports_unbounded_psnr_for_identical_frames(tmp_path: Path) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    frames = np.full((4, 48, 64, 3), 0.5, dtype=np.float32)
+    artifact = _video(tmp_path / "marked.mp4")
+    reference = _video(tmp_path / "clean.mp4")
+    row = _video_row(artifact, reference=reference, adapter="fake-video")
+    case = MODULE.load_manifest(_write_manifest(tmp_path / "manifest.jsonl", [row]))[0]
+
+    fidelity = MODULE._video_fidelity(case, frames, lambda _path: frames)
+
+    assert fidelity["status"] == "measured"
+    assert fidelity["identical"] is True
+    assert fidelity["mean_psnr_db"] is None
+    assert fidelity["psnr_status"] == "unbounded_identical"
+
+
+def test_run_benchmark_evaluates_video_rows_end_to_end(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    artifact = _video(tmp_path / "marked.mp4")
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [_video_row(artifact)])
+    adapter = _FakeAdapter(status="detected", label="a1b2")
+    monkeypatch.setattr(MODULE, "default_adapters", lambda: {"videoseal": adapter})
+    output = tmp_path / "result.jsonl"
+
+    counts = MODULE.run_benchmark(manifest, output)
+
+    record = json.loads(output.read_text(encoding="utf-8"))
+    assert counts == {"detected": 1}
+    assert record["media_type"] == "video"
+    assert record["detection"]["label"] == "a1b2"
+    assert record["fidelity"]["status"] == "not_measured"
+
+
+def test_detection_record_carries_temporal_evidence_when_present() -> None:
+    outcome = MODULE.DetectorOutcome(status="detected", label="ff00", temporal=(0.8, 0.9, 1.0))
+
+    record = MODULE._detection_record(outcome, "detected", 1.5)
+
+    temporal = record["temporal"]
+    assert temporal["frames"] == 3
+    assert temporal["min_bit_accuracy"] == 0.8
+    assert temporal["mean_bit_accuracy"] == pytest.approx(0.9)
+    assert temporal["max_bit_accuracy"] == 1.0
+    assert temporal["per_frame_bit_accuracy"] == [0.8, 0.9, 1.0]
+
+
+def test_detection_record_omits_temporal_for_non_temporal_adapters() -> None:
+    outcome = MODULE.DetectorOutcome(status="not_detected", label=None)
+
+    assert "temporal" not in MODULE._detection_record(outcome, "not_detected", None)
+
+
+def test_run_benchmark_records_videoseal_temporal_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("video rows decode through the system ffmpeg")
+    artifact = _video(tmp_path / "marked.mp4")
+    manifest = _write_manifest(tmp_path / "manifest.jsonl", [_video_row(artifact)])
+
+    class TemporalAdapter(_FakeAdapter):
+        def detect(self, path: Path, frames: Any) -> Any:
+            self.calls.append(path)
+            return MODULE.DetectorOutcome(status=self.status, label=self.label, temporal=(0.5, 0.75, 1.0))
+
+    adapter = TemporalAdapter(status="detected", label="ff00")
+    monkeypatch.setattr(MODULE, "default_adapters", lambda: {"videoseal": adapter})
+    output = tmp_path / "result.jsonl"
+
+    MODULE.run_benchmark(manifest, output)
+
+    record = json.loads(output.read_text(encoding="utf-8"))
+    assert record["detection"]["temporal"]["frames"] == 3

tests/test_watermark_benchmark_audio_cohort.py

@@ -0,0 +1,126 @@
+"""Contracts for the audio benchmark cohort builder."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
+
+
+def _load(name: str, filename: str) -> object:
+    spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+COHORT = _load("watermark_benchmark_audio_cohort", "watermark_benchmark_audio_cohort.py")
+EXPERIMENT = _load("audioseal_experiment_cohort_test", "audioseal_experiment.py")
+BENCHMARK = sys.modules.get("watermark_benchmark") or _load("watermark_benchmark", "watermark_benchmark.py")
+sys.path.insert(0, str(SCRIPTS))
+import audioseal_oracle  # noqa: E402
+
+pytestmark = pytest.mark.skipif(
+    shutil.which("ffmpeg") is None or not audioseal_oracle.available(),
+    reason="audio cohort builder requires ffmpeg and the dev extra (audioseal, torch)",
+)
+
+
+def test_builder_refuses_existing_output_directory(tmp_path: Path) -> None:
+    (tmp_path / "cohort").mkdir()
+
+    with pytest.raises(FileExistsError):
+        COHORT.build_cohort(tmp_path / "cohort", duration_s=0.2)
+
+
+def test_builder_rejects_unknown_carriers_and_attacks(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="unknown carriers"):
+        COHORT.build_cohort(tmp_path / "a", carriers=("silence",), duration_s=0.2)
+    with pytest.raises(ValueError, match="unknown attacks"):
+        COHORT.build_cohort(tmp_path / "b", attacks=("reverb",), duration_s=0.2)
+
+
+def test_built_cohort_manifest_is_strict_and_complete(tmp_path: Path) -> None:
+    manifest = COHORT.build_cohort(tmp_path / "cohort", duration_s=0.5)
+
+    cases = BENCHMARK.load_manifest(manifest)
+    assert len(cases) == 20
+    by_case = {case.case_id: case for case in cases}
+    assert by_case["tone_stack-removed"].state == "removed"
+    assert by_case["tone_stack-removed"].expected == "not_detected"
+    assert by_case["tone_stack-forged"].state == "forged"
+    assert by_case["tone_stack-forged"].arm == "wrong_key"
+    assert by_case["tone_stack-forged"].expected == "detected"
+    assert by_case["tone_stack-clean"].arm == "matched_negative"
+    assert by_case["tone_stack-clean"].expected == "not_detected"
+    assert by_case["tone_stack-marked"].arm == "positive"
+    assert by_case["tone_stack-mp3_128k"].state == "attacked"
+    assert by_case["tone_stack-mp3_128k"].expected == "unresolved"
+    assert by_case["white_noise_hard_negative-clean"].arm == "hard_negative"
+    assert all(case.media_type == "audio" for case in cases)
+    assert all(case.adapter == "audioseal" for case in cases)
+    assert all(case.path.is_file() for case in cases)
+    marked = by_case["tone_stack-marked"]
+    assert marked.reference_path == by_case["tone_stack-clean"].path
+
+    rows = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
+    embed_parameters = next(row for row in rows if row["case_id"] == "tone_stack-marked")
+    assert embed_parameters["transform"]["parameters"]["message"] == EXPERIMENT.message_bits()
+
+
+def test_carrier_artifacts_are_deterministic_across_rebuilds(tmp_path: Path) -> None:
+    manifest = COHORT.build_cohort(tmp_path / "cohort", duration_s=0.5)
+
+    cases = {case.case_id: case for case in BENCHMARK.load_manifest(manifest)}
+    first = (tmp_path / "cohort" / "artifacts" / "tone_stack-clean.wav").read_bytes()
+    second = EXPERIMENT.wav_pcm16_bytes(EXPERIMENT.synth_carrier("tone_stack", 0.5), 16_000)
+    assert first == second
+    rebuilt = COHORT.build_cohort(tmp_path / "cohort-2", duration_s=0.5)
+    cases_2 = {case.case_id: case for case in BENCHMARK.load_manifest(rebuilt)}
+    for case_id, case in cases.items():
+        if case.state == "clean":
+            assert case.sha256 == cases_2[case_id].sha256, case_id
+
+
+def test_hard_negative_is_distinct_from_every_marked_carrier(tmp_path: Path) -> None:
+    hard = COHORT.synth_carrier_hard_negative(0.5)
+    carrier = EXPERIMENT.synth_carrier("white_noise", 0.5)
+
+    assert hard.shape == carrier.shape
+    assert not np.array_equal(hard, carrier)
+
+
+@pytest.mark.skipif(shutil.which("say") is None, reason="speech carriers require the macOS say tool")
+def test_speech_extension_adds_provenance_backed_rows(tmp_path: Path) -> None:
+    manifest = COHORT.build_cohort(tmp_path / "cohort", duration_s=0.5, include_speech=True)
+
+    cases = BENCHMARK.load_manifest(manifest)
+    assert len(cases) == 35
+    speech = {case.case_id: case for case in cases if case.pair_id.startswith("speech_")}
+    assert len(speech) == 15
+    assert {case.pair_id for case in speech.values()} == {
+        "speech_Samantha",
+        "speech_Daniel",
+        "speech_Milena",
+    }
+    marked = next(case for case in speech.values() if case.state == "marked")
+    assert marked.reference_path is not None
+    digests = {case.sha256 for case in speech.values() if case.state == "clean"}
+    assert len(digests) == 3, "aliased system voices would collapse clean-carrier digests"
+
+    rows = {json.loads(line)["case_id"]: json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()}
+    tts = rows["speech_Samantha-clean"]["transform"]["parameters"]["tts"]
+    assert tts["tts"] == "say"
+    assert tts["macos"]
+    assert tts["voice"] == "Samantha"
+    assert tts["text_sha256"]

tests/test_watermark_benchmark_report.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import hashlib
 import importlib.util
 import json
+import shutil
 import sys
 from pathlib import Path
 from typing import Any
@@ -325,3 +327,73 @@ def test_report_refuses_overwrite_and_preserves_interpretation_caveat(tmp_path:
     assert "not evidence that a watermark is absent" in report
     with pytest.raises(FileExistsError, match="refusing to overwrite"):
         MODULE.write_report(output, report)
+
+
+def test_report_accepts_audio_rows_and_reports_snr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    if shutil.which("ffmpeg") is None:
+        pytest.skip("audio rows decode through the system ffmpeg")
+    import sys
+
+    scripts = Path(__file__).resolve().parents[1] / "scripts"
+    if str(scripts) not in sys.path:
+        sys.path.insert(0, str(scripts))
+    from audioseal_experiment import wav_pcm16_bytes
+
+    samples = np.linspace(-0.5, 0.5, 16000, dtype=np.float32)
+    artifact = tmp_path / "marked.wav"
+    reference = tmp_path / "clean.wav"
+    artifact.write_bytes(wav_pcm16_bytes(samples, 16_000))
+    reference.write_bytes(wav_pcm16_bytes(samples * 0.9, 16_000))
+
+    def sha(path: Path) -> str:
+        return hashlib.sha256(path.read_bytes()).hexdigest()
+
+    manifest = tmp_path / "manifest.jsonl"
+    manifest.write_text(
+        json.dumps(
+            {
+                "schema_version": 1,
+                "case_id": "audio-1",
+                "pair_id": "audio-1",
+                "media_type": "audio",
+                "adapter": "audioseal",
+                "arm": "positive",
+                "state": "marked",
+                "path": artifact.name,
+                "sha256": sha(artifact),
+                "reference_path": reference.name,
+                "reference_sha256": sha(reference),
+                "source_revision": "fixture@1",
+                "transform": {"name": "embed", "revision": "v1", "parameters": {}},
+                "seed": 1,
+                "expected": "detected",
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    case = BENCHMARK.load_manifest(manifest)[0]
+    monkeypatch.setattr(
+        BENCHMARK,
+        "default_adapters",
+        lambda: {"audioseal": _FakeAdapter("detected")},
+    )
+    record = BENCHMARK.evaluate_case(
+        case,
+        adapters=BENCHMARK.default_adapters(),
+        repository={"commit": "abc", "dirty": False},
+    )
+
+    result_path = tmp_path / "run.jsonl"
+    BENCHMARK.write_jsonl(result_path, [record])
+
+    summary = MODULE.summarize(MODULE.load_results([result_path]))
+    fidelity = next(row for row in summary["fidelity"] if row["adapter"] == "audioseal")
+
+    assert fidelity["measured"] == 1
+    assert fidelity["finite_snr"] == 1
+    assert fidelity["p50_snr_db"] is not None
+    assert fidelity["finite_psnr"] == 0
+
+    markdown = MODULE.render_markdown(summary)
+    assert "p50 SNR dB" in markdown

tests/test_watermark_benchmark_video_cohort.py

@@ -0,0 +1,98 @@
+"""Contracts for the video benchmark cohort builder."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
+
+
+def _load(name: str, filename: str) -> object:
+    spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+COHORT = _load("watermark_benchmark_video_cohort", "watermark_benchmark_video_cohort.py")
+BENCHMARK = sys.modules.get("watermark_benchmark") or _load("watermark_benchmark", "watermark_benchmark.py")
+sys.path.insert(0, str(SCRIPTS))
+import videoseal_oracle  # noqa: E402
+
+pytestmark = pytest.mark.skipif(
+    shutil.which("ffmpeg") is None or not videoseal_oracle.available(),
+    reason="video cohort builder requires ffmpeg and the dev extra (torch)",
+)
+
+
+def test_builder_refuses_existing_output_directory(tmp_path: Path) -> None:
+    (tmp_path / "cohort").mkdir()
+
+    with pytest.raises(FileExistsError):
+        COHORT.build_cohort(tmp_path / "cohort")
+
+
+def test_builder_rejects_unknown_carriers_and_attacks(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="unknown carriers"):
+        COHORT.build_cohort(tmp_path / "a", carriers=("black",))
+    with pytest.raises(ValueError, match="unknown attacks"):
+        COHORT.build_cohort(tmp_path / "b", attacks=("rotate",))
+
+
+def test_carriers_are_deterministic_and_distinct() -> None:
+    first = COHORT.synth_carrier("moving_gradient")
+    second = COHORT.synth_carrier("moving_gradient")
+    texture = COHORT.synth_carrier("moving_texture")
+    hard = COHORT.synth_carrier("moving_texture", seed_offset=991)
+
+    assert first.shape == (COHORT.FRAME_COUNT, COHORT.HEIGHT, COHORT.WIDTH, 3)
+    np.testing.assert_array_equal(first, second)
+    assert not np.array_equal(first, texture)
+    assert not np.array_equal(texture, hard)
+
+
+def test_built_cohort_manifest_is_strict_and_complete(tmp_path: Path) -> None:
+    manifest = COHORT.build_cohort(tmp_path / "cohort")
+
+    cases = BENCHMARK.load_manifest(manifest)
+    assert len(cases) == 15
+    by_case = {case.case_id: case for case in cases}
+    assert by_case["moving_gradient-removed"].state == "removed"
+    assert by_case["moving_gradient-removed"].expected == "not_detected"
+    assert by_case["moving_gradient-forged"].state == "forged"
+    assert by_case["moving_gradient-forged"].arm == "wrong_key"
+    assert by_case["moving_gradient-forged"].expected == "not_detected"
+    assert by_case["moving_gradient-clean"].arm == "matched_negative"
+    assert by_case["moving_gradient-marked"].arm == "positive"
+    assert by_case["moving_gradient-marked"].reference_path == by_case["moving_gradient-clean"].path
+    assert by_case["moving_gradient-h264_crf23"].state == "attacked"
+    assert by_case["moving_gradient-h264_crf23"].expected == "unresolved"
+    assert by_case["moving_texture_hard_negative-clean"].arm == "hard_negative"
+    assert all(case.media_type == "video" for case in cases)
+    assert all(case.adapter == "videoseal" for case in cases)
+    assert all(case.path.is_file() for case in cases)
+
+    rows = {json.loads(line)["case_id"]: json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()}
+    marked = rows["moving_gradient-marked"]["transform"]["parameters"]
+    assert marked["frames"] == COHORT.FRAME_COUNT
+    assert marked["message_sha256"]
+    assert rows["moving_gradient-h264_crf23"]["transform"]["parameters"]["crf"] == 23
+
+
+def test_scale_and_fps_attacks_carry_no_reference(tmp_path: Path) -> None:
+    manifest = COHORT.build_cohort(tmp_path / "cohort")
+
+    cases = {case.case_id: case for case in BENCHMARK.load_manifest(manifest)}
+    assert cases["moving_gradient-h264_crf23"].reference_path is not None
+    assert cases["moving_gradient-scale_075"].reference_path is None
+    assert cases["moving_gradient-fps_half"].reference_path is None