mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 21:40:47 +02:00
Victor Kuznetsov
464fd142ea
Extend the development-only benchmark family from images to audio and video, and close four research directions from the watermark-landscape map with local, revision-pinned evidence. Benchmark schema: media_type audio and video rows, audioseal and videoseal adapters with media-checked adapter binding, sample-domain audio fidelity (SNR with unbounded_identical), frame-domain video fidelity (mean PSNR, changed-frame fraction), and per-frame temporal evidence in videoseal detection records. The report accepts all three media types and carries SNR columns. Audio: shared pinned AudioSeal oracle, synthetic and system-voice speech cohorts with matched negatives, codec, noise, removed, and forged arms, and the provenance experiment proving visual-only video cleaning leaves the audio bitstream and watermark verdict invariant. Video: TorchScript-only VideoSeal oracle (the pip package is unusable on macOS arm64 and its loader resolves paths relative to cwd), video cohort with crf/scale/fps attacks and four-state arms, temporal evaluation measuring the crf decay curve, the frame-aggregation matrix (computed locally after finding the TorchScript build ignores the aggregation argument), and real provider clips as carriers. Forgery: double-embedding study showing the last writer wins completely under both oracles, scoped remover-trace probe, and the forged/ removed cohort arms encoding each scheme's correct expected verdict (presence rule for audioseal, message rule for videoseal). C2PA resolution: measured the five vendors' resolver contracts and the C2PA 2.2 18.10.5 privacy gradient; product decision recorded that this tool never resolves over the network. A registry guard pins the published resolution-API set. Gate: ruff and pyright src/ clean, 2114 passed / 3 skipped; maintain.sh stops only on the known unfixed lightning 2.6.5 CVEs, pre-existing.
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
"""Tests for the pinned VideoSeal TorchScript oracle's pure helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
import videoseal_oracle # noqa: E402
|
|
|
|
|
|
class TestMessage:
|
|
def test_message_is_deterministic_and_256_bits(self) -> None:
|
|
first = videoseal_oracle.message_bits()
|
|
second = videoseal_oracle.message_bits()
|
|
|
|
assert first == second
|
|
assert len(first) == 256
|
|
assert set(first) <= {0, 1}
|
|
|
|
def test_seed_changes_the_message(self) -> None:
|
|
assert videoseal_oracle.message_bits(seed=8) != videoseal_oracle.message_bits(seed=7)
|
|
|
|
|
|
class TestLabel:
|
|
def test_label_is_hex_of_the_decoded_bits(self) -> None:
|
|
bits = tuple([1] * 8 + [0] * 8)
|
|
reading = videoseal_oracle.VideoSealReading(
|
|
bit_accuracy=1.0,
|
|
decoded_bits=bits,
|
|
per_frame_bit_accuracy=(1.0,),
|
|
)
|
|
|
|
assert reading.label == "ff00"
|
|
|
|
def test_detection_rule_is_the_documented_threshold(self) -> None:
|
|
below = videoseal_oracle.VideoSealReading(0.89, (0,), ())
|
|
exact = videoseal_oracle.VideoSealReading(0.9, (0,), ())
|
|
|
|
assert below.detected is False
|
|
assert exact.detected is True
|
|
|
|
|
|
class TestPins:
|
|
def test_checkpoint_pin_is_a_sha256_digest(self) -> None:
|
|
digest = videoseal_oracle.JIT_SHA256
|
|
|
|
assert len(digest) == 64
|
|
assert all(char in "0123456789abcdef" for char in digest)
|
|
|
|
def test_source_commit_is_pinned(self) -> None:
|
|
assert len(videoseal_oracle.SOURCE_COMMIT) == 40
|
|
|
|
|
|
class TestCacheOverride:
|
|
def test_cache_dir_env_override_is_honored(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setenv("VIDEOSEAL_CACHE_DIR", str(tmp_path / "cache"))
|
|
|
|
import importlib
|
|
|
|
importlib.reload(videoseal_oracle)
|
|
|
|
try:
|
|
assert tmp_path / "cache" == videoseal_oracle.CACHE_DIR
|
|
finally:
|
|
importlib.reload(videoseal_oracle)
|
|
|
|
|
|
class TestAggregationFormulas:
|
|
def test_aggregations_match_the_upstream_formulas(self) -> None:
|
|
torch = pytest.importorskip("torch")
|
|
# Two frames, four bits; hand-computed from the upstream extract_message
|
|
# formulas so a silent mutation of the port fails here.
|
|
bit_preds = torch.tensor([[2.0, -1.0, 0.5, -3.0], [0.1, 0.2, -0.4, 0.3]])
|
|
|
|
avg = videoseal_oracle._aggregate_bit_preds(bit_preds, "avg")
|
|
squared = videoseal_oracle._aggregate_bit_preds(bit_preds, "squared_avg")
|
|
l1 = videoseal_oracle._aggregate_bit_preds(bit_preds, "l1norm_avg")
|
|
l2 = videoseal_oracle._aggregate_bit_preds(bit_preds, "l2norm_avg")
|
|
|
|
torch.testing.assert_close(avg, bit_preds.mean(dim=0))
|
|
torch.testing.assert_close(squared, (bit_preds * bit_preds.abs()).mean(dim=0))
|
|
w1 = torch.norm(bit_preds, p=1, dim=1).unsqueeze(1)
|
|
torch.testing.assert_close(l1, (bit_preds * w1).mean(dim=0))
|
|
w2 = torch.norm(bit_preds, p=2, dim=1).unsqueeze(1)
|
|
torch.testing.assert_close(l2, (bit_preds * w2).mean(dim=0))
|
|
|
|
def test_unknown_aggregation_is_rejected(self) -> None:
|
|
torch = pytest.importorskip("torch")
|
|
|
|
with pytest.raises(ValueError, match="unknown aggregation"):
|
|
videoseal_oracle._aggregate_bit_preds(torch.zeros(2, 4), "median")
|