mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 13:30:55 +02:00
Victor Kuznetsov
464fd142ea
Extend the development-only benchmark family from images to audio and video, and close four research directions from the watermark-landscape map with local, revision-pinned evidence. Benchmark schema: media_type audio and video rows, audioseal and videoseal adapters with media-checked adapter binding, sample-domain audio fidelity (SNR with unbounded_identical), frame-domain video fidelity (mean PSNR, changed-frame fraction), and per-frame temporal evidence in videoseal detection records. The report accepts all three media types and carries SNR columns. Audio: shared pinned AudioSeal oracle, synthetic and system-voice speech cohorts with matched negatives, codec, noise, removed, and forged arms, and the provenance experiment proving visual-only video cleaning leaves the audio bitstream and watermark verdict invariant. Video: TorchScript-only VideoSeal oracle (the pip package is unusable on macOS arm64 and its loader resolves paths relative to cwd), video cohort with crf/scale/fps attacks and four-state arms, temporal evaluation measuring the crf decay curve, the frame-aggregation matrix (computed locally after finding the TorchScript build ignores the aggregation argument), and real provider clips as carriers. Forgery: double-embedding study showing the last writer wins completely under both oracles, scoped remover-trace probe, and the forged/ removed cohort arms encoding each scheme's correct expected verdict (presence rule for audioseal, message rule for videoseal). C2PA resolution: measured the five vendors' resolver contracts and the C2PA 2.2 18.10.5 privacy gradient; product decision recorded that this tool never resolves over the network. A registry guard pins the published resolution-API set. Gate: ruff and pyright src/ clean, 2114 passed / 3 skipped; maintain.sh stops only on the known unfixed lightning 2.6.5 CVEs, pre-existing.
801 lines
30 KiB
Python
801 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Local AudioSeal audio-provenance experiment (development-only).
|
|
|
|
The study answers one question the image benchmark cannot: does a video stay
|
|
provenance-positive through its untouched audio track after a visual-only
|
|
cleaning pass? AudioSeal (Meta, MIT license) is embedded locally into
|
|
synthetic carriers, muxed into videos next to a visible Sora-like mark, and the
|
|
production ``remove_video_visible`` path is run on the result. That path
|
|
transcodes the video stream and stream-copies audio, so the experiment measures
|
|
three separable things and never merges them:
|
|
|
|
- ``bitstream_identity``: whether the cleaned video's audio packets are
|
|
byte-identical to the source video's audio packets.
|
|
- ``cleaning_invariance``: whether the detector verdict on the decoded audio is
|
|
unchanged by the visual cleaning pass.
|
|
- ``audio_attack``: how the watermark responds to audio-path processing only
|
|
(codecs, resampling, noise, gain, lowpass) with no video involved.
|
|
|
|
Everything runs locally. No provider oracle or API is contacted, the package
|
|
runtime is untouched, and generated artifacts stay outside the repository under
|
|
``.local-eval/``. Weights are pinned to an exact Hugging Face revision and
|
|
verified against pinned SHA-256 digests before use.
|
|
|
|
uv run python scripts/audioseal_experiment.py \
|
|
--output-dir .local-eval/audioseal-experiment-v1
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import zlib
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from time import perf_counter_ns
|
|
from typing import TYPE_CHECKING
|
|
|
|
import numpy as np
|
|
from PIL import Image, ImageDraw
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
sys.path.insert(0, str(REPO / "scripts"))
|
|
|
|
import audioseal_oracle as oracle # noqa: E402
|
|
from audioseal_oracle import SAMPLE_RATE # noqa: E402
|
|
|
|
log = logging.getLogger("audioseal_experiment")
|
|
|
|
DURATION_S = 8.0
|
|
FRAME_COUNT = 24
|
|
FRAME_RATE = 12.0
|
|
FRAME_WIDTH = 840
|
|
FRAME_HEIGHT = 480
|
|
CARRIER_SEED = 20260907
|
|
MESSAGE_SEED = 7
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Detection:
|
|
"""AudioSeal detector readings on one decoded audio tensor."""
|
|
|
|
mean_detect_prob: float
|
|
frac_above_threshold: float
|
|
bit_accuracy: float | None
|
|
decoded_bits: list[int]
|
|
|
|
def as_dict(self) -> dict[str, object]:
|
|
return {
|
|
"mean_detect_prob": round(self.mean_detect_prob, 6),
|
|
"frac_above_threshold": round(self.frac_above_threshold, 6),
|
|
"bit_accuracy": None if self.bit_accuracy is None else round(self.bit_accuracy, 6),
|
|
"decoded_bits": self.decoded_bits,
|
|
}
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
# --- Pure carrier and WAV helpers (unit-tested without torch) --------------
|
|
|
|
|
|
def carrier_seed(name: str) -> int:
|
|
"""Stable per-carrier seed; Python's hash() is salted per process."""
|
|
return CARRIER_SEED + zlib.crc32(name.encode()) % 1000
|
|
|
|
|
|
def synth_carrier(name: str, seconds: float) -> np.ndarray:
|
|
"""Synthesize one deterministic carrier by registered name."""
|
|
factory = CARRIERS.get(name)
|
|
if factory is None:
|
|
raise KeyError(f"unknown carrier {name!r}; expected one of {sorted(CARRIERS)}")
|
|
master = np.random.default_rng(carrier_seed(name))
|
|
return np.asarray(factory(int(seconds * SAMPLE_RATE), master), dtype=np.float32)
|
|
|
|
|
|
def tone_stack_carrier(n: int, rng: np.random.Generator) -> np.ndarray:
|
|
t = np.arange(n, dtype=np.float64) / SAMPLE_RATE
|
|
am = 0.5 + 0.5 * np.sin(2 * np.pi * 1.1 * t)
|
|
carrier = (
|
|
0.5 * am * np.sin(2 * np.pi * 220.0 * t)
|
|
+ 0.3 * am * np.sin(2 * np.pi * 440.0 * t)
|
|
+ 0.2 * am * np.sin(2 * np.pi * 880.0 * t)
|
|
)
|
|
return carrier + rng.normal(0.0, 0.005, n)
|
|
|
|
|
|
def pinkish_carrier(n: int, rng: np.random.Generator) -> np.ndarray:
|
|
white = rng.normal(0.0, 1.0, n)
|
|
kernel = np.ones(9) / 9.0
|
|
smoothed = np.convolve(white, kernel, mode="same")
|
|
return smoothed / np.std(smoothed) * 0.1
|
|
|
|
|
|
def white_carrier(n: int, rng: np.random.Generator) -> np.ndarray:
|
|
return rng.normal(0.0, 0.1, n)
|
|
|
|
|
|
CARRIERS: dict[str, object] = {
|
|
"tone_stack": tone_stack_carrier,
|
|
"pinkish_noise": pinkish_carrier,
|
|
"white_noise": white_carrier,
|
|
}
|
|
|
|
|
|
def message_bits(seed: int = MESSAGE_SEED) -> list[int]:
|
|
rng = np.random.default_rng(seed)
|
|
return [int(b) for b in rng.integers(0, 2, oracle.N_BITS)]
|
|
|
|
|
|
def wav_pcm16_bytes(samples: np.ndarray, sample_rate: int) -> bytes:
|
|
"""Serialize mono float samples in [-1, 1) as a 16-bit PCM WAV file."""
|
|
clipped = np.clip(samples, -1.0, 1.0)
|
|
pcm = (clipped * 32767.0).astype("<i2")
|
|
payload = pcm.tobytes()
|
|
return b"".join(
|
|
(
|
|
b"RIFF",
|
|
struct.pack("<I", 36 + len(payload)),
|
|
b"WAVEfmt ",
|
|
struct.pack("<IHHIIHH", 16, 1, 1, sample_rate, sample_rate * 2, 2, 16),
|
|
b"data",
|
|
struct.pack("<I", len(payload)),
|
|
payload,
|
|
)
|
|
)
|
|
|
|
|
|
def snr_db(signal: np.ndarray, residual: np.ndarray) -> float:
|
|
power = float(np.sum(signal.astype(np.float64) ** 2))
|
|
noise = float(np.sum(residual.astype(np.float64) ** 2))
|
|
return 10.0 * np.log10(power / max(noise, 1e-12))
|
|
|
|
|
|
# --- ffmpeg helpers --------------------------------------------------------
|
|
|
|
|
|
def require_tools() -> tuple[str, str]:
|
|
ffmpeg = shutil.which("ffmpeg")
|
|
ffprobe = shutil.which("ffprobe")
|
|
if ffmpeg is None or ffprobe is None:
|
|
raise SystemExit("audioseal_experiment requires ffmpeg and ffprobe on PATH")
|
|
return ffmpeg, ffprobe
|
|
|
|
|
|
def run_ffmpeg(ffmpeg: str, args: Sequence[str], *, stdin: bytes | None = None) -> bytes:
|
|
result = subprocess.run( # noqa: S603
|
|
[ffmpeg, "-y", "-loglevel", "error", *args],
|
|
input=stdin,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"ffmpeg failed ({result.returncode}): {result.stderr.decode(errors='replace')[:400]}")
|
|
return result.stdout
|
|
|
|
|
|
def ffmpeg_version(ffmpeg: str) -> str:
|
|
result = subprocess.run([ffmpeg, "-version"], capture_output=True, check=False) # noqa: S603
|
|
return result.stdout.decode(errors="replace").splitlines()[0].strip()
|
|
|
|
|
|
def decode_audio_f32(ffmpeg: str, path: Path) -> np.ndarray:
|
|
raw = run_ffmpeg(
|
|
ffmpeg,
|
|
["-i", str(path), "-ac", "1", "-ar", str(SAMPLE_RATE), "-f", "f32le", "pipe:1"],
|
|
)
|
|
return np.frombuffer(raw, dtype="<f4").astype(np.float32)
|
|
|
|
|
|
def aac_packet_bytes(ffmpeg: str, path: Path) -> bytes:
|
|
return run_ffmpeg(ffmpeg, ["-i", str(path), "-map", "0:a:0", "-c:a", "copy", "-f", "adts", "pipe:1"])
|
|
|
|
|
|
def pcm_packet_bytes(ffmpeg: str, path: Path) -> bytes:
|
|
return run_ffmpeg(ffmpeg, ["-i", str(path), "-map", "0:a:0", "-c:a", "copy", "-f", "s16le", "pipe:1"])
|
|
|
|
|
|
# --- Synthetic video (independent Sora-like mark, moving background) -------
|
|
|
|
|
|
def moving_background(frame_index: int) -> np.ndarray:
|
|
x = np.arange(FRAME_WIDTH, dtype=np.float32)[None, :]
|
|
y = np.arange(FRAME_HEIGHT, dtype=np.float32)[:, None]
|
|
luma = 42 + 10 * np.sin((x + frame_index * 5) / 38) + 5 * np.cos((y - frame_index * 2) / 54)
|
|
frame = np.stack(
|
|
(
|
|
np.clip(luma - 5, 0, 255),
|
|
np.clip(luma + 1, 0, 255),
|
|
np.clip(luma + 7, 0, 255),
|
|
),
|
|
axis=2,
|
|
).astype(np.uint8)
|
|
moving_x = 30 + frame_index * 8
|
|
frame[moving_x : moving_x + 72, 70:132] = (80, 140, 210)
|
|
frame[220 + frame_index : 223 + frame_index, :] = (110, 70, 45)
|
|
return frame
|
|
|
|
|
|
def sora_like_mark() -> tuple[np.ndarray, int, int]:
|
|
"""Render a Sora-like mark without reusing the detector's template."""
|
|
mark = Image.new("L", (180, 64), 0)
|
|
draw = ImageDraw.Draw(mark)
|
|
draw.ellipse((1, 14, 32, 54), fill=255)
|
|
draw.ellipse((25, 8, 62, 58), fill=255)
|
|
draw.ellipse((15, 20, 28, 44), fill=0)
|
|
draw.ellipse((37, 18, 50, 43), fill=0)
|
|
from PIL import ImageFont
|
|
|
|
try:
|
|
font = ImageFont.load_default(size=49)
|
|
except TypeError:
|
|
font = ImageFont.load_default()
|
|
draw.text((68, 1), "Sora", font=font, fill=255, stroke_width=1)
|
|
resized = mark.resize((124, 44), Image.Resampling.BOX)
|
|
return np.asarray(resized, dtype=np.uint8), 620, 398
|
|
|
|
|
|
def stamped_frame(frame_index: int, mark: np.ndarray, mark_x: int, mark_y: int) -> np.ndarray:
|
|
frame = moving_background(frame_index)
|
|
height, width = mark.shape
|
|
crop = frame[mark_y : mark_y + height, mark_x : mark_x + width].astype(np.float32)
|
|
alpha = mark[:, :, None].astype(np.float32) / 255.0 * 0.78
|
|
frame[mark_y : mark_y + height, mark_x : mark_x + width] = np.clip(crop * (1 - alpha) + 255 * alpha, 0, 255).astype(
|
|
np.uint8
|
|
)
|
|
return frame
|
|
|
|
|
|
def encode_video(
|
|
ffmpeg: str,
|
|
path: Path,
|
|
frames: list[np.ndarray],
|
|
audio: Path,
|
|
*,
|
|
audio_codec: str,
|
|
audio_rate: int,
|
|
) -> None:
|
|
"""Encode raw BGR frames plus a WAV audio input into one container."""
|
|
height, width = frames[0].shape[:2]
|
|
duration = len(frames) / FRAME_RATE
|
|
args = [
|
|
"-f",
|
|
"rawvideo",
|
|
"-pix_fmt",
|
|
"bgr24",
|
|
"-s:v",
|
|
f"{width}x{height}",
|
|
"-r",
|
|
f"{FRAME_RATE:.12g}",
|
|
"-i",
|
|
"pipe:0",
|
|
"-i",
|
|
str(audio),
|
|
"-map",
|
|
"0:v:0",
|
|
"-map",
|
|
"1:a:0",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"ultrafast",
|
|
"-crf",
|
|
"8",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
]
|
|
if audio_codec == "pcm_s16le":
|
|
args += ["-c:a", "pcm_s16le"]
|
|
else:
|
|
args += [
|
|
"-c:a",
|
|
audio_codec,
|
|
"-b:a",
|
|
"128k",
|
|
"-ar",
|
|
str(audio_rate),
|
|
]
|
|
args += ["-t", f"{duration:.12g}", str(path)]
|
|
payload = b"".join(frame.astype(np.uint8).tobytes() for frame in frames)
|
|
run_ffmpeg(ffmpeg, args, stdin=payload)
|
|
|
|
|
|
# --- AudioSeal oracle (shared, revision-pinned) -----------------------------
|
|
|
|
|
|
def load_pinned_models() -> tuple[object, object, dict[str, str]]:
|
|
"""Load the shared pinned oracle models and record their provenance."""
|
|
import importlib.metadata as md
|
|
import inspect
|
|
|
|
import torch
|
|
from audioseal_oracle import load_pinned_models as shared_load
|
|
|
|
generator, detector = shared_load()
|
|
versions = {
|
|
"audioseal": md.version("audioseal"),
|
|
"torch": torch.__version__,
|
|
"huggingface_revision": oracle.HF_REVISION,
|
|
"generator_sha256": oracle.GENERATOR_SHA256,
|
|
"detector_sha256": oracle.DETECTOR_SHA256,
|
|
"oracle_source_sha256": sha256_file(Path(inspect.getfile(shared_load))),
|
|
}
|
|
return generator, detector, versions
|
|
|
|
|
|
class DetectorClock:
|
|
"""Records the first (cold) detector call separately from warm calls."""
|
|
|
|
def __init__(self) -> None:
|
|
self.cold_ms: float | None = None
|
|
self.warm_ms: list[float] = []
|
|
|
|
def measure(self, call: object, wav: object) -> object:
|
|
started = perf_counter_ns()
|
|
result = call(wav)
|
|
elapsed = (perf_counter_ns() - started) / 1e6
|
|
if self.cold_ms is None:
|
|
self.cold_ms = elapsed
|
|
else:
|
|
self.warm_ms.append(elapsed)
|
|
return result
|
|
|
|
|
|
def detect_audio(detector: object, clock: DetectorClock, samples: np.ndarray) -> Detection:
|
|
"""Read one detection under the clock and score it against the message."""
|
|
reading = clock.measure(lambda wav: oracle.read(detector, wav), samples)
|
|
expected = message_bits()
|
|
bits = list(reading.decoded_bits)
|
|
accuracy = sum(int(b == e) for b, e in zip(bits, expected, strict=True)) / oracle.N_BITS
|
|
return Detection(
|
|
mean_detect_prob=reading.mean_detect_prob,
|
|
frac_above_threshold=reading.frac_above_threshold,
|
|
bit_accuracy=accuracy,
|
|
decoded_bits=bits,
|
|
)
|
|
|
|
|
|
def embed_audio(generator: object, samples: np.ndarray, message: list[int]) -> np.ndarray:
|
|
"""Embed the fixed message through the shared pinned oracle."""
|
|
return np.asarray(oracle.embed(generator, samples, message), dtype=np.float32)
|
|
|
|
|
|
# --- Attack transforms -----------------------------------------------------
|
|
|
|
|
|
def numpy_attacks(samples: np.ndarray) -> dict[str, np.ndarray]:
|
|
rng = np.random.default_rng(CARRIER_SEED + 1)
|
|
out: dict[str, np.ndarray] = {}
|
|
power = float(np.sum(samples.astype(np.float64) ** 2))
|
|
noise = rng.normal(0.0, 1.0, samples.shape[0]).astype(np.float32)
|
|
noise *= np.sqrt(power / max(np.sum(noise.astype(np.float64) ** 2), 1e-12) / 10.0)
|
|
out["noise_snr10"] = samples + noise
|
|
out["gain_0.5"] = samples * 0.5
|
|
return out
|
|
|
|
|
|
def partial_noise_attack(samples: np.ndarray, start_s: float, end_s: float) -> np.ndarray:
|
|
"""Mix 10 dB-SNR gaussian noise into one time window, leaving the rest bit-identical."""
|
|
rng = np.random.default_rng(CARRIER_SEED + 2)
|
|
out = samples.copy()
|
|
start = int(start_s * SAMPLE_RATE)
|
|
end = int(end_s * SAMPLE_RATE)
|
|
window = out[start:end]
|
|
power = float(np.sum(window.astype(np.float64) ** 2))
|
|
noise = rng.normal(0.0, 1.0, window.shape[0]).astype(np.float32)
|
|
noise *= np.sqrt(power / max(np.sum(noise.astype(np.float64) ** 2), 1e-12) / 10.0)
|
|
out[start:end] = window + noise
|
|
return out
|
|
|
|
|
|
# --- Local speech carriers (system TTS, no network) --------------------------
|
|
|
|
SPEECH_TEXT = (
|
|
"Provenance through audio remains measurable after visual cleaning. "
|
|
"The second sentence extends this carrier past three seconds."
|
|
)
|
|
SPEECH_VOICES: tuple[str, ...] = ("Samantha", "Daniel", "Milena")
|
|
|
|
|
|
def assert_distinct_voices(artifacts: dict[str, bytes]) -> None:
|
|
"""Reject voice aliasing: two names rendering byte-identical audio.
|
|
|
|
macOS 26.6.2 aliases ``Alex`` to ``Samantha``; a carrier-diversity claim
|
|
built on aliased voices is one carrier counted twice. Verified by content
|
|
hash, never by voice name.
|
|
"""
|
|
seen: dict[str, str] = {}
|
|
for voice, payload in artifacts.items():
|
|
digest = sha256_bytes(payload)
|
|
duplicate = seen.get(digest)
|
|
if duplicate is not None:
|
|
raise SystemExit(
|
|
f"system voices {duplicate!r} and {voice!r} render byte-identical audio; pick genuinely distinct voices"
|
|
)
|
|
seen[digest] = voice
|
|
|
|
|
|
def speech_provenance() -> dict[str, str]:
|
|
"""Record everything that pins system-synthesized speech bytes."""
|
|
import platform
|
|
|
|
product = ""
|
|
sw_vers = shutil.which("sw_vers")
|
|
if sw_vers is not None:
|
|
result = subprocess.run( # noqa: S603 - resolved sw_vers with fixed arguments
|
|
[sw_vers, "-productVersion"], capture_output=True, text=True, check=False
|
|
)
|
|
product = result.stdout.strip()
|
|
return {
|
|
"tts": "say",
|
|
"macos": product or platform.mac_ver()[0],
|
|
"text_sha256": sha256_bytes(SPEECH_TEXT.encode()),
|
|
}
|
|
|
|
|
|
def synthesize_speech(voice: str, workdir: Path) -> Path:
|
|
"""Render one deterministic utterance through the local system voice."""
|
|
say = shutil.which("say")
|
|
if say is None:
|
|
raise SystemExit("speech carriers require the macOS say tool")
|
|
target = workdir / f"speech-{voice}.aiff"
|
|
result = subprocess.run( # noqa: S603 - resolved say with fixed arguments
|
|
[say, "-o", str(target), "-v", voice, SPEECH_TEXT],
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0 or not target.is_file():
|
|
raise SystemExit(f"system voice {voice!r} is unavailable: {result.stderr.decode(errors='replace')[:200]}")
|
|
return target
|
|
|
|
|
|
def speech_samples(voice: str, workdir: Path, ffmpeg: str) -> np.ndarray:
|
|
"""Decode one system-voice utterance to mono 16 kHz float32."""
|
|
return decode_audio_f32(ffmpeg, synthesize_speech(voice, workdir))
|
|
|
|
|
|
def ffmpeg_attack(
|
|
ffmpeg: str, name: str, suffix: str, args: Sequence[str], source: Path, workdir: Path, *, stem: str = "attack"
|
|
) -> Path:
|
|
target = workdir / f"{stem}-{name}.{suffix}"
|
|
run_ffmpeg(ffmpeg, ["-i", str(source), *args, str(target)])
|
|
return target
|
|
|
|
|
|
FFMPEG_ATTACKS: dict[str, tuple[str, list[str]]] = {
|
|
# AAC attacks use the MP4/M4A container so the decode honors the encoder's
|
|
# priming edit list. A raw ADTS decode keeps the 1024-sample encoder-delay
|
|
# head in the stream, which measures the no-gapless decode path rather
|
|
# than the codec itself; that path is kept as its own explicit arm below.
|
|
"aac_128k": ("m4a", ["-c:a", "aac", "-b:a", "128k"]),
|
|
"aac_128k_48k": ("m4a", ["-c:a", "aac", "-b:a", "128k", "-ar", "48000"]),
|
|
"aac_128k_adts_raw": ("adts", ["-c:a", "aac", "-b:a", "128k"]),
|
|
"mp3_128k": ("mp3", ["-c:a", "libmp3lame", "-b:a", "128k"]),
|
|
"mp3_64k": ("mp3", ["-c:a", "libmp3lame", "-b:a", "64k"]),
|
|
"resample_44k_wav": ("wav", ["-c:a", "pcm_s16le", "-ar", "44100"]),
|
|
"lowpass_8k_wav": ("wav", ["-c:a", "pcm_s16le", "-af", "lowpass=f=8000"]),
|
|
}
|
|
|
|
|
|
# --- Experiment ------------------------------------------------------------
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--skip-video", action="store_true", help="audio stages only")
|
|
parser.add_argument(
|
|
"--skip-speech", action="store_true", help="skip system-voice speech carriers and interval readings"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
out_dir: Path = args.output_dir.resolve()
|
|
if out_dir.exists():
|
|
raise SystemExit(f"refusing to overwrite existing output directory {out_dir}")
|
|
out_dir.mkdir(parents=True)
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
|
ffmpeg, _ffprobe = require_tools()
|
|
|
|
generator, detector, versions = load_pinned_models()
|
|
clock = DetectorClock()
|
|
rows: list[dict[str, object]] = []
|
|
|
|
def record(stage: str, **fields: object) -> None:
|
|
row = {"stage": stage, **fields}
|
|
rows.append(row)
|
|
log.info("%s", json.dumps(row, sort_keys=True, default=str)[:400])
|
|
|
|
def detect_and_record(stage: str, label: str, samples: np.ndarray, path: Path | None, **extra: object) -> Detection:
|
|
det = detect_audio(detector, clock, samples)
|
|
record(
|
|
stage,
|
|
label=label,
|
|
detection=det.as_dict(),
|
|
sha256=None if path is None else sha256_file(path),
|
|
**extra,
|
|
)
|
|
return det
|
|
|
|
message = message_bits()
|
|
record("pins", message=message, **versions, ffmpeg=ffmpeg_version(ffmpeg))
|
|
|
|
# Stage 1: carriers, clean vs marked.
|
|
marked_wavs: dict[str, Path] = {}
|
|
for name in CARRIERS:
|
|
clean = synth_carrier(name, DURATION_S)
|
|
clean_path = out_dir / f"{name}-clean.wav"
|
|
clean_path.write_bytes(wav_pcm16_bytes(clean, SAMPLE_RATE))
|
|
detect_and_record("carrier_clean", f"{name}/clean", clean, clean_path)
|
|
|
|
marked = embed_audio(generator, clean, message)
|
|
marked_path = out_dir / f"{name}-marked.wav"
|
|
marked_path.write_bytes(wav_pcm16_bytes(marked, SAMPLE_RATE))
|
|
detect_and_record(
|
|
"carrier_marked",
|
|
f"{name}/marked",
|
|
marked,
|
|
marked_path,
|
|
watermark_snr_db=round(snr_db(clean, marked - clean), 3),
|
|
)
|
|
marked_wavs[name] = marked_path
|
|
|
|
# Stage 2: audio attacks on the tone_stack marked wav only; the codec arms
|
|
# double as the audio tracks used for the video arms below.
|
|
primary = marked_wavs["tone_stack"]
|
|
attack_paths: dict[str, Path] = {}
|
|
for name, (suffix, ffargs) in FFMPEG_ATTACKS.items():
|
|
target = ffmpeg_attack(ffmpeg, name, suffix, ffargs, primary, out_dir)
|
|
attack_paths[name] = target
|
|
detect_and_record("audio_attack", name, decode_audio_f32(ffmpeg, target), target)
|
|
for name, samples in numpy_attacks(decode_audio_f32(ffmpeg, primary)).items():
|
|
target = out_dir / f"attack-{name}.wav"
|
|
target.write_bytes(wav_pcm16_bytes(samples, SAMPLE_RATE))
|
|
attack_paths[name] = target
|
|
detect_and_record("audio_attack", name, samples, target)
|
|
|
|
# Stage 2b: system-voice speech carriers with per-second interval readings.
|
|
# Speech is the detector's training domain, and the interval report shows
|
|
# WHERE in time the signal lives - including its localized death under an
|
|
# attack confined to seconds two through four.
|
|
if not args.skip_speech:
|
|
from audioseal_oracle import read_intervals
|
|
|
|
provenance = speech_provenance()
|
|
raw_speech: dict[str, Path] = {}
|
|
for voice in SPEECH_VOICES:
|
|
raw_speech[voice] = synthesize_speech(voice, out_dir)
|
|
assert_distinct_voices({voice: path.read_bytes() for voice, path in raw_speech.items()})
|
|
for voice in SPEECH_VOICES:
|
|
label = f"speech_{voice}"
|
|
clean_speech = decode_audio_f32(ffmpeg, raw_speech[voice])
|
|
clean_path = out_dir / f"{label}-clean.wav"
|
|
clean_path.write_bytes(wav_pcm16_bytes(clean_speech, SAMPLE_RATE))
|
|
detect_and_record(
|
|
"speech_carrier",
|
|
f"{label}/clean",
|
|
clean_speech,
|
|
clean_path,
|
|
tts={**provenance, "voice": voice},
|
|
duration_s=round(clean_speech.shape[0] / SAMPLE_RATE, 3),
|
|
)
|
|
|
|
marked_speech = embed_audio(generator, clean_speech, message)
|
|
marked_path = out_dir / f"{label}-marked.wav"
|
|
marked_path.write_bytes(wav_pcm16_bytes(marked_speech, SAMPLE_RATE))
|
|
detect_and_record(
|
|
"speech_carrier",
|
|
f"{label}/marked",
|
|
marked_speech,
|
|
marked_path,
|
|
tts={**provenance, "voice": voice},
|
|
watermark_snr_db=round(snr_db(clean_speech, marked_speech - clean_speech), 3),
|
|
)
|
|
partial = partial_noise_attack(marked_speech, 2.0, 4.0)
|
|
partial_path = out_dir / f"{label}-partial_noise_2s_4s.wav"
|
|
partial_path.write_bytes(wav_pcm16_bytes(partial, SAMPLE_RATE))
|
|
for tag, samples, artifact_path in (
|
|
("marked", marked_speech, marked_path),
|
|
("partial_noise_2s_4s", partial, partial_path),
|
|
):
|
|
intervals = read_intervals(detector, samples)
|
|
record(
|
|
"speech_intervals",
|
|
label=f"{label}/{tag}",
|
|
per_second=[
|
|
{
|
|
"start_s": index,
|
|
"mean_detect_prob": round(reading.mean_detect_prob, 6),
|
|
"frac_above_threshold": round(reading.frac_above_threshold, 6),
|
|
"detected": reading.detected,
|
|
}
|
|
for index, reading in enumerate(intervals)
|
|
],
|
|
sha256=sha256_file(artifact_path),
|
|
)
|
|
|
|
# Stage 3: video arms. Each arm muxes the SAME marked wav next to a
|
|
# stamped Sora-like mark, differing only in how audio is carried.
|
|
if not args.skip_video:
|
|
from remove_ai_watermarks.video import remove_video_visible
|
|
|
|
mark, mark_x, mark_y = sora_like_mark()
|
|
frames = [stamped_frame(i, mark, mark_x, mark_y) for i in range(FRAME_COUNT)]
|
|
arms: dict[str, tuple[str, int | None]] = {
|
|
"pcm_mov": ("pcm_s16le", None),
|
|
"aac_16k": ("aac", SAMPLE_RATE),
|
|
"aac_48k": ("aac", 48_000),
|
|
}
|
|
for arm, (codec, rate) in arms.items():
|
|
suffix = "mov" if codec == "pcm_s16le" else "mp4"
|
|
source = out_dir / f"video-{arm}-source.{suffix}"
|
|
encode_video(ffmpeg, source, frames, primary, audio_codec=codec, audio_rate=rate or SAMPLE_RATE)
|
|
packetize = pcm_packet_bytes if codec == "pcm_s16le" else aac_packet_bytes
|
|
source_packets = packetize(ffmpeg, source)
|
|
source_digest = sha256_bytes(source_packets)
|
|
|
|
cleaned = out_dir / f"video-{arm}-cleaned.{suffix}"
|
|
result = remove_video_visible(source, cleaned, mark="sora", backend="cv2")
|
|
if result.output is None:
|
|
raise SystemExit(f"visual cleaning wrote no output for arm {arm}")
|
|
cleaned_packets = packetize(ffmpeg, cleaned)
|
|
cleaned_digest = sha256_bytes(cleaned_packets)
|
|
identical = source_digest == cleaned_digest
|
|
|
|
det_source = detect_and_record(
|
|
"video_audio",
|
|
f"{arm}/source",
|
|
decode_audio_f32(ffmpeg, source),
|
|
source,
|
|
audio_codec=codec,
|
|
)
|
|
det_cleaned = detect_and_record(
|
|
"video_audio_cleaned",
|
|
f"{arm}/cleaned",
|
|
decode_audio_f32(ffmpeg, cleaned),
|
|
cleaned,
|
|
audio_codec=codec,
|
|
bitstream_identical=identical,
|
|
source_packet_sha256=source_digest,
|
|
cleaned_packet_sha256=cleaned_digest,
|
|
visual_removed_frames=result.removed_frames,
|
|
)
|
|
invariance = (
|
|
det_source.mean_detect_prob == det_cleaned.mean_detect_prob
|
|
and det_source.frac_above_threshold == det_cleaned.frac_above_threshold
|
|
and det_source.decoded_bits == det_cleaned.decoded_bits
|
|
)
|
|
record(
|
|
"cleaning_invariance",
|
|
arm=arm,
|
|
bitstream_identical=identical,
|
|
verdict_invariant=invariance,
|
|
)
|
|
|
|
# Timing summary and reports.
|
|
warm = sorted(clock.warm_ms)
|
|
timing = {
|
|
"detector_cold_ms": None if clock.cold_ms is None else round(clock.cold_ms, 3),
|
|
"detector_warm_n": len(warm),
|
|
"detector_warm_median_ms": None if not warm else round(warm[len(warm) // 2], 3),
|
|
"detector_warm_max_ms": None if not warm else round(warm[-1], 3),
|
|
}
|
|
cases_path = out_dir / "cases.jsonl"
|
|
cases_path.write_text("".join(json.dumps(r, default=str) + "\n" for r in rows))
|
|
summary_path = out_dir / "summary.md"
|
|
summary_path.write_text(render_summary(rows, timing, versions, ffmpeg, out_dir))
|
|
print(f"wrote {cases_path}")
|
|
print(f"wrote {summary_path}")
|
|
return 0
|
|
|
|
|
|
def render_summary(
|
|
rows: list[dict[str, object]],
|
|
timing: dict[str, object],
|
|
versions: dict[str, str],
|
|
ffmpeg: str,
|
|
out_dir: Path,
|
|
) -> str:
|
|
def table(stage: str, columns: list[str]) -> list[str]:
|
|
selected = [r for r in rows if r["stage"] == stage]
|
|
lines = [
|
|
f"### {stage}",
|
|
"",
|
|
"| " + " | ".join(columns) + " |",
|
|
"| " + " | ".join(["---"] * len(columns)) + " |",
|
|
]
|
|
for r in selected:
|
|
det = r.get("detection", {}) if isinstance(r.get("detection"), dict) else {}
|
|
cells = []
|
|
for col in columns:
|
|
if col == "mean_detect":
|
|
cells.append(f"{det.get('mean_detect_prob', '')}")
|
|
elif col == "frac>0.5":
|
|
cells.append(f"{det.get('frac_above_threshold', '')}")
|
|
elif col == "bit_acc":
|
|
cells.append(f"{det.get('bit_accuracy', '')}")
|
|
elif col == "label":
|
|
cells.append(str(r.get("label", r.get("arm", ""))))
|
|
else:
|
|
cells.append(str(r.get(col, "")))
|
|
lines.append("| " + " | ".join(cells) + " |")
|
|
lines.append("")
|
|
return lines
|
|
|
|
def interval_table() -> list[str]:
|
|
selected = [r for r in rows if r["stage"] == "speech_intervals"]
|
|
lines = [
|
|
"### speech_intervals",
|
|
"",
|
|
"| label | second | mean_detect | frac>0.5 | detected |",
|
|
"| --- | --- | --- | --- | --- |",
|
|
]
|
|
for r in selected:
|
|
for window in r.get("per_second", []):
|
|
if not isinstance(window, dict):
|
|
continue
|
|
lines.append(
|
|
"| "
|
|
+ " | ".join(
|
|
(
|
|
str(r.get("label", "")),
|
|
str(window.get("start_s", "")),
|
|
str(window.get("mean_detect_prob", "")),
|
|
str(window.get("frac_above_threshold", "")),
|
|
str(window.get("detected", "")),
|
|
)
|
|
)
|
|
+ " |"
|
|
)
|
|
lines.append("")
|
|
return lines
|
|
|
|
lines = [
|
|
"# AudioSeal audio-provenance experiment",
|
|
"",
|
|
f"- output: `{out_dir.name}`",
|
|
f"- revision: `{versions['huggingface_revision']}`",
|
|
f"- audioseal {versions['audioseal']}, torch {versions['torch']}",
|
|
f"- ffmpeg: `{ffmpeg_version(ffmpeg)}`",
|
|
f"- detector timing: {json.dumps(timing)}",
|
|
"",
|
|
*table("carrier_clean", ["label", "mean_detect", "frac>0.5"]),
|
|
*table("speech_carrier", ["label", "mean_detect", "frac>0.5", "bit_acc", "watermark_snr_db"]),
|
|
*interval_table(),
|
|
*table("carrier_marked", ["label", "mean_detect", "frac>0.5", "bit_acc", "watermark_snr_db"]),
|
|
*table("audio_attack", ["label", "mean_detect", "frac>0.5", "bit_acc"]),
|
|
*table("video_audio", ["label", "mean_detect", "frac>0.5", "bit_acc"]),
|
|
*table("video_audio_cleaned", ["label", "mean_detect", "frac>0.5", "bit_acc", "bitstream_identical"]),
|
|
*table("cleaning_invariance", ["arm", "bitstream_identical", "verdict_invariant"]),
|
|
"",
|
|
"`bitstream_identical` compares audio packet digests of the source and",
|
|
"visually cleaned videos. `verdict_invariant` additionally requires the",
|
|
"detector readings to match. Rows are measurements on synthetic",
|
|
"carriers, not detector-quality claims.",
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|