mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-18 15:31:06 +02:00
Add Perth and PixelSeal benchmark oracles
This commit is contained in:
12 files changed
+1141
-24
No files matched your search
@@ -44,7 +44,7 @@ The published Agent Skill is [`skills/remove-ai-watermarks/`](skills/remove-ai-w
|
||||
|
||||
[`docs/module-internals.md`](docs/module-internals.md) is the canonical per-module map, including design decisions, thresholds, calibration history, incident records, and regression guards. Read the relevant section before changing a subsystem.
|
||||
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, Perth, PixelSeal, VideoSeal) cover image, audio, and video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
|
||||
## Data safety
|
||||
|
||||
|
||||
+112
-8
@@ -2,14 +2,14 @@
|
||||
|
||||
`scripts/watermark_benchmark.py` is a development-only runner for reproducible
|
||||
image-, audio-, and video-watermark experiments. It calls the existing local
|
||||
DWT-DCT and TrustMark detectors plus the revision-pinned AudioSeal and
|
||||
VideoSeal oracles through thin adapters. It does not add a runtime command,
|
||||
download a corpus, or contact a provenance oracle or provider API. The
|
||||
DWT-DCT and TrustMark detectors plus the revision-pinned AudioSeal, Perth,
|
||||
PixelSeal, and VideoSeal oracles through thin adapters. It does not add a
|
||||
runtime command, download a corpus, or contact a provenance oracle or provider API. The
|
||||
optional TrustMark package can fetch its official Adobe model weights when its
|
||||
local package cache is incomplete, the audioseal adapter fetches its pinned
|
||||
checkpoints the same way, and the videoseal adapter downloads one pinned
|
||||
TorchScript file into a user cache; prepare those dependencies before an
|
||||
offline run.
|
||||
checkpoints the same way, and the PixelSeal and VideoSeal adapters download
|
||||
pinned weights into a user cache; prepare those dependencies before an offline
|
||||
run. Perth's checkpoint is bundled in its pinned source revision.
|
||||
|
||||
The kernel answers three different questions and never merges their answers:
|
||||
|
||||
@@ -41,9 +41,9 @@ The fixed fields are:
|
||||
| `case_id` | Unique case identity. |
|
||||
| `pair_id` | Groups related clean, marked, attacked, and removed cases. |
|
||||
| `media_type` | `image`, `audio`, or `video` in schema v1. |
|
||||
| `adapter` | `dwt-dct` or `trustmark` for `image`; `audioseal` for `audio`; `videoseal` for `video`. The loader rejects a known adapter named against the wrong media type. |
|
||||
| `adapter` | `dwt-dct`, `trustmark`, or `pixelseal` for `image`; `audioseal` or `perth` for `audio`; `videoseal` for `video`. The loader rejects a known adapter named against the wrong media type. |
|
||||
| `arm` | `positive`, `matched_negative`, `wrong_key`, or `hard_negative`. |
|
||||
| `state` | `clean`, `marked`, `attacked`, `removed`, or `forged`. `forged` names an artifact carrying a watermark with a message different from the adapter oracle's fixed one. Expected detection depends on the adapter: VideoSeal checks the fixed message, while AudioSeal's presence rule can still report `detected` for a foreign message. The study records message accuracy separately. |
|
||||
| `state` | `clean`, `marked`, `attacked`, `removed`, or `forged`. `forged` names an artifact carrying a watermark with a message different from the adapter oracle's fixed one. Expected detection depends on the adapter: PixelSeal and VideoSeal check the fixed message, while AudioSeal's presence rule can still report `detected` for a foreign message. Perth has an implicit fixed mark and no message payload, so it has no meaningful forged arm. The study records message accuracy separately where the scheme exposes a message. |
|
||||
| `path`, `sha256` | Artifact path and pinned content digest. |
|
||||
| `reference_path`, `reference_sha256` | Both strings or both `null`; the fidelity reference. |
|
||||
| `source_revision` | Corpus, generator, or acquisition revision. |
|
||||
@@ -74,6 +74,10 @@ dirty, and SHA-256 digests of the benchmark kernel and adapter source. Detector
|
||||
dependency absence is `unavailable`; an undecodable artifact or adapter
|
||||
exception is `error`. Neither is counted as `not_detected`.
|
||||
|
||||
Message-matched adapters record `detection.message_bit_accuracy`. Score-based
|
||||
presence adapters record `detection.score`. A missing field means the adapter
|
||||
does not expose that quantity; it is not a numeric zero.
|
||||
|
||||
`detection.adapter_elapsed_ms` measures the wall time of the named detector
|
||||
adapter call only. Image decoding and fidelity calculation are outside that
|
||||
interval. The value is `null` when the adapter was unavailable or the artifact
|
||||
@@ -111,6 +115,21 @@ as a binary string, so a case whose payload differs from the embedded one is
|
||||
still `detected` with a different label, never `not_detected`. A `wrong_key`
|
||||
arm for this adapter is therefore a different-message row, not a clean row.
|
||||
|
||||
The `perth` adapter reads `scripts/perth_oracle.py`, which pins Resemble AI's
|
||||
[Perth source](https://github.com/resemble-ai/Perth/tree/ff1c8ac55a976971245cdd53c18d6131ca00d993)
|
||||
at commit `ff1c8ac` and verifies the bundled
|
||||
`perth_net_250000.pth.tar` checkpoint as
|
||||
`sha256:a15bce457ebc53ce5e6c9c3f11df78cf7ee2bf9cdab0a798902135b4c4027670`.
|
||||
The upstream package name is `resemble-perth`, the pinned source declares
|
||||
version 1.1.0, and its [license is MIT](https://github.com/resemble-ai/Perth/blob/ff1c8ac55a976971245cdd53c18d6131ca00d993/LICENSE).
|
||||
That permits a public, development-only oracle. The published 1.0.1 wheel
|
||||
pins an obsolete Torch stack, so the benchmark installs the exact source
|
||||
revision without adding it to project or runtime dependencies. Perth v1 is an
|
||||
implicit presence mark: the detector exposes one confidence and upstream's
|
||||
strictly-above-0.5 decision rule, but no user message. The kernel therefore
|
||||
records `score`, uses `label: perth-v1` only above threshold, and leaves
|
||||
message accuracy not applicable.
|
||||
|
||||
Audio fidelity compares the decoded sample arrays against the explicit
|
||||
reference: identical decoded samples produce `snr_db: null` with
|
||||
`snr_status: unbounded_identical`, never a misleading numeric zero; a silent
|
||||
@@ -118,6 +137,91 @@ reference reports `zero_reference_power`; a length mismatch stays an explicit
|
||||
`incomparable` state. SNR here is reference signal power over error power, the
|
||||
sample-domain analogue of the image PSNR contract.
|
||||
|
||||
## Perth and PixelSeal local oracle study
|
||||
|
||||
`scripts/perth_pixelseal_study.py` builds a local-only study from three
|
||||
deterministic synthetic audio carriers, three macOS system-voice speech
|
||||
carriers, and two publication-cleared tracked image carriers. It never calls a
|
||||
provider oracle or uploads an artifact. Generated media, the strict manifest,
|
||||
and results stay outside git.
|
||||
|
||||
PixelSeal comes from Meta's
|
||||
[VideoSeal source tree](https://github.com/facebookresearch/videoseal/tree/870ca7fb33578b90f14c602016b6c2788096226e),
|
||||
package `videoseal`, at the same exact `870ca7f` revision as the VideoSeal
|
||||
oracle. The project and weights are
|
||||
[MIT licensed](https://github.com/facebookresearch/videoseal/blob/870ca7fb33578b90f14c602016b6c2788096226e/LICENSE).
|
||||
The adapter verifies the official
|
||||
[PixelSeal model card](https://github.com/facebookresearch/videoseal/blob/870ca7fb33578b90f14c602016b6c2788096226e/videoseal/cards/pixelseal.yaml)
|
||||
as `sha256:96c4a448f7d81810780e5bde5a51b039db544b7ec2baf65a605302c1b17362c9`
|
||||
and its 1,237,429,197-byte `checkpoint.pth` as
|
||||
`sha256:0c5665cff20eb6ce1b5aaa7d91c19dafb418bfee32d02dd3344e4ed60d9d75bd`.
|
||||
The source metadata declares package version 1.0. The 1.0.1 PyPI wheel omits
|
||||
the PixelSeal card, so this development oracle likewise requires the exact
|
||||
source revision and remains outside installed CLI and library paths. Its
|
||||
explicit matched-message decision is 256-bit accuracy at or above 0.9, aligned
|
||||
with the VideoSeal adapter; this is the benchmark's rule, not a general
|
||||
watermark-presence classifier.
|
||||
|
||||
```bash
|
||||
uv run python scripts/perth_pixelseal_study.py \
|
||||
--output-dir .local-eval/perth-pixelseal-2026-09-28-v4
|
||||
```
|
||||
|
||||
### Development baseline, 2026-09-28
|
||||
|
||||
The final local run used recipe
|
||||
`sha256:b43953678294d97a72bb323623797ec8d7b8e476b101485e9ecfefbe02353c0e`,
|
||||
kernel
|
||||
`sha256:26dce20f42f7131daa4885eed073baf7372ebb07c0e8ccb4ec50cd603da082bf`,
|
||||
Perth oracle
|
||||
`sha256:a99b9b64a7667b2ee46ac8b57e7c29eec6b13646e3297856e67816984f1a3a9c`,
|
||||
and PixelSeal oracle
|
||||
`sha256:fc15c444e21404c1413d94251e7746964a347d9596faecfdd94b5e45ab2d0949`.
|
||||
Its manifest is
|
||||
`sha256:ecdcca929a73b127db3682636f8a2322fc128abb632f3b779e415a7130e0150d`,
|
||||
case rows are
|
||||
`sha256:ec0a794f5a6580e7bc50e122e1ebae0ea193c9b35b69c7408126d00304c45b38`,
|
||||
and study metadata is
|
||||
`sha256:5906d8ef48132f0fb371e052870a67a682d4b85c7c5c0e23730a951dca970cc1`.
|
||||
All asserted rows matched their expected result; unresolved synthetic Perth
|
||||
positives remain measurements rather than hidden failures.
|
||||
|
||||
| Adapter and state | Cases | Detected | Not detected | Message accuracy or score |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| PixelSeal clean | 2 | 0 | 2 | 0.473-0.504 fixed-message accuracy |
|
||||
| PixelSeal marked | 2 | 2 | 0 | 1.000 on both |
|
||||
| PixelSeal JPEG q90 | 2 | 2 | 0 | 0.996-1.000 |
|
||||
| PixelSeal foreign-message forged | 2 | 0 | 2 | 0.469-0.473 against the fixed message |
|
||||
| Perth speech clean | 3 | 0 | 3 | 0-0.016 confidence |
|
||||
| Perth speech marked | 3 | 3 | 0 | 1.000 confidence |
|
||||
| Perth synthetic clean | 3 | 0 | 3 | 0-0.000083 confidence |
|
||||
| Perth synthetic marked | 3 | 1 | 2 | 0.190, 0.246, and 1.000 confidence |
|
||||
|
||||
PixelSeal embedding measured 46.15-49.46 dB PSNR against the two clean
|
||||
carriers. JPEG q90 retained the mark at 37.19-39.87 dB PSNR. The fixed-message
|
||||
verifier rejected both images carrying the foreign message; that proves only
|
||||
matched-key rejection, not absence of a PixelSeal payload.
|
||||
|
||||
Perth was reliable on the in-domain speech sample but carrier-dependent on the
|
||||
deliberately out-of-domain synthetic set: only white noise crossed the rule,
|
||||
while the tone stack and pinkish noise remained at 0.190 and 0.246. The three
|
||||
synthetic marked rows measured 12.91-14.98 dB SNR. Perth's internal 16 to 32 to
|
||||
16 kHz resampling shortened the three marked speech outputs by 53-104 samples,
|
||||
so the kernel correctly records their SNR as `incomparable` rather than
|
||||
silently trimming them.
|
||||
|
||||
The production visible-video cleaning path removed the visible mark from all
|
||||
24 frames while copying the marked AAC packets byte-for-byte. Both the source
|
||||
and cleaned audio scored 1.000, and the source and output packet streams share
|
||||
`sha256:d6be870baea63ef8dceea97b2c04b27a1a35ba3bd1500cd022ccb167fd0795c6`.
|
||||
This directly confirms that the current video pipeline does not affect Perth.
|
||||
|
||||
PixelSeal removal was not run on this Apple Silicon host. The applicable
|
||||
`qwen-zimage`, `sdxl-zimage`, and `chroma-zimage` invisible-removal profiles
|
||||
require an NVIDIA CUDA device; MPS is not a supported execution path. The
|
||||
missing removed-state rows are therefore an explicit hardware gap, not a clean
|
||||
or failed result.
|
||||
|
||||
## Audio cohort v1
|
||||
|
||||
`scripts/watermark_benchmark_audio_cohort.py` builds the audio analogue of the
|
||||
|
||||
@@ -265,9 +265,12 @@ A local matched-oracle study has now measured the boundary concretely:
|
||||
into synthetic carriers, runs the visible video-cleaning path, and shows the
|
||||
audio packets byte-identical and the watermark verdict invariant across
|
||||
cleaning, while additive noise on the audio path is the effective lever. The
|
||||
boundary above still stands for provider marks: this tool still does not
|
||||
decode them, and a `copied_if_present` audio track remains `unverified` in
|
||||
runtime output.
|
||||
[Perth and PixelSeal benchmark study](benchmark-kernel.md#perth-and-pixelseal-local-oracle-study)
|
||||
independently repeats the pass-through check with Resemble Perth: all 24 visual
|
||||
frames are cleaned, the marked AAC packets remain byte-identical, and Perth's
|
||||
score stays 1.0. The boundary above still stands for provider marks: this tool
|
||||
still does not decode them, and a `copied_if_present` audio track remains
|
||||
`unverified` in runtime output.
|
||||
|
||||
The shipped engine streams sampled frames in bounded batches, computes its
|
||||
fidelity metrics incrementally, and pipes regenerated pixels directly to
|
||||
|
||||
@@ -70,10 +70,16 @@ This page is an index. Each finding lives in its canonical page.
|
||||
gray block on Veo, and ROSE retained the Veo wordmark on 17/17 frames. They
|
||||
are not shipped
|
||||
([video inpainting research](video-inpainting-research.md)).
|
||||
Resemble Perth and Meta PixelSeal were added as exact-revision, MIT-licensed
|
||||
local benchmark oracles. The 2026-09-28 study found Perth on all three speech
|
||||
positives and confirmed that visible-video cleaning copies its marked AAC
|
||||
packets unchanged; PixelSeal survived JPEG q90 on both cleared image
|
||||
carriers, while its CUDA-only removal profiles could not run on this host
|
||||
([benchmark study](benchmark-kernel.md#perth-and-pixelseal-local-oracle-study)).
|
||||
Still open:
|
||||
Z-Image-Turbo as a global stage (with an Apple Silicon arm through mflux),
|
||||
linear probes on newer frozen backbones for `classify`, Resemble Perth and
|
||||
Meta PixelSeal as benchmark oracles, re-watermarking as a removal primitive
|
||||
linear probes on newer frozen backbones for `classify`, re-watermarking as a
|
||||
removal primitive
|
||||
(arXiv:2605.16796, MIT code), RAVEN view-synthesis removal
|
||||
(arXiv:2601.08832, no code released as of 2026-09-24).
|
||||
MiniMax-Remover (CC-BY-NC-4.0 weights) and ProPainter (S-Lab License,
|
||||
|
||||
@@ -449,10 +449,12 @@ audio-path attacks separately. Its carriers are synthetic non-speech audio
|
||||
and its oracle is AudioSeal, so it validates the copy semantics, not provider
|
||||
audio marks. The audio and video arms are now first-class benchmark rows:
|
||||
[benchmark kernel](benchmark-kernel.md) accepts `media_type: audio` and
|
||||
`media_type: video` with the revision-pinned `audioseal` and `videoseal`
|
||||
adapters, and both cohorts carry matched negatives, codec, and degradation
|
||||
arms as strict manifest cases. The temporal layer is closed too: videoseal
|
||||
detection records carry per-frame bit accuracy, and
|
||||
`media_type: video` with the revision-pinned `audioseal`, `perth`, and
|
||||
`videoseal` adapters, plus the `pixelseal` image adapter. The Perth and
|
||||
PixelSeal study records matched negatives, marked rows, attacks, fixed-message
|
||||
accuracy, and the copied-audio path as strict manifest cases. The temporal
|
||||
layer is closed too: videoseal detection records carry per-frame bit accuracy,
|
||||
and
|
||||
[VideoSeal temporal evaluation](videoseal-temporal-evaluation.md) measures
|
||||
the aggregation matrix and the crf decay curve on synthetic and real clips.
|
||||
What remains open here is a wider real-content sweep and the learned temporal
|
||||
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Revision-pinned local Perth oracle for development-only benchmarks.
|
||||
|
||||
Perth v1 is an implicit audio watermark: its public model returns one presence
|
||||
confidence, not a user-selected payload. This adapter therefore exposes a
|
||||
thresholded presence reading and deliberately has no message label or forgery
|
||||
operation.
|
||||
|
||||
The official ``resemble-perth`` wheel predates current PyTorch releases and
|
||||
pins an obsolete stack. Development runs install the exact source revision
|
||||
below without dependency resolution, then verify the bundled checkpoint before
|
||||
loading it. Nothing here ships in the installed CLI or public Python API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE_COMMIT = "ff1c8ac55a976971245cdd53c18d6131ca00d993"
|
||||
PACKAGE_NAME = "resemble-perth"
|
||||
PACKAGE_VERSION = "1.1.0"
|
||||
CHECKPOINT_SHA256 = "a15bce457ebc53ce5e6c9c3f11df78cf7ee2bf9cdab0a798902135b4c4027670"
|
||||
CHECKPOINT_RELATIVE_PATH = Path("perth_net/pretrained/implicit/perth_net_250000.pth.tar")
|
||||
SAMPLE_RATE = 16_000
|
||||
DETECTION_THRESHOLD = 0.5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerthReading:
|
||||
"""One implicit-watermark presence score."""
|
||||
|
||||
confidence: float
|
||||
|
||||
@property
|
||||
def detected(self) -> bool:
|
||||
"""Apply the upstream round-at-one-half decision rule."""
|
||||
return self.confidence > DETECTION_THRESHOLD
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def import_failure() -> Exception | None:
|
||||
"""Return the missing or incompatible dependency error, if any."""
|
||||
try:
|
||||
import librosa # noqa: F401
|
||||
import perth
|
||||
import torch # noqa: F401
|
||||
import torchaudio # noqa: F401
|
||||
|
||||
if importlib.metadata.version(PACKAGE_NAME) != PACKAGE_VERSION:
|
||||
raise RuntimeError(f"{PACKAGE_NAME} must be installed at version {PACKAGE_VERSION}")
|
||||
if perth.PerthImplicitWatermarker is None:
|
||||
raise RuntimeError("resemble-perth imported without PerthImplicitWatermarker")
|
||||
except (ImportError, importlib.metadata.PackageNotFoundError, RuntimeError) as exc:
|
||||
return exc
|
||||
return None
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Whether the pinned Perth source stack is importable."""
|
||||
return import_failure() is None
|
||||
|
||||
|
||||
def checkpoint_path() -> Path:
|
||||
"""Return the package checkpoint after verifying its pinned digest."""
|
||||
failure = import_failure()
|
||||
if failure is not None:
|
||||
raise RuntimeError(
|
||||
"Perth oracle requires the pinned development source and its local dependencies; "
|
||||
f"install resemble-ai/Perth@{SOURCE_COMMIT} without adding it to runtime dependencies"
|
||||
) from failure
|
||||
import perth
|
||||
|
||||
path = Path(perth.__file__).resolve().parent / CHECKPOINT_RELATIVE_PATH
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"pinned Perth checkpoint is missing: {path}")
|
||||
digest = _sha256_file(path)
|
||||
if digest != CHECKPOINT_SHA256:
|
||||
raise RuntimeError(f"pinned Perth checkpoint digest mismatch: {digest} != {CHECKPOINT_SHA256}")
|
||||
return path
|
||||
|
||||
|
||||
def load_model() -> object:
|
||||
"""Load the pinned Perth implicit watermarker on CPU."""
|
||||
checkpoint_path()
|
||||
import perth
|
||||
|
||||
return perth.PerthImplicitWatermarker(device="cpu")
|
||||
|
||||
|
||||
def embed(model: object, samples: object) -> object:
|
||||
"""Embed Perth's fixed implicit watermark into mono float samples."""
|
||||
import numpy as np
|
||||
|
||||
array = np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
return np.asarray(model.apply_watermark(array, sample_rate=SAMPLE_RATE), dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
def read(model: object, samples: object) -> PerthReading:
|
||||
"""Return the raw implicit-watermark confidence for mono float samples."""
|
||||
import numpy as np
|
||||
|
||||
array = np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
raw = np.asarray(model.get_watermark(array, sample_rate=SAMPLE_RATE, round=False), dtype=np.float32).reshape(-1)
|
||||
if raw.size != 1 or not np.isfinite(raw).all():
|
||||
raise RuntimeError(f"Perth returned an invalid confidence shape/value: {raw!r}")
|
||||
return PerthReading(confidence=float(raw[0]))
|
||||
Executable
+485
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and run the local Perth and PixelSeal benchmark study.
|
||||
|
||||
The study uses only deterministic synthetic audio and the two publication-
|
||||
cleared engine-selection carriers named below. It measures Perth before and
|
||||
after the production visible-video cleaning path, and PixelSeal before and
|
||||
after JPEG plus a foreign-message embed. Invisible image-removal profiles are
|
||||
recorded as unavailable when the host has no CUDA device; this script never
|
||||
falls back to a provider oracle or uploads an artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "src"))
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import perth_oracle # noqa: E402
|
||||
import pixelseal_oracle # noqa: E402
|
||||
from audioseal_experiment import ( # noqa: E402
|
||||
FRAME_COUNT,
|
||||
SPEECH_VOICES,
|
||||
aac_packet_bytes,
|
||||
assert_distinct_voices,
|
||||
carrier_seed,
|
||||
decode_audio_f32,
|
||||
encode_video,
|
||||
ffmpeg_version,
|
||||
require_tools,
|
||||
run_ffmpeg,
|
||||
sora_like_mark,
|
||||
speech_provenance,
|
||||
stamped_frame,
|
||||
synth_carrier,
|
||||
synthesize_speech,
|
||||
wav_pcm16_bytes,
|
||||
)
|
||||
from watermark_benchmark import SCHEMA_VERSION, run_benchmark, sha256_file, write_jsonl # noqa: E402
|
||||
|
||||
from remove_ai_watermarks.video import remove_video_visible # noqa: E402
|
||||
|
||||
log = logging.getLogger("perth_pixelseal_study")
|
||||
|
||||
RECIPE_VERSION = "perth-pixelseal-study-v2"
|
||||
IMAGE_SIZE = 512
|
||||
IMAGE_CARRIERS: tuple[Path, ...] = (
|
||||
REPO / "data/evaluations/engine-selection/originals/openai/00.png",
|
||||
REPO / "data/evaluations/engine-selection/originals/meta/00.png",
|
||||
)
|
||||
AUDIO_CARRIERS: tuple[str, ...] = ("tone_stack", "pinkish_noise", "white_noise")
|
||||
AUDIO_DURATION_S = 8.0
|
||||
|
||||
|
||||
def _row(
|
||||
*,
|
||||
root: Path,
|
||||
case_id: str,
|
||||
pair_id: str,
|
||||
media_type: str,
|
||||
adapter: str,
|
||||
arm: str,
|
||||
state: str,
|
||||
path: Path,
|
||||
reference: Path | None,
|
||||
source_revision: str,
|
||||
transform_name: str,
|
||||
transform_revision: str,
|
||||
parameters: dict[str, object],
|
||||
seed: int | None,
|
||||
expected: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"case_id": case_id,
|
||||
"pair_id": pair_id,
|
||||
"media_type": media_type,
|
||||
"adapter": adapter,
|
||||
"arm": arm,
|
||||
"state": state,
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"sha256": sha256_file(path),
|
||||
"reference_path": reference.relative_to(root).as_posix() if reference is not None else None,
|
||||
"reference_sha256": sha256_file(reference) if reference is not None else None,
|
||||
"source_revision": source_revision,
|
||||
"transform": {
|
||||
"name": transform_name,
|
||||
"revision": transform_revision,
|
||||
"parameters": parameters,
|
||||
},
|
||||
"seed": seed,
|
||||
"expected": expected,
|
||||
}
|
||||
|
||||
|
||||
def _save_rgb(path: Path, values: np.ndarray) -> None:
|
||||
pixels = np.rint(np.clip(values, 0.0, 1.0) * 255.0).astype(np.uint8)
|
||||
Image.fromarray(pixels, "RGB").save(path)
|
||||
|
||||
|
||||
def _build_pixelseal_rows(output_dir: Path, artifacts: Path) -> list[dict[str, object]]:
|
||||
if not pixelseal_oracle.available():
|
||||
failure = pixelseal_oracle.import_failure()
|
||||
raise SystemExit(f"PixelSeal development stack is unavailable: {failure}")
|
||||
model = pixelseal_oracle.load_model()
|
||||
fixed = pixelseal_oracle.message_bits()
|
||||
foreign = pixelseal_oracle.message_bits(seed=8)
|
||||
pins = {
|
||||
"source_commit": pixelseal_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": pixelseal_oracle.CHECKPOINT_SHA256,
|
||||
"model_card_sha256": pixelseal_oracle.MODEL_CARD_SHA256,
|
||||
"message": list(fixed),
|
||||
}
|
||||
rows: list[dict[str, object]] = []
|
||||
source_manifest = REPO / "data/evaluations/engine-selection/carrier-manifest.csv"
|
||||
source_revision = f"engine-selection@sha256:{sha256_file(source_manifest)}"
|
||||
for source in IMAGE_CARRIERS:
|
||||
provider = source.parent.name
|
||||
pair_id = f"pixelseal-{provider}-00"
|
||||
clean_path = artifacts / f"{pair_id}-clean.png"
|
||||
clean = Image.open(source).convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.Resampling.LANCZOS)
|
||||
clean.save(clean_path)
|
||||
clean_values = np.asarray(clean, dtype=np.float32) / 255.0
|
||||
|
||||
marked_path = artifacts / f"{pair_id}-marked.png"
|
||||
_save_rgb(marked_path, np.asarray(pixelseal_oracle.embed(model, clean_values, fixed)))
|
||||
attacked_path = artifacts / f"{pair_id}-jpeg-q90.jpg"
|
||||
Image.open(marked_path).save(attacked_path, quality=90, subsampling=0)
|
||||
forged_path = artifacts / f"{pair_id}-forged.png"
|
||||
_save_rgb(forged_path, np.asarray(pixelseal_oracle.embed(model, clean_values, foreign)))
|
||||
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": pair_id,
|
||||
"media_type": "image",
|
||||
"adapter": "pixelseal",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="standardize-carrier",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"source_sha256": sha256_file(source), "size": IMAGE_SIZE},
|
||||
seed=None,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="pixelseal-embed",
|
||||
transform_revision=f"videoseal@{pixelseal_oracle.SOURCE_COMMIT}",
|
||||
parameters=pins,
|
||||
seed=pixelseal_oracle.MESSAGE_SEED,
|
||||
expected="detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-jpeg-q90",
|
||||
arm="positive",
|
||||
state="attacked",
|
||||
path=attacked_path,
|
||||
reference=clean_path,
|
||||
transform_name="jpeg",
|
||||
transform_revision="pillow",
|
||||
parameters={"quality": 90, "subsampling": 0, **pins},
|
||||
seed=None,
|
||||
expected="unresolved",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-forged",
|
||||
arm="wrong_key",
|
||||
state="forged",
|
||||
path=forged_path,
|
||||
reference=clean_path,
|
||||
transform_name="pixelseal-embed-foreign-message",
|
||||
transform_revision=f"videoseal@{pixelseal_oracle.SOURCE_COMMIT}",
|
||||
parameters={**pins, "foreign_message": list(foreign), "foreign_message_seed": 8},
|
||||
seed=8,
|
||||
expected="not_detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
del model
|
||||
gc.collect()
|
||||
return rows
|
||||
|
||||
|
||||
def _extract_audio(ffmpeg: str, source: Path, target: Path) -> None:
|
||||
run_ffmpeg(
|
||||
ffmpeg,
|
||||
[
|
||||
"-i",
|
||||
str(source),
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
str(perth_oracle.SAMPLE_RATE),
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
str(target),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _build_perth_rows(
|
||||
output_dir: Path,
|
||||
artifacts: Path,
|
||||
ffmpeg: str,
|
||||
) -> tuple[list[dict[str, object]], dict[str, object]]:
|
||||
if not perth_oracle.available():
|
||||
failure = perth_oracle.import_failure()
|
||||
raise SystemExit(f"Perth development stack is unavailable: {failure}")
|
||||
model = perth_oracle.load_model()
|
||||
pins = {
|
||||
"source_commit": perth_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": perth_oracle.CHECKPOINT_SHA256,
|
||||
"implicit_payload": True,
|
||||
}
|
||||
source_revision = f"{RECIPE_VERSION}@sha256:{sha256_file(Path(__file__))}"
|
||||
rows: list[dict[str, object]] = []
|
||||
samantha_marked_path: Path | None = None
|
||||
for name in AUDIO_CARRIERS:
|
||||
executed_seed = carrier_seed(name)
|
||||
clean_samples = synth_carrier(name, AUDIO_DURATION_S, seed=executed_seed)
|
||||
clean_path = artifacts / f"perth-{name}-clean.wav"
|
||||
clean_path.write_bytes(wav_pcm16_bytes(clean_samples, perth_oracle.SAMPLE_RATE))
|
||||
marked_path = artifacts / f"perth-{name}-marked.wav"
|
||||
marked = np.asarray(perth_oracle.embed(model, clean_samples), dtype=np.float32)
|
||||
marked_path.write_bytes(wav_pcm16_bytes(marked, perth_oracle.SAMPLE_RATE))
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": f"perth-{name}",
|
||||
"media_type": "audio",
|
||||
"adapter": "perth",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"perth-{name}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="synthesize-carrier",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"carrier": name, "duration_s": AUDIO_DURATION_S},
|
||||
seed=executed_seed,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"perth-{name}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="perth-embed",
|
||||
transform_revision=f"resemble-ai/Perth@{perth_oracle.SOURCE_COMMIT}",
|
||||
parameters=pins,
|
||||
seed=None,
|
||||
# Perth is speech-trained; these deliberately out-of-domain
|
||||
# carriers measure dependence without asserting detection.
|
||||
expected="unresolved",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
raw_speech = {voice: synthesize_speech(voice, artifacts) for voice in SPEECH_VOICES}
|
||||
assert_distinct_voices({voice: path.read_bytes() for voice, path in raw_speech.items()})
|
||||
tts = speech_provenance()
|
||||
for voice, aiff in raw_speech.items():
|
||||
pair_id = f"perth-speech-{voice.casefold()}"
|
||||
clean_samples = decode_audio_f32(ffmpeg, aiff)
|
||||
clean_path = artifacts / f"{pair_id}-clean.wav"
|
||||
clean_path.write_bytes(wav_pcm16_bytes(clean_samples, perth_oracle.SAMPLE_RATE))
|
||||
marked = np.asarray(perth_oracle.embed(model, clean_samples), dtype=np.float32)
|
||||
marked_path = artifacts / f"{pair_id}-marked.wav"
|
||||
marked_path.write_bytes(wav_pcm16_bytes(marked, perth_oracle.SAMPLE_RATE))
|
||||
if voice == "Samantha":
|
||||
samantha_marked_path = marked_path
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": pair_id,
|
||||
"media_type": "audio",
|
||||
"adapter": "perth",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
provenance = {**tts, "voice": voice, "duration_s": round(clean_samples.shape[0] / perth_oracle.SAMPLE_RATE, 6)}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="synthesize-speech",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"tts": provenance},
|
||||
seed=None,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="perth-embed",
|
||||
transform_revision=f"resemble-ai/Perth@{perth_oracle.SOURCE_COMMIT}",
|
||||
parameters={**pins, "tts": provenance},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
mark, mark_x, mark_y = sora_like_mark()
|
||||
frames = [stamped_frame(index, mark, mark_x, mark_y) for index in range(FRAME_COUNT)]
|
||||
if samantha_marked_path is None:
|
||||
raise RuntimeError("the configured speech carriers must include Samantha")
|
||||
source_video = artifacts / "perth-video-source.mp4"
|
||||
encode_video(
|
||||
ffmpeg,
|
||||
source_video,
|
||||
frames,
|
||||
samantha_marked_path,
|
||||
audio_codec="aac",
|
||||
audio_rate=perth_oracle.SAMPLE_RATE,
|
||||
)
|
||||
cleaned_video = artifacts / "perth-video-cleaned.mp4"
|
||||
result = remove_video_visible(source_video, cleaned_video, mark="sora", backend="cv2")
|
||||
if result.output is None:
|
||||
raise SystemExit("visible-video cleaning did not produce an output")
|
||||
source_packets = aac_packet_bytes(ffmpeg, source_video)
|
||||
cleaned_packets = aac_packet_bytes(ffmpeg, cleaned_video)
|
||||
source_audio = artifacts / "perth-video-source-audio.wav"
|
||||
cleaned_audio = artifacts / "perth-video-cleaned-audio.wav"
|
||||
_extract_audio(ffmpeg, source_video, source_audio)
|
||||
_extract_audio(ffmpeg, cleaned_video, cleaned_audio)
|
||||
packet_identity = source_packets == cleaned_packets
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
root=output_dir,
|
||||
case_id="perth-video-source-audio",
|
||||
pair_id="perth-video-pass-through",
|
||||
media_type="audio",
|
||||
adapter="perth",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=source_audio,
|
||||
reference=None,
|
||||
source_revision=source_revision,
|
||||
transform_name="mux-aac",
|
||||
transform_revision=ffmpeg_version(ffmpeg),
|
||||
parameters={"codec": "aac", "bitrate": "128k", **pins},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
_row(
|
||||
root=output_dir,
|
||||
case_id="perth-video-cleaned-audio",
|
||||
pair_id="perth-video-pass-through",
|
||||
media_type="audio",
|
||||
adapter="perth",
|
||||
arm="positive",
|
||||
state="attacked",
|
||||
path=cleaned_audio,
|
||||
reference=source_audio,
|
||||
source_revision=source_revision,
|
||||
transform_name="remove-video-visible-audio-pass-through",
|
||||
transform_revision=source_revision,
|
||||
parameters={
|
||||
"audio_packets_identical": packet_identity,
|
||||
"visual_removed_frames": result.removed_frames,
|
||||
**pins,
|
||||
},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
del model
|
||||
gc.collect()
|
||||
return rows, {
|
||||
"audio_packets_identical": packet_identity,
|
||||
"source_audio_packet_sha256": hashlib.sha256(source_packets).hexdigest(),
|
||||
"cleaned_audio_packet_sha256": hashlib.sha256(cleaned_packets).hexdigest(),
|
||||
"visual_removed_frames": result.removed_frames,
|
||||
}
|
||||
|
||||
|
||||
def _cuda_available() -> bool:
|
||||
try:
|
||||
import torch
|
||||
|
||||
return bool(torch.cuda.is_available())
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def build_and_run(output_dir: Path) -> Path:
|
||||
"""Create the study artifacts, manifest, metadata, and kernel results."""
|
||||
if output_dir.exists():
|
||||
raise FileExistsError(f"output directory already exists: {output_dir}")
|
||||
ffmpeg, _ffprobe = require_tools()
|
||||
output_dir.mkdir(parents=True)
|
||||
artifacts = output_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
rows = _build_pixelseal_rows(output_dir, artifacts)
|
||||
perth_rows, pass_through = _build_perth_rows(output_dir, artifacts, ffmpeg)
|
||||
rows.extend(perth_rows)
|
||||
manifest = output_dir / "manifest.jsonl"
|
||||
write_jsonl(manifest, rows)
|
||||
metadata = {
|
||||
"schema_version": 1,
|
||||
"recipe": RECIPE_VERSION,
|
||||
"pixelseal": {
|
||||
"source_commit": pixelseal_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": pixelseal_oracle.CHECKPOINT_SHA256,
|
||||
"removal_profiles": {
|
||||
"status": "not_run" if not _cuda_available() else "available_not_automatic",
|
||||
"reason": "qwen-zimage, sdxl-zimage, and chroma-zimage require an NVIDIA CUDA device",
|
||||
},
|
||||
},
|
||||
"perth": {
|
||||
"source_commit": perth_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": perth_oracle.CHECKPOINT_SHA256,
|
||||
"message": "not_applicable_fixed_implicit_mark",
|
||||
"video_visible_path": pass_through,
|
||||
},
|
||||
}
|
||||
(output_dir / "study-metadata.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
results = output_dir / "results.jsonl"
|
||||
run_benchmark(manifest, results)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
try:
|
||||
results = build_and_run(args.output_dir.resolve())
|
||||
except (FileNotFoundError, OSError, RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
log.info("Wrote study results to %s", results)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+214
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Revision-pinned local PixelSeal oracle for development-only benchmarks.
|
||||
|
||||
Meta publishes PixelSeal in the VideoSeal source tree, but not as a standalone
|
||||
TorchScript artifact. This loader uses the exact upstream source revision and
|
||||
official checkpoint, verifies both the model card and checkpoint digests, and
|
||||
builds only the image inference graph. It avoids the upstream convenience
|
||||
loader because that loader imports training datasets and writes downloads
|
||||
relative to the current working directory.
|
||||
|
||||
Nothing here ships in the installed CLI or public Python API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import logging
|
||||
import os
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE_COMMIT = "870ca7fb33578b90f14c602016b6c2788096226e"
|
||||
PACKAGE_NAME = "videoseal"
|
||||
PACKAGE_VERSION = "1.0"
|
||||
CHECKPOINT_URL = "https://dl.fbaipublicfiles.com/videoseal/pixelseal/checkpoint.pth"
|
||||
CHECKPOINT_SHA256 = "0c5665cff20eb6ce1b5aaa7d91c19dafb418bfee32d02dd3344e4ed60d9d75bd"
|
||||
MODEL_CARD_SHA256 = "96c4a448f7d81810780e5bde5a51b039db544b7ec2baf65a605302c1b17362c9"
|
||||
N_BITS = 256
|
||||
MESSAGE_SEED = 7
|
||||
DETECTION_BIT_ACCURACY_THRESHOLD = 0.9
|
||||
CACHE_DIR = Path(
|
||||
os.environ.get(
|
||||
"PIXELSEAL_CACHE_DIR",
|
||||
str(Path.home() / ".cache" / "remove-ai-watermarks-dev" / "pixelseal"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PixelSealReading:
|
||||
"""Decoded bits and matched-message accuracy for one image."""
|
||||
|
||||
bit_accuracy: float
|
||||
decoded_bits: tuple[int, ...]
|
||||
|
||||
@property
|
||||
def detected(self) -> bool:
|
||||
"""Apply this adapter's explicit matched-message rule."""
|
||||
return self.bit_accuracy >= DETECTION_BIT_ACCURACY_THRESHOLD
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return "".join(f"{byte:02x}" for byte in _bytes(self.decoded_bits))
|
||||
|
||||
|
||||
def _bytes(bits: Sequence[int]) -> bytes:
|
||||
return bytes(sum(bit << (7 - index) for index, bit in enumerate(bits[i : i + 8])) for i in range(0, len(bits), 8))
|
||||
|
||||
|
||||
def message_bits(seed: int = MESSAGE_SEED) -> tuple[int, ...]:
|
||||
"""The oracle's fixed 256-bit message, deterministic across processes."""
|
||||
import numpy as np
|
||||
|
||||
rng = np.random.default_rng(seed)
|
||||
return tuple(int(bit) for bit in rng.integers(0, 2, N_BITS))
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def import_failure() -> Exception | None:
|
||||
"""Return the missing or incompatible dependency error, if any."""
|
||||
try:
|
||||
import omegaconf # noqa: F401
|
||||
import timm # noqa: F401
|
||||
import torch # noqa: F401
|
||||
import torchvision # noqa: F401
|
||||
import videoseal # noqa: F401
|
||||
|
||||
if importlib.metadata.version(PACKAGE_NAME) != PACKAGE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"{PACKAGE_NAME} must be installed from source commit {SOURCE_COMMIT} "
|
||||
f"(package version {PACKAGE_VERSION})"
|
||||
)
|
||||
except (ImportError, importlib.metadata.PackageNotFoundError, RuntimeError) as exc:
|
||||
return exc
|
||||
return None
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Whether the pinned PixelSeal source stack is importable."""
|
||||
return import_failure() is None
|
||||
|
||||
|
||||
def checkpoint_path() -> Path:
|
||||
"""Return the pinned local checkpoint, downloading and verifying it once."""
|
||||
target = CACHE_DIR / "checkpoint.pth"
|
||||
if target.is_file() and _sha256_file(target) == CHECKPOINT_SHA256:
|
||||
return target
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
temporary = CACHE_DIR / "checkpoint.pth.part"
|
||||
log.info("Downloading pinned PixelSeal checkpoint from %s", CHECKPOINT_URL)
|
||||
urllib.request.urlretrieve(CHECKPOINT_URL, temporary)
|
||||
digest = _sha256_file(temporary)
|
||||
if digest != CHECKPOINT_SHA256:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"pinned PixelSeal checkpoint digest mismatch: {digest} != {CHECKPOINT_SHA256}")
|
||||
temporary.replace(target)
|
||||
return target
|
||||
|
||||
|
||||
def load_model() -> object:
|
||||
"""Build PixelSeal from the pinned official source and weights on CPU."""
|
||||
failure = import_failure()
|
||||
if failure is not None:
|
||||
raise RuntimeError(
|
||||
"PixelSeal oracle requires facebookresearch/videoseal at source commit "
|
||||
f"{SOURCE_COMMIT}, plus torch, torchvision, timm, and omegaconf"
|
||||
) from failure
|
||||
|
||||
import torch
|
||||
import videoseal
|
||||
from omegaconf import OmegaConf
|
||||
from videoseal.augmentation.augmenter import get_dummy_augmenter
|
||||
from videoseal.models import Videoseal, build_embedder, build_extractor
|
||||
from videoseal.modules.jnd import JND
|
||||
|
||||
package_root = Path(videoseal.__file__).resolve().parent
|
||||
card_path = package_root / "cards" / "pixelseal.yaml"
|
||||
if not card_path.is_file() or _sha256_file(card_path) != MODEL_CARD_SHA256:
|
||||
raise RuntimeError("installed videoseal source does not carry the pinned PixelSeal model card")
|
||||
config = OmegaConf.load(card_path)
|
||||
args = config.args
|
||||
embedder = build_embedder(
|
||||
config.embedder.model,
|
||||
config.embedder.params,
|
||||
args.nbits,
|
||||
args.hidden_size_multiplier,
|
||||
)
|
||||
extractor = build_extractor(
|
||||
config.extractor.model,
|
||||
config.extractor.params,
|
||||
args.img_size_proc,
|
||||
args.nbits,
|
||||
)
|
||||
model = Videoseal(
|
||||
embedder,
|
||||
extractor,
|
||||
get_dummy_augmenter(),
|
||||
attenuation=JND(in_channels=1, out_channels=1),
|
||||
scaling_w=args.scaling_w,
|
||||
scaling_i=args.scaling_i,
|
||||
img_size=args.img_size_proc,
|
||||
chunk_size=args.videowam_chunk_size,
|
||||
step_size=args.videowam_step_size,
|
||||
)
|
||||
checkpoint = torch.load(checkpoint_path(), map_location="cpu", weights_only=True)
|
||||
loaded = model.load_state_dict(checkpoint["model"], strict=False)
|
||||
if loaded.missing_keys or loaded.unexpected_keys:
|
||||
raise RuntimeError(
|
||||
f"PixelSeal checkpoint/source mismatch: missing={loaded.missing_keys}, unexpected={loaded.unexpected_keys}"
|
||||
)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def _image_tensor(image: object) -> object:
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
array = np.asarray(image)
|
||||
if array.ndim != 3 or array.shape[2] != 3:
|
||||
raise ValueError(f"PixelSeal expects HxWx3 RGB pixels, got {array.shape}")
|
||||
values = array.astype(np.float32) / 255.0 if array.dtype == np.uint8 else np.asarray(array, dtype=np.float32)
|
||||
if not np.isfinite(values).all() or float(values.min()) < 0.0 or float(values.max()) > 1.0:
|
||||
raise ValueError("PixelSeal RGB pixels must be finite and in [0, 1]")
|
||||
return torch.from_numpy(np.ascontiguousarray(values)).permute(2, 0, 1).unsqueeze(0)
|
||||
|
||||
|
||||
def embed(model: object, image: object, message: Sequence[int]) -> object:
|
||||
"""Embed one message into an RGB image, returning float RGB in [0, 1]."""
|
||||
import torch
|
||||
|
||||
tensor = _image_tensor(image)
|
||||
msg = torch.tensor([list(message)], dtype=torch.float32)
|
||||
with torch.inference_mode():
|
||||
marked = model.embed(tensor, msgs=msg, is_video=False)["imgs_w"]
|
||||
return marked[0].permute(1, 2, 0).numpy()
|
||||
|
||||
|
||||
def read(model: object, image: object, *, message: Sequence[int] | None = None) -> PixelSealReading:
|
||||
"""Decode one image and compare its bits with the oracle's fixed message."""
|
||||
import torch
|
||||
|
||||
expected = message_bits() if message is None else tuple(message)
|
||||
tensor = _image_tensor(image)
|
||||
with torch.inference_mode():
|
||||
predictions = model.detect(tensor)["preds"]
|
||||
decoded = tuple(int(bit) for bit in (predictions[0, 1:] > 0).to(torch.int64).tolist())
|
||||
accuracy = sum(int(bit == want) for bit, want in zip(decoded, expected, strict=True)) / N_BITS
|
||||
return PixelSealReading(bit_accuracy=accuracy, decoded_bits=decoded)
|
||||
@@ -29,6 +29,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
@@ -71,7 +72,9 @@ MEDIA_TYPES: tuple[Media, ...] = ("image", "audio", "video")
|
||||
ADAPTER_MEDIA: dict[str, Media] = {
|
||||
"dwt-dct": "image",
|
||||
"trustmark": "image",
|
||||
"pixelseal": "image",
|
||||
"audioseal": "audio",
|
||||
"perth": "audio",
|
||||
"videoseal": "video",
|
||||
}
|
||||
_FIELDS = {
|
||||
@@ -131,6 +134,8 @@ class DetectorOutcome:
|
||||
label: str | None
|
||||
error: str | None = None
|
||||
temporal: tuple[float, ...] | None = None
|
||||
message_bit_accuracy: float | None = None
|
||||
score: float | None = None
|
||||
|
||||
|
||||
class DetectorAdapter(Protocol):
|
||||
@@ -173,11 +178,66 @@ class VideoSealAdapter:
|
||||
def detect(self, path: Path, frames: NDArray[Any]) -> DetectorOutcome:
|
||||
from videoseal_oracle import read
|
||||
|
||||
reading = read(_videoseal_model(), frames)
|
||||
reading = read(_oracle_model("videoseal_oracle"), frames)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label=reading.label if reading.detected else None,
|
||||
temporal=reading.per_frame_bit_accuracy,
|
||||
message_bit_accuracy=reading.bit_accuracy,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerthAdapter:
|
||||
"""Adapter for Perth's implicit presence score."""
|
||||
|
||||
name: str
|
||||
source_file: Path
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
"""Probe the optional stack only when a Perth row is evaluated."""
|
||||
from perth_oracle import available
|
||||
|
||||
return available()
|
||||
|
||||
def detect(self, path: Path, samples: NDArray[Any]) -> DetectorOutcome:
|
||||
from perth_oracle import read
|
||||
|
||||
reading = read(_oracle_model("perth_oracle"), samples)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label="perth-v1" if reading.detected else None,
|
||||
score=reading.confidence,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PixelSealAdapter:
|
||||
"""Adapter for PixelSeal's fixed-message image verifier."""
|
||||
|
||||
name: str
|
||||
source_file: Path
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
"""Probe the optional stack only when a PixelSeal row is evaluated."""
|
||||
from pixelseal_oracle import available
|
||||
|
||||
return available()
|
||||
|
||||
def detect(self, path: Path, image: NDArray[Any]) -> DetectorOutcome:
|
||||
import cv2
|
||||
from pixelseal_oracle import read
|
||||
|
||||
from remove_ai_watermarks.image_io import to_bgr
|
||||
|
||||
rgb = cv2.cvtColor(to_bgr(image), cv2.COLOR_BGR2RGB)
|
||||
reading = read(_oracle_model("pixelseal_oracle"), rgb)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label=reading.label if reading.detected else None,
|
||||
message_bit_accuracy=reading.bit_accuracy,
|
||||
)
|
||||
|
||||
|
||||
@@ -409,11 +469,15 @@ def default_adapters() -> dict[str, DetectorAdapter]:
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
import audioseal_oracle
|
||||
import perth_oracle
|
||||
import pixelseal_oracle
|
||||
import videoseal_oracle
|
||||
|
||||
dwt_source = Path(invisible_watermark.__file__).resolve()
|
||||
trustmark_source = Path(trustmark_detector.__file__).resolve()
|
||||
audio_oracle_source = Path(audioseal_oracle.__file__).resolve()
|
||||
perth_oracle_source = Path(perth_oracle.__file__).resolve()
|
||||
pixel_oracle_source = Path(pixelseal_oracle.__file__).resolve()
|
||||
video_oracle_source = Path(videoseal_oracle.__file__).resolve()
|
||||
return {
|
||||
"dwt-dct": FunctionAdapter(
|
||||
@@ -428,12 +492,20 @@ def default_adapters() -> dict[str, DetectorAdapter]:
|
||||
available=trustmark_detector.is_available(),
|
||||
detector=_detect_trustmark,
|
||||
),
|
||||
"pixelseal": PixelSealAdapter(
|
||||
name="pixelseal",
|
||||
source_file=pixel_oracle_source,
|
||||
),
|
||||
"audioseal": FunctionAdapter(
|
||||
name="audioseal",
|
||||
source_file=audio_oracle_source,
|
||||
available=audioseal_oracle.available(),
|
||||
detector=_detect_audioseal,
|
||||
),
|
||||
"perth": PerthAdapter(
|
||||
name="perth",
|
||||
source_file=perth_oracle_source,
|
||||
),
|
||||
"videoseal": VideoSealAdapter(
|
||||
name="videoseal",
|
||||
source_file=video_oracle_source,
|
||||
@@ -585,13 +657,13 @@ def _audioseal_detector() -> object:
|
||||
|
||||
|
||||
@cache
|
||||
def _videoseal_model() -> object:
|
||||
def _oracle_model(module_name: str) -> object:
|
||||
"""Load and cache one development-only oracle model."""
|
||||
scripts_dir = Path(__file__).resolve().parent
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
from videoseal_oracle import load_model
|
||||
|
||||
return load_model()
|
||||
module = importlib.import_module(module_name)
|
||||
return module.load_model()
|
||||
|
||||
|
||||
def _image_fidelity(
|
||||
@@ -745,6 +817,10 @@ def _detection_record(
|
||||
"max_bit_accuracy": max(values) if values else None,
|
||||
"per_frame_bit_accuracy": [round(value, 6) for value in values],
|
||||
}
|
||||
if outcome.message_bit_accuracy is not None:
|
||||
record["message_bit_accuracy"] = float(outcome.message_bit_accuracy)
|
||||
if outcome.score is not None:
|
||||
record["score"] = float(outcome.score)
|
||||
if outcome.error is not None:
|
||||
record["error"] = outcome.error
|
||||
return record
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Tests for the pinned Perth oracle's pure contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import perth_oracle # noqa: E402
|
||||
|
||||
|
||||
def test_detection_rule_is_upstream_round_at_one_half() -> None:
|
||||
assert perth_oracle.PerthReading(0.499).detected is False
|
||||
assert perth_oracle.PerthReading(0.5).detected is False
|
||||
assert perth_oracle.PerthReading(0.501).detected is True
|
||||
|
||||
|
||||
def test_checkpoint_and_source_are_immutable_pins() -> None:
|
||||
assert len(perth_oracle.SOURCE_COMMIT) == 40
|
||||
assert len(perth_oracle.CHECKPOINT_SHA256) == 64
|
||||
assert all(char in "0123456789abcdef" for char in perth_oracle.CHECKPOINT_SHA256)
|
||||
|
||||
|
||||
def test_perth_has_no_message_payload_contract() -> None:
|
||||
reading = perth_oracle.PerthReading(0.75)
|
||||
|
||||
assert not hasattr(reading, "decoded_bits")
|
||||
assert not hasattr(perth_oracle, "message_bits")
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for the pinned PixelSeal oracle's pure helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import pixelseal_oracle # noqa: E402
|
||||
|
||||
|
||||
class TestMessage:
|
||||
def test_message_is_deterministic_and_256_bits(self) -> None:
|
||||
first = pixelseal_oracle.message_bits()
|
||||
second = pixelseal_oracle.message_bits()
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 256
|
||||
assert set(first) <= {0, 1}
|
||||
|
||||
def test_seed_changes_the_message(self) -> None:
|
||||
assert pixelseal_oracle.message_bits(seed=8) != pixelseal_oracle.message_bits(seed=7)
|
||||
|
||||
|
||||
class TestReading:
|
||||
def test_label_is_hex_of_decoded_bits(self) -> None:
|
||||
reading = pixelseal_oracle.PixelSealReading(1.0, tuple([1] * 8 + [0] * 8))
|
||||
|
||||
assert reading.label == "ff00"
|
||||
|
||||
def test_detection_rule_is_the_documented_adapter_threshold(self) -> None:
|
||||
assert pixelseal_oracle.PixelSealReading(0.899, ()).detected is False
|
||||
assert pixelseal_oracle.PixelSealReading(0.9, ()).detected is True
|
||||
|
||||
|
||||
class TestPins:
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[pixelseal_oracle.CHECKPOINT_SHA256, pixelseal_oracle.MODEL_CARD_SHA256],
|
||||
)
|
||||
def test_digests_are_sha256(self, value: str) -> None:
|
||||
assert len(value) == 64
|
||||
assert all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
def test_source_commit_is_pinned(self) -> None:
|
||||
assert len(pixelseal_oracle.SOURCE_COMMIT) == 40
|
||||
|
||||
|
||||
def test_cache_dir_env_override_is_honored(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
original_cache = pixelseal_oracle.CACHE_DIR
|
||||
try:
|
||||
with monkeypatch.context() as environment:
|
||||
environment.setenv("PIXELSEAL_CACHE_DIR", str(tmp_path / "cache"))
|
||||
importlib.reload(pixelseal_oracle)
|
||||
assert tmp_path / "cache" == pixelseal_oracle.CACHE_DIR
|
||||
finally:
|
||||
importlib.reload(pixelseal_oracle)
|
||||
assert original_cache == pixelseal_oracle.CACHE_DIR
|
||||
@@ -393,10 +393,12 @@ def test_jsonl_writer_refuses_overwrite_and_emits_strict_json(tmp_path: Path) ->
|
||||
def test_builtin_adapters_name_their_exact_source_files() -> None:
|
||||
adapters = MODULE.default_adapters()
|
||||
|
||||
assert set(adapters) == {"dwt-dct", "trustmark", "audioseal", "videoseal"}
|
||||
assert set(adapters) == {"dwt-dct", "trustmark", "pixelseal", "audioseal", "perth", "videoseal"}
|
||||
assert adapters["dwt-dct"].source_file.name == "invisible_watermark.py"
|
||||
assert adapters["trustmark"].source_file.name == "trustmark_detector.py"
|
||||
assert adapters["pixelseal"].source_file.name == "pixelseal_oracle.py"
|
||||
assert adapters["audioseal"].source_file.name == "audioseal_oracle.py"
|
||||
assert adapters["perth"].source_file.name == "perth_oracle.py"
|
||||
assert adapters["videoseal"].source_file.name == "videoseal_oracle.py"
|
||||
|
||||
|
||||
@@ -741,6 +743,20 @@ def test_detection_record_omits_temporal_for_non_temporal_adapters() -> None:
|
||||
assert "temporal" not in MODULE._detection_record(outcome, "not_detected", None)
|
||||
|
||||
|
||||
def test_detection_record_carries_message_accuracy_and_presence_score() -> None:
|
||||
outcome = MODULE.DetectorOutcome(
|
||||
status="detected",
|
||||
label="fixed-message",
|
||||
message_bit_accuracy=0.996,
|
||||
score=0.75,
|
||||
)
|
||||
|
||||
record = MODULE._detection_record(outcome, "detected", 2.0)
|
||||
|
||||
assert record["message_bit_accuracy"] == 0.996
|
||||
assert record["score"] == 0.75
|
||||
|
||||
|
||||
def test_run_benchmark_records_videoseal_temporal_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
pytest.skip("video rows decode through the system ffmpeg")
|
||||
|
||||
Reference in new issue
Block a user