mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-20 16:07:17 +02:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ca59c032a1 | ||
|
|
c5e138357e | ||
|
|
21661bea35 | ||
|
|
5cb4f2f808 | ||
|
|
33f748b32a |
No files matched your search
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Agent skill and Claude Code plugin for the remove-ai-watermarks CLI.",
|
||||
"version": "1.0.9"
|
||||
"version": "1.0.10"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -44,10 +44,19 @@ Playwright CLI runner, copied cookies, or a new login.
|
||||
real Chrome profile and navigate directly to
|
||||
`https://gemini.google.com/u/<google_account_index>/app`.
|
||||
3. Prepare an immutable `gemini-web` batch outside the repository.
|
||||
For every image check, visually inspect the source first. Remove every visible
|
||||
AI-provider logo or label with the project's visible-mark remover, derive all
|
||||
paired controls and candidates from that cleaned image, and let `prepare` strip
|
||||
AI metadata. Inspect the prepared upload again and do not submit it if a visible
|
||||
provider mark remains. A model answer based on a visible logo is not a SynthID
|
||||
result.
|
||||
4. Before each upload, request the action-time confirmation required for sending
|
||||
that exact prepared file to Gemini. Do not upload the source path directly.
|
||||
5. Attach one prepared upload and ask: `Was this image/video/audio created or
|
||||
edited by Google AI?`
|
||||
5. Attach one prepared upload and ask: `Does this image/video/audio contain a
|
||||
SynthID watermark? Use the SynthID verification tool only; do not infer from
|
||||
visible content, logos, text, or metadata.` Do not substitute a general
|
||||
Google-AI-origin question; it permits visual-content reasoning instead of a
|
||||
SynthID verdict.
|
||||
6. Read the settled SynthID tool outcome, not the surrounding model reasoning.
|
||||
Preserve unclear, quota, and refusal wording rather than interpreting it as
|
||||
clean.
|
||||
|
||||
@@ -44,7 +44,7 @@ The published Agent Skill is [`skills/remove-ai-watermarks/`](skills/remove-ai-w
|
||||
|
||||
[`docs/module-internals.md`](docs/module-internals.md) is the canonical per-module map, including design decisions, thresholds, calibration history, incident records, and regression guards. Read the relevant section before changing a subsystem.
|
||||
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, Perth, PixelSeal, VideoSeal) cover image, audio, and video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
|
||||
## Data safety
|
||||
|
||||
|
||||
+112
-8
@@ -2,14 +2,14 @@
|
||||
|
||||
`scripts/watermark_benchmark.py` is a development-only runner for reproducible
|
||||
image-, audio-, and video-watermark experiments. It calls the existing local
|
||||
DWT-DCT and TrustMark detectors plus the revision-pinned AudioSeal and
|
||||
VideoSeal oracles through thin adapters. It does not add a runtime command,
|
||||
download a corpus, or contact a provenance oracle or provider API. The
|
||||
DWT-DCT and TrustMark detectors plus the revision-pinned AudioSeal, Perth,
|
||||
PixelSeal, and VideoSeal oracles through thin adapters. It does not add a
|
||||
runtime command, download a corpus, or contact a provenance oracle or provider API. The
|
||||
optional TrustMark package can fetch its official Adobe model weights when its
|
||||
local package cache is incomplete, the audioseal adapter fetches its pinned
|
||||
checkpoints the same way, and the videoseal adapter downloads one pinned
|
||||
TorchScript file into a user cache; prepare those dependencies before an
|
||||
offline run.
|
||||
checkpoints the same way, and the PixelSeal and VideoSeal adapters download
|
||||
pinned weights into a user cache; prepare those dependencies before an offline
|
||||
run. Perth's checkpoint is bundled in its pinned source revision.
|
||||
|
||||
The kernel answers three different questions and never merges their answers:
|
||||
|
||||
@@ -41,9 +41,9 @@ The fixed fields are:
|
||||
| `case_id` | Unique case identity. |
|
||||
| `pair_id` | Groups related clean, marked, attacked, and removed cases. |
|
||||
| `media_type` | `image`, `audio`, or `video` in schema v1. |
|
||||
| `adapter` | `dwt-dct` or `trustmark` for `image`; `audioseal` for `audio`; `videoseal` for `video`. The loader rejects a known adapter named against the wrong media type. |
|
||||
| `adapter` | `dwt-dct`, `trustmark`, or `pixelseal` for `image`; `audioseal` or `perth` for `audio`; `videoseal` for `video`. The loader rejects a known adapter named against the wrong media type. |
|
||||
| `arm` | `positive`, `matched_negative`, `wrong_key`, or `hard_negative`. |
|
||||
| `state` | `clean`, `marked`, `attacked`, `removed`, or `forged`. `forged` names an artifact carrying a watermark with a message different from the adapter oracle's fixed one. Expected detection depends on the adapter: VideoSeal checks the fixed message, while AudioSeal's presence rule can still report `detected` for a foreign message. The study records message accuracy separately. |
|
||||
| `state` | `clean`, `marked`, `attacked`, `removed`, or `forged`. `forged` names an artifact carrying a watermark with a message different from the adapter oracle's fixed one. Expected detection depends on the adapter: PixelSeal and VideoSeal check the fixed message, while AudioSeal's presence rule can still report `detected` for a foreign message. Perth has an implicit fixed mark and no message payload, so it has no meaningful forged arm. The study records message accuracy separately where the scheme exposes a message. |
|
||||
| `path`, `sha256` | Artifact path and pinned content digest. |
|
||||
| `reference_path`, `reference_sha256` | Both strings or both `null`; the fidelity reference. |
|
||||
| `source_revision` | Corpus, generator, or acquisition revision. |
|
||||
@@ -74,6 +74,10 @@ dirty, and SHA-256 digests of the benchmark kernel and adapter source. Detector
|
||||
dependency absence is `unavailable`; an undecodable artifact or adapter
|
||||
exception is `error`. Neither is counted as `not_detected`.
|
||||
|
||||
Message-matched adapters record `detection.message_bit_accuracy`. Score-based
|
||||
presence adapters record `detection.score`. A missing field means the adapter
|
||||
does not expose that quantity; it is not a numeric zero.
|
||||
|
||||
`detection.adapter_elapsed_ms` measures the wall time of the named detector
|
||||
adapter call only. Image decoding and fidelity calculation are outside that
|
||||
interval. The value is `null` when the adapter was unavailable or the artifact
|
||||
@@ -111,6 +115,21 @@ as a binary string, so a case whose payload differs from the embedded one is
|
||||
still `detected` with a different label, never `not_detected`. A `wrong_key`
|
||||
arm for this adapter is therefore a different-message row, not a clean row.
|
||||
|
||||
The `perth` adapter reads `scripts/perth_oracle.py`, which pins Resemble AI's
|
||||
[Perth source](https://github.com/resemble-ai/Perth/tree/ff1c8ac55a976971245cdd53c18d6131ca00d993)
|
||||
at commit `ff1c8ac` and verifies the bundled
|
||||
`perth_net_250000.pth.tar` checkpoint as
|
||||
`sha256:a15bce457ebc53ce5e6c9c3f11df78cf7ee2bf9cdab0a798902135b4c4027670`.
|
||||
The upstream package name is `resemble-perth`, the pinned source declares
|
||||
version 1.1.0, and its [license is MIT](https://github.com/resemble-ai/Perth/blob/ff1c8ac55a976971245cdd53c18d6131ca00d993/LICENSE).
|
||||
That permits a public, development-only oracle. The published 1.0.1 wheel
|
||||
pins an obsolete Torch stack, so the benchmark installs the exact source
|
||||
revision without adding it to project or runtime dependencies. Perth v1 is an
|
||||
implicit presence mark: the detector exposes one confidence and upstream's
|
||||
strictly-above-0.5 decision rule, but no user message. The kernel therefore
|
||||
records `score`, uses `label: perth-v1` only above threshold, and leaves
|
||||
message accuracy not applicable.
|
||||
|
||||
Audio fidelity compares the decoded sample arrays against the explicit
|
||||
reference: identical decoded samples produce `snr_db: null` with
|
||||
`snr_status: unbounded_identical`, never a misleading numeric zero; a silent
|
||||
@@ -118,6 +137,91 @@ reference reports `zero_reference_power`; a length mismatch stays an explicit
|
||||
`incomparable` state. SNR here is reference signal power over error power, the
|
||||
sample-domain analogue of the image PSNR contract.
|
||||
|
||||
## Perth and PixelSeal local oracle study
|
||||
|
||||
`scripts/perth_pixelseal_study.py` builds a local-only study from three
|
||||
deterministic synthetic audio carriers, three macOS system-voice speech
|
||||
carriers, and two publication-cleared tracked image carriers. It never calls a
|
||||
provider oracle or uploads an artifact. Generated media, the strict manifest,
|
||||
and results stay outside git.
|
||||
|
||||
PixelSeal comes from Meta's
|
||||
[VideoSeal source tree](https://github.com/facebookresearch/videoseal/tree/870ca7fb33578b90f14c602016b6c2788096226e),
|
||||
package `videoseal`, at the same exact `870ca7f` revision as the VideoSeal
|
||||
oracle. The project and weights are
|
||||
[MIT licensed](https://github.com/facebookresearch/videoseal/blob/870ca7fb33578b90f14c602016b6c2788096226e/LICENSE).
|
||||
The adapter verifies the official
|
||||
[PixelSeal model card](https://github.com/facebookresearch/videoseal/blob/870ca7fb33578b90f14c602016b6c2788096226e/videoseal/cards/pixelseal.yaml)
|
||||
as `sha256:96c4a448f7d81810780e5bde5a51b039db544b7ec2baf65a605302c1b17362c9`
|
||||
and its 1,237,429,197-byte `checkpoint.pth` as
|
||||
`sha256:0c5665cff20eb6ce1b5aaa7d91c19dafb418bfee32d02dd3344e4ed60d9d75bd`.
|
||||
The source metadata declares package version 1.0. The 1.0.1 PyPI wheel omits
|
||||
the PixelSeal card, so this development oracle likewise requires the exact
|
||||
source revision and remains outside installed CLI and library paths. Its
|
||||
explicit matched-message decision is 256-bit accuracy at or above 0.9, aligned
|
||||
with the VideoSeal adapter; this is the benchmark's rule, not a general
|
||||
watermark-presence classifier.
|
||||
|
||||
```bash
|
||||
uv run python scripts/perth_pixelseal_study.py \
|
||||
--output-dir .local-eval/perth-pixelseal-2026-09-28-v4
|
||||
```
|
||||
|
||||
### Development baseline, 2026-09-28
|
||||
|
||||
The final local run used recipe
|
||||
`sha256:b43953678294d97a72bb323623797ec8d7b8e476b101485e9ecfefbe02353c0e`,
|
||||
kernel
|
||||
`sha256:26dce20f42f7131daa4885eed073baf7372ebb07c0e8ccb4ec50cd603da082bf`,
|
||||
Perth oracle
|
||||
`sha256:a99b9b64a7667b2ee46ac8b57e7c29eec6b13646e3297856e67816984f1a3a9c`,
|
||||
and PixelSeal oracle
|
||||
`sha256:fc15c444e21404c1413d94251e7746964a347d9596faecfdd94b5e45ab2d0949`.
|
||||
Its manifest is
|
||||
`sha256:ecdcca929a73b127db3682636f8a2322fc128abb632f3b779e415a7130e0150d`,
|
||||
case rows are
|
||||
`sha256:ec0a794f5a6580e7bc50e122e1ebae0ea193c9b35b69c7408126d00304c45b38`,
|
||||
and study metadata is
|
||||
`sha256:5906d8ef48132f0fb371e052870a67a682d4b85c7c5c0e23730a951dca970cc1`.
|
||||
All asserted rows matched their expected result; unresolved synthetic Perth
|
||||
positives remain measurements rather than hidden failures.
|
||||
|
||||
| Adapter and state | Cases | Detected | Not detected | Message accuracy or score |
|
||||
| --- | ---: | ---: | ---: | --- |
|
||||
| PixelSeal clean | 2 | 0 | 2 | 0.473-0.504 fixed-message accuracy |
|
||||
| PixelSeal marked | 2 | 2 | 0 | 1.000 on both |
|
||||
| PixelSeal JPEG q90 | 2 | 2 | 0 | 0.996-1.000 |
|
||||
| PixelSeal foreign-message forged | 2 | 0 | 2 | 0.469-0.473 against the fixed message |
|
||||
| Perth speech clean | 3 | 0 | 3 | 0-0.016 confidence |
|
||||
| Perth speech marked | 3 | 3 | 0 | 1.000 confidence |
|
||||
| Perth synthetic clean | 3 | 0 | 3 | 0-0.000083 confidence |
|
||||
| Perth synthetic marked | 3 | 1 | 2 | 0.190, 0.246, and 1.000 confidence |
|
||||
|
||||
PixelSeal embedding measured 46.15-49.46 dB PSNR against the two clean
|
||||
carriers. JPEG q90 retained the mark at 37.19-39.87 dB PSNR. The fixed-message
|
||||
verifier rejected both images carrying the foreign message; that proves only
|
||||
matched-key rejection, not absence of a PixelSeal payload.
|
||||
|
||||
Perth was reliable on the in-domain speech sample but carrier-dependent on the
|
||||
deliberately out-of-domain synthetic set: only white noise crossed the rule,
|
||||
while the tone stack and pinkish noise remained at 0.190 and 0.246. The three
|
||||
synthetic marked rows measured 12.91-14.98 dB SNR. Perth's internal 16 to 32 to
|
||||
16 kHz resampling shortened the three marked speech outputs by 53-104 samples,
|
||||
so the kernel correctly records their SNR as `incomparable` rather than
|
||||
silently trimming them.
|
||||
|
||||
The production visible-video cleaning path removed the visible mark from all
|
||||
24 frames while copying the marked AAC packets byte-for-byte. Both the source
|
||||
and cleaned audio scored 1.000, and the source and output packet streams share
|
||||
`sha256:d6be870baea63ef8dceea97b2c04b27a1a35ba3bd1500cd022ccb167fd0795c6`.
|
||||
This directly confirms that the current video pipeline does not affect Perth.
|
||||
|
||||
PixelSeal removal was not run on this Apple Silicon host. The applicable
|
||||
`qwen-zimage`, `sdxl-zimage`, and `chroma-zimage` invisible-removal profiles
|
||||
require an NVIDIA CUDA device; MPS is not a supported execution path. The
|
||||
missing removed-state rows are therefore an explicit hardware gap, not a clean
|
||||
or failed result.
|
||||
|
||||
## Audio cohort v1
|
||||
|
||||
`scripts/watermark_benchmark_audio_cohort.py` builds the audio analogue of the
|
||||
|
||||
@@ -712,3 +712,102 @@ configuration, since `qwen-zimage` includes its source-based face stage and
|
||||
klein ran global-only; the CJK loss is not. Outputs, reports and
|
||||
oracle records: `.local-eval/flux2-klein-2026-09-27/`.
|
||||
|
||||
## Z-Image-Turbo as a global stage (2026-09-29)
|
||||
|
||||
Verdict: **do not replace `qwen-zimage` with Z-Image-Turbo globally**. It is a
|
||||
promising research direction at the lower OpenAI no-face operating point, but
|
||||
it is not ready as a general no-face complement: the Google 0.40 outputs have
|
||||
visible non-face defects. A face detector alone is not a sufficient safety
|
||||
gate. The narrower route needs a completed Google boundary and a broader
|
||||
content-fidelity study before it can ship. **Decision: do not integrate it; keep
|
||||
the harness and measurements as research evidence only.** This decision is
|
||||
limited to a new global or mflux profile. It does not remove or change the
|
||||
existing source-conditioned Z-Image face-repair stage shared by
|
||||
`qwen-zimage`, `sdxl-zimage`, and `chroma-zimage`.
|
||||
|
||||
The package already uses `Tongyi-MAI/Z-Image-Turbo` through DiffSynth's
|
||||
`ZImagePipeline` for source-conditioned face-crop repair after the qwen global
|
||||
stage. This experiment instead ran the same official Apache-2.0 checkpoint at
|
||||
revision `f332072aa78be7aecdf3ee76d5c247082da564a6` through diffusers 0.40.0
|
||||
`ZImageImg2ImgPipeline`. The stock diffusers path supports strength-controlled
|
||||
img2img and uses the Z-Image scheduler's static shift of 3; unlike klein
|
||||
inpainting, it does not re-inject clean source latents as reference tokens.
|
||||
Primary sources: the
|
||||
[checkpoint](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) and
|
||||
[diffusers pipeline documentation](https://huggingface.co/docs/diffusers/api/pipelines/z_image).
|
||||
|
||||
The global-only H100 ladder used seed 0, the neutral scrub prompt, guidance
|
||||
0.0, eight effective distilled steps, and strengths 0.025, 0.05, 0.10, 0.15,
|
||||
0.20, 0.30, and 0.40 on the same seven tracked originals as the klein study.
|
||||
The real shifted start sigmas were 0.0714, 0.1364, 0.25, 0.375, 0.4286, 0.60,
|
||||
and 0.6667. The 49 generations took 264.5 seconds of inference in total,
|
||||
about 2.4 seconds per OpenAI image and 7.6 seconds per larger Google image.
|
||||
The rough Modal H100 cost was $0.29 for inference or $0.40 including the cold
|
||||
model load. Harness: `z_image_turbo_ladder.py`, kept outside this repository.
|
||||
|
||||
The official OpenAI oracle gave first-clean strengths 0.10, 0.10, and 0.15.
|
||||
The project margin rule therefore selects 0.20, versus the shipped
|
||||
`qwen-zimage` floor of 0.15625. At those respective operating points,
|
||||
`scripts/fidelity_metrics.py` found:
|
||||
|
||||
| Carrier | `qwen-zimage` | Z-Image-Turbo |
|
||||
| --- | --- | --- |
|
||||
| 9-face grid | ID 0.853, face LPIPS 0.054, image LPIPS 0.091 | ID 0.658, face LPIPS 0.092, image LPIPS 0.101 |
|
||||
| Typography 1 | NED 0.545, image LPIPS 0.123, PSNR 22.5 dB | NED 0.311, image LPIPS 0.086, PSNR 27.3 dB |
|
||||
| Typography 2 | NED 0.562, image LPIPS 0.116, PSNR 24.0 dB | NED 0.291, image LPIPS 0.074, PSNR 27.7 dB |
|
||||
|
||||
Z-Image-Turbo preserves the two no-face typography cards better on every
|
||||
listed metric, but the face-grid identity loss is materially worse. That is
|
||||
enough to reject it as the universal OpenAI stage.
|
||||
|
||||
Google bracketing is incomplete on one carrier. The party and crowd images
|
||||
were detected at 0.15 and clean at 0.20. The Cyrillic sign was detected at
|
||||
0.20 and clean at 0.30. The CJK sign remained detected through 0.15; its 0.20
|
||||
checks produced two explicit quota failures, and the remaining authenticated
|
||||
surface could not accept the prepared upload while its activity setting was
|
||||
off. Those are unreachable results, not clean verdicts. If that carrier first
|
||||
cleans at 0.20 or 0.30, the margin rule selects 0.40; if it first cleans at
|
||||
0.40, the rule selects 0.60. Consequently, 0.40 is only a provisional fidelity
|
||||
probe, not an oracle-established Google operating point.
|
||||
|
||||
Against `qwen-zimage` at 0.35, that provisional 0.40 probe shows the same
|
||||
content split as OpenAI:
|
||||
|
||||
| Carrier | Faces | `qwen-zimage` | Z-Image-Turbo |
|
||||
| --- | ---: | --- | --- |
|
||||
| Party scene | 5 | ID 0.933, NED 0.386, image LPIPS 0.367 | ID 0.236, NED 0.286, image LPIPS 0.326 |
|
||||
| Crowd | 18 | ID 0.752, image LPIPS 0.339 | ID 0.448, image LPIPS 0.297 |
|
||||
| Cyrillic sign | 0 | NED 0.367, LPIPS 0.424, SSIM 0.523 | NED 0.245, LPIPS 0.356, SSIM 0.547 |
|
||||
| CJK sign | 0 | NED 0.074, LPIPS 0.380, SSIM 0.555 | NED 0.000, LPIPS 0.320, SSIM 0.587 |
|
||||
|
||||
Visual inspection agrees with the metrics. At 0.40 Z-Image-Turbo changes age,
|
||||
facial structure, and skin texture and can introduce doubled facial contours.
|
||||
The no-face images are not safe either. The CJK output has NED 0.000, but that
|
||||
only says OCR recovered the same character sequence: the model visibly redraws
|
||||
the path, foliage, lighting, sign geometry, wood grain, post, and fasteners.
|
||||
Both engines also make visible Cyrillic errors. The package's existing Z-Image
|
||||
face stage does not contradict this result. It takes an expanded crop from the
|
||||
original image, enlarges the detected face toward 768 px, runs DiffSynth at an
|
||||
adaptive strength of only 0.025-0.14, and composites only the SAM face mask into
|
||||
the already-clean global result. The global experiment instead ran diffusers at
|
||||
up to 0.40 over the full frame, where each face was small and every pixel could
|
||||
be redrawn. The face stage also does not have to remove SynthID: a matched
|
||||
isolation test found its output alone still detected, while adding it after the
|
||||
clean Qwen pass improved identity cosine from 0.589 to 0.852. It is a low-noise
|
||||
masked fidelity repair, not a watermark-removal stage.
|
||||
|
||||
As an Apple Silicon side probe, [mflux](https://github.com/mflux-community/mflux)
|
||||
0.20.0 successfully ran img2img with
|
||||
`filipstrand/Z-Image-Turbo-mflux-4bit` on an M5 with 32 GB. At 1440x1072,
|
||||
image strength 0.8 (equivalent denoise fraction 0.20), 40 requested steps and
|
||||
eight effective steps, model load took 5.3 seconds and two deterministic runs
|
||||
took 301.5 and 380.0 seconds. The pixel outputs were identical, but there was
|
||||
no warm-run speedup. This is slow for an interactive default, but it is also the
|
||||
only measured native Apple Silicon image-regeneration path: every shipped
|
||||
invisible-image profile is CUDA-only and refuses MPS or CPU. The mflux route
|
||||
therefore proves M5 feasibility, but the visible defects at the tested Google
|
||||
strength keep it from being a usable fallback. Shipping it would require a
|
||||
lower safe operating point, the missing Google boundary, broader content
|
||||
fidelity, packaging, and product-level tests. Outputs, fidelity logs,
|
||||
manifests, and the local timing report are under
|
||||
`.local-eval/z-image-turbo-2026-09-27/`.
|
||||
@@ -56,6 +56,7 @@ The current behavior is defined by the code, tests, README, and user guides.
|
||||
- [Text protection research](text-protection-research.md)
|
||||
- [Audio provenance experiment](audio-provenance-experiment.md) (local AudioSeal matched-oracle study of video audio tracks across visual cleaning)
|
||||
- [VideoSeal temporal evaluation](videoseal-temporal-evaluation.md) (per-frame scores, aggregation matrix, and crf decay on synthetic and real clips)
|
||||
- [Temporal video inpainting evaluation](video-inpainting-research.md) (VACE and ROSE against the shipped visible-video fills)
|
||||
- [Watermark forgery study](watermark-forgery-study.md) (four-state model and double-embedding: the last writer wins under both oracles)
|
||||
- [C2PA resolution research](c2pa-resolution-research.md) (measured resolver contracts, the privacy gradient, and the decision to never resolve over the network)
|
||||
- [External benchmark licensing](external-benchmark-licensing.md) (ETI license-blocked, W-Bench dataset MIT with non-commercial code, verified 2026-09-07)
|
||||
|
||||
@@ -164,9 +164,16 @@ faces. The general fidelity corpus and its OCR caveats are tracked in
|
||||
[`data/evaluations/fidelity/`](../data/evaluations/fidelity/README.md), while the
|
||||
current-floor Google replication is recorded in
|
||||
[`module-internals.md`](module-internals.md#google-boundary-replication-2026-09-15).
|
||||
A global Z-Image Turbo prototype preserved text substantially better at low
|
||||
strength, but it has no useful cross-provider operating point and is not a
|
||||
supported profile. Automatic text restorers also remain research-only:
|
||||
A global Z-Image Turbo prototype preserved text substantially better on the
|
||||
no-face carriers but lost face identity, so it is not a universal replacement
|
||||
and is not a supported profile. Its mflux arm is the only measured native Apple
|
||||
Silicon image-regeneration path; it ran on an M5 with 32 GB but took 301-380
|
||||
seconds per image. The Google no-face outputs at 0.40 visibly redraw scene and
|
||||
object details despite better OCR metrics, so absence of faces is not a
|
||||
sufficient safety gate. The mflux result proves technical feasibility rather
|
||||
than a usable fallback; it still lacks one Google boundary, a lower safe
|
||||
operating point, and broad content-fidelity validation. Automatic text
|
||||
restorers also remain research-only:
|
||||
fresh-font and silhouette variants visibly changed typography. The higher-fidelity
|
||||
`vae-glyphs` route is available only as an experimental opt-in with verified strings
|
||||
and line geometry. It builds its donor internally but still requires an independently
|
||||
@@ -265,9 +272,12 @@ A local matched-oracle study has now measured the boundary concretely:
|
||||
into synthetic carriers, runs the visible video-cleaning path, and shows the
|
||||
audio packets byte-identical and the watermark verdict invariant across
|
||||
cleaning, while additive noise on the audio path is the effective lever. The
|
||||
boundary above still stands for provider marks: this tool still does not
|
||||
decode them, and a `copied_if_present` audio track remains `unverified` in
|
||||
runtime output.
|
||||
[Perth and PixelSeal benchmark study](benchmark-kernel.md#perth-and-pixelseal-local-oracle-study)
|
||||
independently repeats the pass-through check with Resemble Perth: all 24 visual
|
||||
frames are cleaned, the marked AAC packets remain byte-identical, and Perth's
|
||||
score stays 1.0. The boundary above still stands for provider marks: this tool
|
||||
still does not decode them, and a `copied_if_present` audio track remains
|
||||
`unverified` in runtime output.
|
||||
|
||||
The shipped engine streams sampled frames in bounded batches, computes its
|
||||
fidelity metrics incrementally, and pipes regenerated pixels directly to
|
||||
|
||||
@@ -472,6 +472,14 @@ backgrounds need MI-GAN or LaMa for better reconstruction. Invisible video
|
||||
stages must continue to reuse the image and metadata implementations rather
|
||||
than copying their logic.
|
||||
|
||||
Apache-2.0 temporal video inpainting was measured on 2026-09-27 and is not a
|
||||
backend: VACE and ROSE had higher masked frame-to-frame error than MI-GAN on
|
||||
all three publication-cleared Sora, Veo, and Kling clips. VACE left a gray
|
||||
block on Veo, while ROSE visibly reproduced the wordmark and made the Veo
|
||||
detector accept 17/17 output frames. The model pins, mask-conditioning audit,
|
||||
decoded metrics, H100 cost, and frame review are in the
|
||||
[temporal video inpainting evaluation](video-inpainting-research.md).
|
||||
|
||||
Regression coverage:
|
||||
|
||||
- [`test_video.py`](../tests/test_video.py), including a real ffmpeg full-clip
|
||||
|
||||
@@ -327,6 +327,17 @@ byte for byte; the manifest explicitly records that their metadata was not
|
||||
stripped. Use the matched-control protocols in [`synthid.md`](synthid.md) for
|
||||
video claims.
|
||||
|
||||
Before preparing any image for a SynthID oracle, visually inspect it and remove
|
||||
every visible AI-provider logo or label with the project's visible-mark remover.
|
||||
Derive both the positive control and every processed candidate from that same
|
||||
visible-clean source, then run `prepare` to strip metadata losslessly. Inspect the
|
||||
prepared upload again and refuse the request if a visible provider mark remains.
|
||||
This separation is mandatory: Gemini can answer from a visible sparkle or from
|
||||
image-content reasoning without returning a SynthID tool verdict. Record such an
|
||||
answer as `indeterminate`, never `detected`. The source-to-visible-clean pixel
|
||||
change belongs in the experiment report; `prepare`'s pixel-preservation claim is
|
||||
only from that cleaned source to the metadata-stripped upload.
|
||||
|
||||
The batch contains:
|
||||
|
||||
- `manifest.json`, immutable surface, slot, source, upload, and hash identities;
|
||||
@@ -369,6 +380,13 @@ requires the user's existing authenticated Chrome session. Invoke the project
|
||||
profile and `/u/N/` Google account, upload each prepared artifact, capture the
|
||||
settled SynthID tool outcome, and record it.
|
||||
|
||||
Ask the direct question stored in the batch manifest: `Does this
|
||||
image/video/audio contain a SynthID watermark? Use the SynthID verification
|
||||
tool only; do not infer from visible content, logos, text, or metadata.` A
|
||||
general question about whether Google AI created or edited the media is not
|
||||
equivalent: it may elicit origin or visual-content reasoning without a SynthID
|
||||
verdict.
|
||||
|
||||
OpenAI Web, Microsoft Web, and Meta run entirely through a fresh isolated
|
||||
Playwright browser. A direct-route example:
|
||||
|
||||
|
||||
@@ -65,13 +65,51 @@ This page is an index. Each finding lives in its canonical page.
|
||||
them it clears both vendors only at lower face and CJK-text fidelity than
|
||||
`qwen-zimage`
|
||||
([Chroma1 research](chroma1-engine-research.md#flux2-klein-as-a-global-stage-2026-09-27)).
|
||||
Still open:
|
||||
Z-Image-Turbo as a global stage (with an Apple Silicon arm through mflux),
|
||||
linear probes on newer frozen backbones for `classify`, Resemble Perth and
|
||||
Meta PixelSeal as benchmark oracles, re-watermarking as a removal primitive
|
||||
(arXiv:2605.16796, MIT code), RAVEN view-synthesis removal
|
||||
(arXiv:2601.08832, no code released as of 2026-09-24), and Apache-2.0 video
|
||||
inpainting (VACE, ROSE) against the per-frame visible video fill.
|
||||
Apache-2.0 temporal video inpainting was also measured: VACE and ROSE were
|
||||
less stable than MI-GAN on all three Sora, Veo, and Kling clips, VACE left a
|
||||
gray block on Veo, and ROSE retained the Veo wordmark on 17/17 frames. They
|
||||
are not shipped
|
||||
([video inpainting research](video-inpainting-research.md)).
|
||||
Resemble Perth and Meta PixelSeal were added as exact-revision, MIT-licensed
|
||||
local benchmark oracles. The 2026-09-28 study found Perth on all three speech
|
||||
positives and confirmed that visible-video cleaning copies its marked AAC
|
||||
packets unchanged; PixelSeal survived JPEG q90 on both cleared image
|
||||
carriers, while its CUDA-only removal profiles could not run on this host
|
||||
([benchmark study](benchmark-kernel.md#perth-and-pixelseal-local-oracle-study)).
|
||||
Re-watermarking (arXiv:2605.16796) was reproduced locally on 2026-09-27:
|
||||
same-encoder message B replaced A on all 12/12 valid TrustMark and 12/12
|
||||
VideoSeal image controls, both VideoSeal video clips, and all 6/6 valid
|
||||
controls in each DWT-DCT direction. Median image clean-to-output PSNR was
|
||||
39.73-44.15 dB; the DWT-DCT controls worked on only half the carriers.
|
||||
This is an open-encoder result, not a SynthID result
|
||||
([removal research](synthid-removal-research.md#re-watermarking-on-open-encoders-2026-09-27)).
|
||||
A one-carrier production probe subsequently kept OpenAI SynthID `detected`
|
||||
on both its metadata-free control and 43.04 dB VideoSeal-overlaid image;
|
||||
a visible-clean, metadata-free Google control and its 42.86 dB VideoSeal
|
||||
overlay also remained `detected` under direct SynthID-tool checks. The Google
|
||||
control and candidate positives came from different approved account slots,
|
||||
so this remains a one-carrier coexistence result
|
||||
([removal research](synthid-removal-research.md#production-synthid-overmark-probe-2026-09-27-local)).
|
||||
Z-Image-Turbo was measured on 2026-09-29. It is not a global replacement:
|
||||
at its OpenAI margin point it improves both no-face typography carriers but
|
||||
loses face identity, and the same face/text split is stronger in the Google
|
||||
fidelity probe. The Google no-face outputs at 0.40 also visibly redraw scene
|
||||
geometry and object detail, despite better OCR and aggregate metrics, so a
|
||||
face detector alone cannot make a safe complement. The last Google boundary
|
||||
is unresolved because the available oracle checks were quota-blocked or
|
||||
could not accept the prepared upload.
|
||||
The mflux arm works on an M5 with 32 GB and is the only measured native Apple
|
||||
Silicon image-regeneration path, since every shipped image profile is
|
||||
CUDA-only. It took 301-380 seconds per image. This proves M5 feasibility but
|
||||
is not a usable fallback at the measured quality
|
||||
([Chroma1 research](chroma1-engine-research.md#z-image-turbo-as-a-global-stage-2026-09-29)).
|
||||
The experiment is closed with a decision not to add a global or mflux
|
||||
profile; retain its harness and local evidence for future comparison only.
|
||||
The existing source-conditioned Z-Image face-repair stage remains part of
|
||||
`qwen-zimage`, `sdxl-zimage`, and `chroma-zimage`.
|
||||
Still open: linear probes on newer frozen backbones for `classify` and RAVEN
|
||||
view-synthesis removal
|
||||
(arXiv:2601.08832, no code released as of 2026-09-24).
|
||||
MiniMax-Remover (CC-BY-NC-4.0 weights) and ProPainter (S-Lab License,
|
||||
non-commercial) are excluded on license.
|
||||
- **New C2PA signers without a vendor row.** The
|
||||
|
||||
@@ -79,6 +79,9 @@ independent frame fill.
|
||||
| `auto` | Depends on installed extras | Selects LaMa, then MI-GAN, then OpenCV |
|
||||
|
||||
The learned backends download model files on first use.
|
||||
[VACE and ROSE were evaluated](video-inpainting-research.md) on the same
|
||||
production masks and were not adopted: neither improved temporal stability
|
||||
over MI-GAN, and ROSE retained the Veo mark.
|
||||
|
||||
## Metadata and provenance
|
||||
|
||||
|
||||
@@ -228,19 +228,21 @@ regeneration is already in [synthid.md](synthid.md#23-removal-attacks-and-forens
|
||||
| Cao et al., [arXiv:2608.10166](https://arxiv.org/abs/2608.10166) (MarkNull, USENIX Security 2026) | On-manifold latent decorrelation via a public diffusion proxy. Claims 100% on 20 Imagen-3 Gemini-verify images. PSNR 25.36 dB, SSIM 0.80 | Small Gemini-verify set | Independent evidence that a no-box latent reconstruction can confuse Gemini. Does not meet this project's 40 dB / 0.99 SSIM release gate. Still generation, not a pixel-only wipe |
|
||||
| Goonatilake and Ateniese, [arXiv:2605.09203](https://arxiv.org/abs/2605.09203) | Six removers all leave a forensic residue a ResNet-50 sees at >98% TPR @ 1% FPR | Applies to UnMarker, Zhao's WatermarkAttacker, CtrlRegen+ | Defeating a provider oracle is not deniability. This is the product remainder |
|
||||
| An et al., [arXiv:2401.08573](https://arxiv.org/abs/2401.08573) (WAVES, ICML 2024) | 26 attacks on StegaStamp, Stable Signature, Tree-Ring. Regeneration, not JPEG, is the attack that matters. StegaStamp TPR@1%FPR 1.00 to 0.01; Tree-Ring 0.99 to 0.12 | Open watermarks | Protocol. Our blur-sigma-7 and 16-32 scramble are closer to WAVES "distortion" than to regeneration |
|
||||
| Bulychev et al., [arXiv:2605.16796](https://arxiv.org/abs/2605.16796) (Watermarks Attack Watermarks) | Apply a second watermark, usually with the victim's own post-processing encoder and a different message. On MS-COCO, its end-to-end pipeline lowers Video Seal victim bit accuracy from 0.998 to 0.520 (TPR@1%FPR 0.046) and Pixel Seal from 0.999 to 0.514 (TPR 0.029); the same-method attacker message recovers at 0.99 bit accuracy for both schemes. Quality is reported as an eight-metric normalized composite, not a raw PSNR operating point | Eight open image schemes, not production SynthID; Video Seal is tested on images | Directly motivated the [local open-encoder replication](#re-watermarking-on-open-encoders-2026-09-27). Its MIT [code](https://github.com/MariaBulychev/Watermarks-Attack-Watermarks/tree/0444f52ad972da9e009555f2dc1028d9abe0610e) calls external encoder packages and weights; the top-level license does not license those dependencies |
|
||||
| Wen et al., [arXiv:2305.20030](https://arxiv.org/abs/2305.20030) (Tree-Ring) | In-generation Fourier pattern in initial noise | Not SynthID | Kill is DDIM inversion plus latent wipe, not a pixel scramble. Lin and Juarez [arXiv:2506.10502](https://arxiv.org/abs/2506.10502) (USENIX 2025) remove it from public knowledge |
|
||||
| Fernandez et al., Stable Signature, ICCV 2023 | Fine-tuned VAE decoder | Not SynthID | Regeneration that replaces the decoder is in-family. Our foreign VAE at 22.3 dB did not kill OpenAI SynthID, which is the expected mismatch |
|
||||
| Google to [The Verge, 2026-04-14](https://www.theverge.com/ai-artificial-intelligence/911579/google-synthid-ai-watermarking-system-reverse-engineered) | reverse-SynthID does not systematically remove SynthID | Google spokesperson on the spectral bypass | Agrees with our V4 audit and with Bypass: reconstruction or decoder confusion, not a payload delete |
|
||||
|
||||
Quiet pixel-only removal is not a published result on production
|
||||
OpenAI or Google SynthID (still true in the 2026-09-22 literature sweep).
|
||||
Against open encoders the fidelity band has moved: re-watermarking
|
||||
([arXiv:2605.16796](https://arxiv.org/abs/2605.16796)), WmForger
|
||||
OpenAI or Google SynthID (still true in the 2026-09-27 literature sweep).
|
||||
Against open encoders the fidelity band has moved: WmForger
|
||||
([arXiv:2510.20468](https://arxiv.org/abs/2510.20468)) and Hide&Seek
|
||||
([arXiv:2603.01067](https://arxiv.org/abs/2603.01067)) report roughly 30 dB,
|
||||
and SPFM-Net and FDDWAN (arXiv:2607.27811, 2607.27800) claim 40-45 dB, but on
|
||||
small images and with bit error rates that leave much of the payload intact.
|
||||
None of these is a result on a production closed watermark.
|
||||
The re-watermarking paper reports normalized quality degradation, not a raw
|
||||
PSNR aggregate; our separate local measurements are below. None of these is a
|
||||
result on a production closed watermark.
|
||||
|
||||
## Product remainder for removal
|
||||
|
||||
@@ -248,6 +250,146 @@ Invisible removal in this package regenerates through `qwen-zimage` or
|
||||
`sdxl-zimage`. It is lossy. There is no shipped pixel-only OpenAI or
|
||||
Google SynthID wipe. Do not add Bayer or geometry as remover arms.
|
||||
|
||||
### Re-watermarking on open encoders (2026-09-27)
|
||||
|
||||
The CC BY 4.0 [paper](https://arxiv.org/abs/2605.16796), v1 submitted on
|
||||
2026-05-16, tests eight open image watermarks: Stable Signature and Tree-Ring
|
||||
(in-generation), and StegaStamp, RoSteALS, ZoDiac, Pixel Seal, WAM, and Video
|
||||
Seal (post-processing). Its policy
|
||||
reapplies the victim encoder with a different message for post-processing
|
||||
marks and uses ZoDiac against in-generation marks. Its Video Seal evaluation
|
||||
is on images, not video. The [MIT repository](https://github.com/MariaBulychev/Watermarks-Attack-Watermarks/tree/0444f52ad972da9e009555f2dc1028d9abe0610e)
|
||||
is pinned at `0444f52` and imports separate upstream encoders. We reproduced
|
||||
the same-encoder, different-message operation with the project's pinned
|
||||
oracles, not the paper's full WAVES dataset, classifier, or exact dependency
|
||||
stack. TrustMark and DWT-DCT are project extensions, not paper subjects.
|
||||
|
||||
Twelve publication-cleared, provider-paired images from six content strata
|
||||
were fitted to 512 x 512 pixels. Message A was embedded and serialized to PNG;
|
||||
message B was then embedded into that decoded artifact and serialized again.
|
||||
All decoder and fidelity readings used the saved RGB bytes. A successful case
|
||||
requires a positive A control, an A-negative final verdict, and a B-positive
|
||||
final verdict under that scheme's pinned rule. The table reports mean final
|
||||
bit accuracy and median clean-to-final PSNR over valid A controls only.
|
||||
|
||||
| Encoder and direction | Valid A controls / all | B replaces A / valid | Final accuracy A / B | Median total PSNR |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| DWT-DCT, FLUX codeword to SDXL codeword | 6 / 12 | 6 / 6 | 0.563 / 1.000 | 39.80 dB |
|
||||
| DWT-DCT, SDXL codeword to FLUX codeword | 6 / 12 | 6 / 6 | 0.566 / 0.997 | 39.73 dB |
|
||||
| TrustMark P, 61-bit messages | 12 / 12 | 12 / 12 | 0.541 / 1.000 | 44.15 dB |
|
||||
| VideoSeal image mode, 256-bit messages | 12 / 12 | 12 / 12 | 0.479 / 0.990 | 43.89 dB |
|
||||
|
||||
The DWT-DCT codewords differ at only 21 of 48 bits, so A accuracy near 0.56
|
||||
after B is compatible with complete replacement; it is not evidence of a
|
||||
surviving A signal. Conversely, its 6/12 A-positive rate in each direction
|
||||
precludes an unconditional success claim. These are fixed messages and a
|
||||
small, deliberately stratified cohort, not a calibrated TPR@1%FPR estimate.
|
||||
The worst-valid-case contact sheet shows no obvious coarse artifacts at
|
||||
256-pixel viewing size, but PSNR and that sheet do not establish full-resolution
|
||||
perceptual, face, or text fidelity.
|
||||
|
||||
The [corrected forgery study](watermark-forgery-study.md#corrected-videoseal-measurements-2026-09-08)
|
||||
already showed VideoSeal replacement on saved video. A fresh run added
|
||||
decoded-artifact fidelity on two 64-frame, 256 x 256, H.264 CRF 8 clips:
|
||||
the synthetic moving gradient went from A accuracy 1.000 to 0.469 and B
|
||||
accuracy 1.000 at 42.88 dB mean clean-to-final PSNR; the publication-cleared
|
||||
Sora clip went from A 1.000 to 0.473 and B 0.996 at 37.60 dB. Both meet the
|
||||
fixed-message replacement rule. This is an extension to video, not a paper
|
||||
replication or a guarantee across codecs and sources.
|
||||
|
||||
`scripts/rewatermarking_study.py` and `scripts/rewatermarking_video_study.py`
|
||||
write case rows, artifact hashes, and reports under the untracked
|
||||
`.local-eval/rewatermarking-2026-09-27/` and
|
||||
`.local-eval/rewatermarking-video-2026-09-27/`. Case-row SHA-256 values are
|
||||
`3270df31320c0c58091f34b9c2e1f4885190fe65577e0dbbe03446b2b8bda890`
|
||||
and `150e6584d5f253e52ddb0024ab9ad17f94b03b64dbfd4c293d57f6c6510607c6`.
|
||||
The retained PNGs and MP4s were independently rehashed and decoded for PSNR
|
||||
verification. Both runs used the local CPU; no Modal GPU was used and there
|
||||
was no cloud GPU charge. This establishes a local open-encoder overwrite
|
||||
primitive, not a production SynthID remover. An open encoder's foreign message
|
||||
may coexist with a secret-keyed provider mark.
|
||||
|
||||
### Production SynthID overmark probe (2026-09-27 local)
|
||||
|
||||
One previously oracle-positive, publication-cleared original per provider
|
||||
was selected from `data/synthid/manifest.csv`. The initial preparation made a
|
||||
native-resolution RGB pixel control and a VideoSeal image-mode 256-bit
|
||||
message-B overlay. It stripped AI metadata without changing decoded pixels,
|
||||
but that rule was insufficient: the Google source retained its visible Gemini
|
||||
sparkle. Its Gemini Web checks therefore cannot serve as pixel-only SynthID
|
||||
positive controls. The OpenAI case had no visible mark; it remains 1122 x 1402
|
||||
at 43.04 dB control-to-overlay PSNR, with B decoded at 1.000. The pair is
|
||||
deliberately small and tests coexistence with an *unrelated* open watermark,
|
||||
not application of a secret SynthID encoder.
|
||||
|
||||
The official OpenAI Content Provenance API reported SynthID `detected` on
|
||||
both the metadata-free pixel control and its VideoSeal overlay, with C2PA
|
||||
`not_present` for each (2026-09-28 UTC). Thus the successful open-encoder
|
||||
overwrite did **not** remove OpenAI SynthID on this carrier.
|
||||
|
||||
The separately approved Google control upload reached Gemini Web's exact
|
||||
`Connecting to Verify AI` state on 2026-09-28 and remained there well beyond
|
||||
the project's documented 90-second indeterminate threshold. A separately
|
||||
approved retry of the same prepared SHA-256 reached that state, then returned
|
||||
only a generic visual-content answer: Gemini attributed the image to Google AI
|
||||
from a visible four-pointed logo and text-rendering characteristics. It exposed
|
||||
no settled SynthID tool outcome. Both observations are watermark
|
||||
`indeterminate` and provenance `unavailable`, not clean or `detected`. The
|
||||
VideoSeal-overlaid Google candidate was not uploaded because the positive
|
||||
control gate was not satisfied. Therefore these runs supply no Google SynthID
|
||||
verdict and do not test re-watermarking against production Google SynthID.
|
||||
|
||||
The preparation protocol now removes detected visible AI marks and AI metadata
|
||||
before deriving either arm. On this Google source, the project-native Gemini
|
||||
remover validated removal of the sparkle and changed 0.213% of pixels. VideoSeal
|
||||
was then applied to that cleaned control: B accuracy is 0.992 and
|
||||
control-to-overlay PSNR is 42.86 dB. Visual inspection found no remaining logo;
|
||||
the local sparkle detector scored the prepared control 0.300, below its 0.35
|
||||
detection threshold. The corrected control SHA-256 is
|
||||
`14ed25fa58a0abc928ea869431248b450892facdf85b4621b3bf959c16a78cf3`;
|
||||
the corrected candidate is
|
||||
`86000c8e143339adde675522dc7709830c8ff230977e276c748cabb5d7cfadb8`.
|
||||
The separately approved corrected-control upload first received the generic
|
||||
question about whether Google AI created or edited the image. Gemini said the
|
||||
amount of text made reliable confirmation difficult and could not confirm its
|
||||
origin. That origin question was the wrong contract for a SynthID experiment;
|
||||
its answer remains `indeterminate`. A confirmed follow-up asked directly
|
||||
whether the same attached image contains a SynthID watermark, required the
|
||||
SynthID verification tool, and prohibited inference from visible content,
|
||||
logos, text, or metadata. Gemini then returned its settled verification-tool
|
||||
positive: the image was created or edited with Google AI, with the Verify AI
|
||||
SynthID help link. The corrected control is therefore watermark `detected` and
|
||||
provenance `unavailable`. The separately approved candidate upload then used
|
||||
the same direct SynthID question on 2026-09-29. Gemini explicitly refused it
|
||||
because the verification-tool quota had been exceeded. That result is
|
||||
watermark `refused` and provenance `unavailable`, not clean, `detected`, or
|
||||
`indeterminate`. With separate approval, the exact same candidate upload was
|
||||
then checked once in the `/u/2` slot. Its settled verification-tool response
|
||||
said the image contained signals indicating Google-AI generation or editing.
|
||||
That batch is watermark `detected` and provenance `unavailable`. Thus the
|
||||
42.86 dB VideoSeal overlay did **not** remove Google SynthID on this carrier.
|
||||
The control and usable candidate verdict came from different authenticated
|
||||
account slots because `/u/1` exhausted its rolling quota; this is a one-carrier
|
||||
coexistence result, not an account-invariance or broad production guarantee.
|
||||
|
||||
Files, hashes, API response payloads, and result verification live under the
|
||||
untracked `.local-eval/rewatermarking-synthid-2026-09-27/`. The OpenAI batch's
|
||||
two prepared upload SHA-256 values are
|
||||
`f79189bcc98c6e2c0bf9737fa033b83870475560f3bff897c6e80508d90a3336`
|
||||
and `6949efa56bbfb285a580994536f8ec623af801187e26ba0f27da895488571340`.
|
||||
That two-result immutable batch passed `verify --require-complete`. The first
|
||||
Google batch passed hash verification with 1/2 rows recorded. A one-row retry
|
||||
batch for the same control passed `verify --require-complete` with one
|
||||
`indeterminate` result. The submitted control SHA-256 in both batches is
|
||||
`fdb34c868e533a2c3847da868c708b0c6bd5e586523ee7e6d7ee1773b321fbc4`.
|
||||
Those batches are retained as evidence of the flawed visible-logo control, not
|
||||
as SynthID controls. The corrected local artifacts and direct-SynthID-question
|
||||
batches live under `.local-eval/rewatermarking-synthid-logo-clean-2026-09-28/`.
|
||||
The control batch's single `detected` record and the candidate batch's single
|
||||
`refused` record in `/u/1`, plus the separately approved `/u/2` candidate
|
||||
batch's single `detected` record, each passed `verify --require-complete`. No
|
||||
cloud GPU was used.
|
||||
|
||||
### Color restoration after regeneration (2026-09-27)
|
||||
|
||||
The NeurIPS 2024 "Erasing the Invisible" winner
|
||||
|
||||
@@ -449,10 +449,12 @@ audio-path attacks separately. Its carriers are synthetic non-speech audio
|
||||
and its oracle is AudioSeal, so it validates the copy semantics, not provider
|
||||
audio marks. The audio and video arms are now first-class benchmark rows:
|
||||
[benchmark kernel](benchmark-kernel.md) accepts `media_type: audio` and
|
||||
`media_type: video` with the revision-pinned `audioseal` and `videoseal`
|
||||
adapters, and both cohorts carry matched negatives, codec, and degradation
|
||||
arms as strict manifest cases. The temporal layer is closed too: videoseal
|
||||
detection records carry per-frame bit accuracy, and
|
||||
`media_type: video` with the revision-pinned `audioseal`, `perth`, and
|
||||
`videoseal` adapters, plus the `pixelseal` image adapter. The Perth and
|
||||
PixelSeal study records matched negatives, marked rows, attacks, fixed-message
|
||||
accuracy, and the copied-audio path as strict manifest cases. The temporal
|
||||
layer is closed too: videoseal detection records carry per-frame bit accuracy,
|
||||
and
|
||||
[VideoSeal temporal evaluation](videoseal-temporal-evaluation.md) measures
|
||||
the aggregation matrix and the crf decay curve on synthetic and real clips.
|
||||
What remains open here is a wider real-content sweep and the learned temporal
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# Temporal video inpainting evaluation
|
||||
|
||||
Development-only study run on 2026-09-27. It asks whether a permissively
|
||||
licensed temporal model improves visible video-mark removal enough to justify
|
||||
an optional backend. The answer on this corpus is no: VACE and ROSE were less
|
||||
temporally stable than the best shipped per-frame backend on every clip, VACE
|
||||
left an obvious block on Veo, and ROSE reproduced the Veo wordmark strongly
|
||||
enough for the production detector to accept all 17 output frames.
|
||||
|
||||
No provider oracle or private input was used. All source clips are tracked,
|
||||
publication-cleared fixtures. Generated videos, masks, reports, and contact
|
||||
sheets remain outside the repository.
|
||||
|
||||
## Models, licenses, and runtime requirements
|
||||
|
||||
| Model | Evaluated code and weights | License evidence | Official inference requirements |
|
||||
| --- | --- | --- | --- |
|
||||
| VACE | [`ali-vilab/VACE`](https://github.com/ali-vilab/VACE) at `48eb44f`, [`Wan-AI/Wan2.1-VACE-1.3B-diffusers`](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) at `ec4d2cb` | Code and 1.3B weights are Apache-2.0. | The upstream repository tested Python 3.10.13, CUDA 12.4, and PyTorch 2.5.1. Its 1.3B profile targets about 81 x 480 x 832. The official [input contract](https://github.com/ali-vilab/VACE/blob/main/UserGuide.md) requires missing pixels to be gray and a separate white-generate/black-retain mask. |
|
||||
| ROSE | [`Kunbyte-AI/ROSE`](https://github.com/Kunbyte-AI/ROSE) at `6be41c5`, [`Kunbyte/ROSE`](https://huggingface.co/Kunbyte/ROSE) at `8a5e57c`, with [`alibaba-pai/Wan2.1-Fun-1.3B-InP`](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) at `9bcbbcf` | The top-level repository license, tuned transformer, and base model declare Apache-2.0. The separate `hugging_face/` demo vendors/imports ProPainter code and was excluded; a product package would have to keep that non-commercial path out. | The upstream recipe uses Python 3.12, CUDA 12.4, PyTorch 2.6.0, and torchvision 0.21.0. It requires `16n+1` frames, defaults to 480 x 720 and 50 steps, and combines the tuned 1.3B transformer with the Wan2.1-Fun base model. |
|
||||
|
||||
MiniMax-Remover and ProPainter were not evaluated because their weights or
|
||||
code are non-commercial. ROSE's core `inference.py` imports only the `rose/`
|
||||
Wan2.1-Fun path used here; the ProPainter-derived interactive demo is a
|
||||
separate license boundary, not evidence that the complete checkout is safe to
|
||||
redistribute as one Apache-only dependency.
|
||||
|
||||
## Design
|
||||
|
||||
The production `scan_video_marks` and `stabilize_localizations` paths selected
|
||||
the first continuous 17-frame accepted run for each mark. The production
|
||||
`_mask_for_region` then built every per-frame mask, including the mark-specific
|
||||
box or silhouette policy and padding. Seventeen frames satisfy ROSE's `16n+1`
|
||||
constraint and keep the cloud run small.
|
||||
|
||||
| Case | Tracked source | Frames | Source to working size | Mask pixels per frame |
|
||||
| --- | --- | ---: | --- | ---: |
|
||||
| Sora moving | `data/fixtures/visible/sora/provider-original.mp4` (`7dd8a9d4`) | 150-166 | 704 x 1280 to 448 x 832 | 4,508 |
|
||||
| Veo fixed | `data/fixtures/visible/veo/provider-original.mp4` (`e778194b`) | 0-16 | 1280 x 720 to 832 x 464 | 209 |
|
||||
| Kling fixed | `data/fixtures/visible/kling/provider-original.mp4` (`f3e8d6a7`) | 0-16 | 1280 x 720 to 832 x 464 | 2,968 |
|
||||
|
||||
The aspect-preserving working sizes are multiples of 16 and stay near the
|
||||
1.3B models' 480p operating point. `cv2`, MI-GAN, and LaMa ran through the
|
||||
library's own `fill` and `stabilize_filled_frame` calls on those same decoded
|
||||
frames and masks. VACE used 30 steps and the required RGB-128 hole. ROSE used
|
||||
its official unmasked-video-plus-mask conditioning at 50 steps. Seed 0 and an
|
||||
empty removal prompt were fixed for both models.
|
||||
|
||||
This is a deliberately small decision study, not a population estimate: one
|
||||
seed, one 17-frame window from each of three real exports, and no ground truth
|
||||
for the pixels hidden by the marks. It spans one moving mark and two fixed
|
||||
marks, but not every registered provider. Hailuo was screened and omitted
|
||||
because its real fixture did not produce an accepted visual run without
|
||||
provenance, so it could not satisfy the experiment's requirement to derive the
|
||||
mask from production detection.
|
||||
|
||||
Only generated pixels inside the production mask were composited into the
|
||||
source. This protects the rest of the image even though the raw VACE output
|
||||
changed outside-mask pixels by 1.10-2.51 levels of RGB MAE and ROSE by
|
||||
2.72-3.22. Every delivered arm was then encoded as H.264 CRF 14 at 16 fps and
|
||||
decoded again for measurement.
|
||||
|
||||
The primary temporal metric is mean absolute frame-to-frame RGB difference
|
||||
inside the union of adjacent masks. `excess` subtracts the same reading from
|
||||
the marked source. Lower is smoother, but it is not ground-truth recovery: a
|
||||
blurred patch can score well. Outside-mask MAE is measured after delivery, so
|
||||
its small non-zero value includes H.264 error even though the compositor kept
|
||||
the source pixels. The last two columns run the library's own detector on the
|
||||
delivered working-resolution video.
|
||||
|
||||
## Results
|
||||
|
||||
| Case | Method | Mask temporal MAD | Excess vs source | Outside-mask MAE | Detector frames / 17, max score | Run seconds |
|
||||
| --- | --- | ---: | ---: | ---: | --- | ---: |
|
||||
| Sora | OpenCV | 1.023 | -1.081 | 0.2613 | 0, 0.336 | 0.125 |
|
||||
| Sora | MI-GAN | 1.072 | -1.032 | 0.2559 | 0, 0.336 | 12.865 |
|
||||
| Sora | LaMa | 1.481 | -0.623 | 0.2577 | 0, 0.336 | 97.539 |
|
||||
| Sora | VACE | 1.619 | -0.485 | 0.2520 | 0, 0.336 | 12.226 |
|
||||
| Sora | ROSE | 1.581 | -0.523 | 0.2655 | 0, 0.336 | 14.051 |
|
||||
| Veo | OpenCV | 0.150 | +0.097 | 0.0047 | 0, 0.449 | 0.105 |
|
||||
| Veo | MI-GAN | 0.040 | -0.013 | 0.0007 | 0, 0.434 | 11.180 |
|
||||
| Veo | LaMa | 0.049 | -0.004 | 0.0039 | 0, 0.434 | 98.762 |
|
||||
| Veo | VACE | 1.338 | +1.286 | 0.0132 | 0, 0.491 | 11.301 |
|
||||
| Veo | ROSE | 0.330 | +0.277 | 0.0043 | **17, 0.663** | 13.000 |
|
||||
| Kling | OpenCV | 0.433 | +0.102 | 0.2653 | 0, 0.000 | 0.296 |
|
||||
| Kling | MI-GAN | 0.234 | -0.097 | 0.2687 | 0, 0.000 | 30.799 |
|
||||
| Kling | LaMa | 0.386 | +0.055 | 0.2756 | 0, 0.000 | 102.011 |
|
||||
| Kling | VACE | 0.737 | +0.406 | 0.2740 | 0, 0.000 | 12.709 |
|
||||
| Kling | ROSE | 0.840 | +0.510 | 0.2554 | 0, 0.000 | 13.045 |
|
||||
|
||||
OpenCV was narrowly lowest on Sora; MI-GAN was the best learned backend there
|
||||
and the lowest-scoring method on Veo and Kling. Relative to MI-GAN, VACE was
|
||||
1.51x, 33.65x, and 3.15x higher on Sora, Veo, and Kling; ROSE was 1.47x, 8.29x,
|
||||
and 3.60x higher. The resized Sora and Veo source controls themselves
|
||||
fell below the temporal acceptance rule (maximum scores 0.627 and 0.522), so a
|
||||
zero accepted-frame count cannot prove removal for those two cases. It remains
|
||||
useful as a mutation check: ROSE raised the resized Veo from 0 accepted frames
|
||||
to 17. The resized Kling source remained accepted on 17/17 frames at 0.675,
|
||||
and every output cleared that detector.
|
||||
|
||||
CPU runtimes were remeasured serially after other experiments stopped, with
|
||||
88.96 percent host CPU idle immediately before the run. They exclude model
|
||||
setup (MI-GAN 1.5-4.1 seconds; LaMa 3.4-13.1 seconds). GPU rows are clean
|
||||
pipeline-call time on one H100, excluding container start and model loading.
|
||||
VACE peaked at 22.60-22.87 GiB allocated and ROSE at 19.71-19.83 GiB.
|
||||
|
||||
At Modal's [2026-09-27 H100 list rate](https://modal.com/pricing) of
|
||||
$0.001097 per second, the measured VACE calls cost about $0.0398 total and the
|
||||
ROSE calls $0.0440, or $0.0837 for all six 17-frame calls. That is an
|
||||
inference-only floor. Cold starts, weight loading, warm-idle time, CPU, memory,
|
||||
and storage make the billed experiment higher.
|
||||
|
||||
## Frame review
|
||||
|
||||
- **Sora:** OpenCV produced a moving gray smear. MI-GAN, LaMa, correctly
|
||||
conditioned VACE, and ROSE all left a small bright sliver near the mascot's
|
||||
former edge; the temporal models did not reconstruct the deck seam better
|
||||
enough to offset their higher temporal MAD.
|
||||
- **Veo:** MI-GAN and LaMa blended the tiny translucent label into the flat
|
||||
background. VACE replaced it with a conspicuous gray rectangular block.
|
||||
ROSE visibly reproduced the `Veo` text on frames 0, 8, and 16, matching the
|
||||
detector's 17/17 result.
|
||||
- **Kling:** every method removed the wordmark, but the temporal models left a
|
||||
darker rectangular water patch. Their frame-to-frame MAD was 3.15-3.60x the
|
||||
MI-GAN result, so the patch moved in intensity rather than providing the
|
||||
expected temporal advantage.
|
||||
|
||||
An initial VACE control mistakenly passed the original marked pixels under the
|
||||
mask. It visibly reconstructed `Sora` and raised the detector score to 0.427.
|
||||
The official VACE preprocessing code replaces the hole with gray first; the
|
||||
control is retained in the external report as `vace`, marked
|
||||
`excluded_from_verdict`, while every tabled VACE row is the corrected
|
||||
`vace-gray` arm. This is direct evidence that a future VACE integration must
|
||||
pin the model's conditioning contract, not merely its mask polarity.
|
||||
|
||||
## Verdict and reproducibility
|
||||
|
||||
Do not add VACE or ROSE as an optional visible-video backend. Neither model
|
||||
beat the practical CPU MI-GAN tier on temporal stability, both introduced
|
||||
visible artifacts on simple backgrounds, and ROSE failed the basic Veo
|
||||
removal case. Their roughly 20-23 GiB H100 footprint, multi-model downloads,
|
||||
cold-start cost, and ROSE demo-license boundary add deployment complexity
|
||||
without a measured quality gain. Revisit only with a temporally trained model
|
||||
that can beat MI-GAN on a wider real corpus while preserving tiny-mask
|
||||
conditioning. This one-seed result rejects an optional backend now; it does not
|
||||
claim that no prompt or seed can improve either model. Any revisit should
|
||||
predefine that tuning budget and still beat the unchanged shipped controls.
|
||||
|
||||
The external harness SHA-256 is
|
||||
`e9bb2232bac2b719ae72d62e395699949609c3cdfdcd46e38f4d681ed3edee29`.
|
||||
The retained JSON report SHA-256 is
|
||||
`76da749eb0410cebb402914082f128ab339f943cc2596f05ae8c86d92cfedb41`.
|
||||
The run is bound to library commit `742a0dc`. The report records the complete
|
||||
model revisions, source and output hashes, masks, per-method metrics, and the
|
||||
invalid VACE control; full-frame and enlarged mask contact sheets were reviewed
|
||||
from the retained artifacts.
|
||||
@@ -119,7 +119,7 @@ against primary sources before adding jurisdiction-specific claims.
|
||||
|
||||
## Visible AI-generation marks + detection methods (deep-research 2026-07-10, adversarially verified)
|
||||
|
||||
**Google Gemini visible "sparkle" -- tier-dependent, and spec-undocumented by Google.** Google primary sources (the Nano Banana Pro blog and the gemini.google image-generation page, both WebFetch-verified) confirm Gemini images carry BOTH the invisible SynthID (on ALL Google-AI media) AND a visible sparkle, but the visible mark is **tier-gated**: applied for FREE and Google AI **Pro** users, and **REMOVED** for Google AI **Ultra** subscribers, inside **Google AI Studio**, and on **API / dev** output. Update 2026-09-23: since 2026-08-14 a Gemini "Media watermark" setting lets any consumer tier switch the visible mark off for images, Omni video and Lyria music, except work accounts and India, South Korea and Vietnam without Ultra (Gemini help 17405358; the setting was observed in the Gemini web app on 2026-09-23). The tier-gating above is therefore no longer the only reason a sparkle is missing. So a Google-C2PA image with NO visible sparkle is expected (setting off, Ultra, API), not evidence it is clean -- this reinforces the `identify` "no visible mark != clean" rule. The ONLY official verifier is the SynthID flow (upload to the Gemini app, ask if it is AI-generated), which reads the INVISIBLE mark; there is **no official visible-sparkle detector**, and Google publishes **no** glyph geometry / size / opacity / color / locale / placement spec. So our capture-based sparkle template is the only source of truth and cannot be validated against a vendor spec -- keep reverse-engineering from real captures (do not expect a published spec).
|
||||
**Google Gemini visible "sparkle" -- tier-dependent, and spec-undocumented by Google.** Google primary sources (the Nano Banana Pro blog and the gemini.google image-generation page, both WebFetch-verified) confirm Gemini images carry BOTH the invisible SynthID (on ALL Google-AI media) AND a visible sparkle, but the visible mark is **tier-gated**: applied for FREE and Google AI **Pro** users, and **REMOVED** for Google AI **Ultra** subscribers, inside **Google AI Studio**, and on **API / dev** output. Update 2026-09-23: since 2026-08-14 a Gemini "Media watermark" setting lets any consumer tier switch the visible mark off for images, Omni video and Lyria music, except work accounts and India, South Korea and Vietnam without Ultra (Gemini help 17405358; the setting was observed in the Gemini web app on 2026-09-23). The tier-gating above is therefore no longer the only reason a sparkle is missing. So a Google-C2PA image with NO visible sparkle is expected (setting off, Ultra, API), not evidence it is clean -- this reinforces the `identify` "no visible mark != clean" rule. The ONLY official verifier is the SynthID flow: upload to the Gemini app and ask directly whether the file contains a SynthID watermark, requiring the SynthID verification tool and prohibiting inference from visible content, logos, text, or metadata. A generic question about AI generation is not an oracle result. The tool reads the INVISIBLE mark; there is **no official visible-sparkle detector**, and Google publishes **no** glyph geometry / size / opacity / color / locale / placement spec. So our capture-based sparkle template is the only source of truth and cannot be validated against a vendor spec -- keep reverse-engineering from real captures (do not expect a published spec).
|
||||
|
||||
**Google Veo video marks use two incompatible visible designs.** Public raw
|
||||
clips verify the current four-point diamond and the legacy bottom-right `Veo`
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
# on it, including the ComfyUI node package. The console script below carries
|
||||
# the same weight, since users have it on PATH.
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.42.0"
|
||||
version = "0.43.0"
|
||||
description = "AI watermark remover for visible, invisible, and provenance marks in images and video"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11,<3.15"
|
||||
|
||||
@@ -70,6 +70,9 @@ explicitly names a tracked canonical result.
|
||||
| `qwen_scrub_prototype.py` | Probe low-strength Qwen regeneration on a GPU. |
|
||||
| `selective_text_restoration.py` | Evaluate text restoration over a scrubbed image. |
|
||||
| `synthid_pixel_probe.py` | Run the experimental local SynthID carrier probe. |
|
||||
| `rewatermarking_study.py` | Measure same-encoder message replacement and decoded-pixel fidelity on publication-cleared images with the open DWT-DCT, TrustMark, and VideoSeal image encoders. |
|
||||
| `rewatermarking_video_study.py` | Measure VideoSeal message replacement and mean PSNR on saved synthetic and publication-cleared MP4 clips. |
|
||||
| `rewatermarking_synthid_candidates.py` | Remove visible AI marks and metadata, then prepare native-resolution controls and VideoSeal-overlaid variants from two cleared SynthID-positive originals without making provider requests. |
|
||||
| `video_synthid_sweep.py` | Build oracle-gated video regeneration candidates. |
|
||||
|
||||
## Generated assets
|
||||
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Revision-pinned local Perth oracle for development-only benchmarks.
|
||||
|
||||
Perth v1 is an implicit audio watermark: its public model returns one presence
|
||||
confidence, not a user-selected payload. This adapter therefore exposes a
|
||||
thresholded presence reading and deliberately has no message label or forgery
|
||||
operation.
|
||||
|
||||
The official ``resemble-perth`` wheel predates current PyTorch releases and
|
||||
pins an obsolete stack. Development runs install the exact source revision
|
||||
below without dependency resolution, then verify the bundled checkpoint before
|
||||
loading it. Nothing here ships in the installed CLI or public Python API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE_COMMIT = "ff1c8ac55a976971245cdd53c18d6131ca00d993"
|
||||
PACKAGE_NAME = "resemble-perth"
|
||||
PACKAGE_VERSION = "1.1.0"
|
||||
CHECKPOINT_SHA256 = "a15bce457ebc53ce5e6c9c3f11df78cf7ee2bf9cdab0a798902135b4c4027670"
|
||||
CHECKPOINT_RELATIVE_PATH = Path("perth_net/pretrained/implicit/perth_net_250000.pth.tar")
|
||||
SAMPLE_RATE = 16_000
|
||||
DETECTION_THRESHOLD = 0.5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerthReading:
|
||||
"""One implicit-watermark presence score."""
|
||||
|
||||
confidence: float
|
||||
|
||||
@property
|
||||
def detected(self) -> bool:
|
||||
"""Apply the upstream round-at-one-half decision rule."""
|
||||
return self.confidence > DETECTION_THRESHOLD
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def import_failure() -> Exception | None:
|
||||
"""Return the missing or incompatible dependency error, if any."""
|
||||
try:
|
||||
import librosa # noqa: F401
|
||||
import perth
|
||||
import torch # noqa: F401
|
||||
import torchaudio # noqa: F401
|
||||
|
||||
if importlib.metadata.version(PACKAGE_NAME) != PACKAGE_VERSION:
|
||||
raise RuntimeError(f"{PACKAGE_NAME} must be installed at version {PACKAGE_VERSION}")
|
||||
if perth.PerthImplicitWatermarker is None:
|
||||
raise RuntimeError("resemble-perth imported without PerthImplicitWatermarker")
|
||||
except (ImportError, importlib.metadata.PackageNotFoundError, RuntimeError) as exc:
|
||||
return exc
|
||||
return None
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Whether the pinned Perth source stack is importable."""
|
||||
return import_failure() is None
|
||||
|
||||
|
||||
def checkpoint_path() -> Path:
|
||||
"""Return the package checkpoint after verifying its pinned digest."""
|
||||
failure = import_failure()
|
||||
if failure is not None:
|
||||
raise RuntimeError(
|
||||
"Perth oracle requires the pinned development source and its local dependencies; "
|
||||
f"install resemble-ai/Perth@{SOURCE_COMMIT} without adding it to runtime dependencies"
|
||||
) from failure
|
||||
import perth
|
||||
|
||||
path = Path(perth.__file__).resolve().parent / CHECKPOINT_RELATIVE_PATH
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"pinned Perth checkpoint is missing: {path}")
|
||||
digest = _sha256_file(path)
|
||||
if digest != CHECKPOINT_SHA256:
|
||||
raise RuntimeError(f"pinned Perth checkpoint digest mismatch: {digest} != {CHECKPOINT_SHA256}")
|
||||
return path
|
||||
|
||||
|
||||
def load_model() -> object:
|
||||
"""Load the pinned Perth implicit watermarker on CPU."""
|
||||
checkpoint_path()
|
||||
import perth
|
||||
|
||||
return perth.PerthImplicitWatermarker(device="cpu")
|
||||
|
||||
|
||||
def embed(model: object, samples: object) -> object:
|
||||
"""Embed Perth's fixed implicit watermark into mono float samples."""
|
||||
import numpy as np
|
||||
|
||||
array = np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
return np.asarray(model.apply_watermark(array, sample_rate=SAMPLE_RATE), dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
def read(model: object, samples: object) -> PerthReading:
|
||||
"""Return the raw implicit-watermark confidence for mono float samples."""
|
||||
import numpy as np
|
||||
|
||||
array = np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
raw = np.asarray(model.get_watermark(array, sample_rate=SAMPLE_RATE, round=False), dtype=np.float32).reshape(-1)
|
||||
if raw.size != 1 or not np.isfinite(raw).all():
|
||||
raise RuntimeError(f"Perth returned an invalid confidence shape/value: {raw!r}")
|
||||
return PerthReading(confidence=float(raw[0]))
|
||||
Executable
+485
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and run the local Perth and PixelSeal benchmark study.
|
||||
|
||||
The study uses only deterministic synthetic audio and the two publication-
|
||||
cleared engine-selection carriers named below. It measures Perth before and
|
||||
after the production visible-video cleaning path, and PixelSeal before and
|
||||
after JPEG plus a foreign-message embed. Invisible image-removal profiles are
|
||||
recorded as unavailable when the host has no CUDA device; this script never
|
||||
falls back to a provider oracle or uploads an artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "src"))
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import perth_oracle # noqa: E402
|
||||
import pixelseal_oracle # noqa: E402
|
||||
from audioseal_experiment import ( # noqa: E402
|
||||
FRAME_COUNT,
|
||||
SPEECH_VOICES,
|
||||
aac_packet_bytes,
|
||||
assert_distinct_voices,
|
||||
carrier_seed,
|
||||
decode_audio_f32,
|
||||
encode_video,
|
||||
ffmpeg_version,
|
||||
require_tools,
|
||||
run_ffmpeg,
|
||||
sora_like_mark,
|
||||
speech_provenance,
|
||||
stamped_frame,
|
||||
synth_carrier,
|
||||
synthesize_speech,
|
||||
wav_pcm16_bytes,
|
||||
)
|
||||
from watermark_benchmark import SCHEMA_VERSION, run_benchmark, sha256_file, write_jsonl # noqa: E402
|
||||
|
||||
from remove_ai_watermarks.video import remove_video_visible # noqa: E402
|
||||
|
||||
log = logging.getLogger("perth_pixelseal_study")
|
||||
|
||||
RECIPE_VERSION = "perth-pixelseal-study-v2"
|
||||
IMAGE_SIZE = 512
|
||||
IMAGE_CARRIERS: tuple[Path, ...] = (
|
||||
REPO / "data/evaluations/engine-selection/originals/openai/00.png",
|
||||
REPO / "data/evaluations/engine-selection/originals/meta/00.png",
|
||||
)
|
||||
AUDIO_CARRIERS: tuple[str, ...] = ("tone_stack", "pinkish_noise", "white_noise")
|
||||
AUDIO_DURATION_S = 8.0
|
||||
|
||||
|
||||
def _row(
|
||||
*,
|
||||
root: Path,
|
||||
case_id: str,
|
||||
pair_id: str,
|
||||
media_type: str,
|
||||
adapter: str,
|
||||
arm: str,
|
||||
state: str,
|
||||
path: Path,
|
||||
reference: Path | None,
|
||||
source_revision: str,
|
||||
transform_name: str,
|
||||
transform_revision: str,
|
||||
parameters: dict[str, object],
|
||||
seed: int | None,
|
||||
expected: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"case_id": case_id,
|
||||
"pair_id": pair_id,
|
||||
"media_type": media_type,
|
||||
"adapter": adapter,
|
||||
"arm": arm,
|
||||
"state": state,
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"sha256": sha256_file(path),
|
||||
"reference_path": reference.relative_to(root).as_posix() if reference is not None else None,
|
||||
"reference_sha256": sha256_file(reference) if reference is not None else None,
|
||||
"source_revision": source_revision,
|
||||
"transform": {
|
||||
"name": transform_name,
|
||||
"revision": transform_revision,
|
||||
"parameters": parameters,
|
||||
},
|
||||
"seed": seed,
|
||||
"expected": expected,
|
||||
}
|
||||
|
||||
|
||||
def _save_rgb(path: Path, values: np.ndarray) -> None:
|
||||
pixels = np.rint(np.clip(values, 0.0, 1.0) * 255.0).astype(np.uint8)
|
||||
Image.fromarray(pixels, "RGB").save(path)
|
||||
|
||||
|
||||
def _build_pixelseal_rows(output_dir: Path, artifacts: Path) -> list[dict[str, object]]:
|
||||
if not pixelseal_oracle.available():
|
||||
failure = pixelseal_oracle.import_failure()
|
||||
raise SystemExit(f"PixelSeal development stack is unavailable: {failure}")
|
||||
model = pixelseal_oracle.load_model()
|
||||
fixed = pixelseal_oracle.message_bits()
|
||||
foreign = pixelseal_oracle.message_bits(seed=8)
|
||||
pins = {
|
||||
"source_commit": pixelseal_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": pixelseal_oracle.CHECKPOINT_SHA256,
|
||||
"model_card_sha256": pixelseal_oracle.MODEL_CARD_SHA256,
|
||||
"message": list(fixed),
|
||||
}
|
||||
rows: list[dict[str, object]] = []
|
||||
source_manifest = REPO / "data/evaluations/engine-selection/carrier-manifest.csv"
|
||||
source_revision = f"engine-selection@sha256:{sha256_file(source_manifest)}"
|
||||
for source in IMAGE_CARRIERS:
|
||||
provider = source.parent.name
|
||||
pair_id = f"pixelseal-{provider}-00"
|
||||
clean_path = artifacts / f"{pair_id}-clean.png"
|
||||
clean = Image.open(source).convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.Resampling.LANCZOS)
|
||||
clean.save(clean_path)
|
||||
clean_values = np.asarray(clean, dtype=np.float32) / 255.0
|
||||
|
||||
marked_path = artifacts / f"{pair_id}-marked.png"
|
||||
_save_rgb(marked_path, np.asarray(pixelseal_oracle.embed(model, clean_values, fixed)))
|
||||
attacked_path = artifacts / f"{pair_id}-jpeg-q90.jpg"
|
||||
Image.open(marked_path).save(attacked_path, quality=90, subsampling=0)
|
||||
forged_path = artifacts / f"{pair_id}-forged.png"
|
||||
_save_rgb(forged_path, np.asarray(pixelseal_oracle.embed(model, clean_values, foreign)))
|
||||
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": pair_id,
|
||||
"media_type": "image",
|
||||
"adapter": "pixelseal",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="standardize-carrier",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"source_sha256": sha256_file(source), "size": IMAGE_SIZE},
|
||||
seed=None,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="pixelseal-embed",
|
||||
transform_revision=f"videoseal@{pixelseal_oracle.SOURCE_COMMIT}",
|
||||
parameters=pins,
|
||||
seed=pixelseal_oracle.MESSAGE_SEED,
|
||||
expected="detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-jpeg-q90",
|
||||
arm="positive",
|
||||
state="attacked",
|
||||
path=attacked_path,
|
||||
reference=clean_path,
|
||||
transform_name="jpeg",
|
||||
transform_revision="pillow",
|
||||
parameters={"quality": 90, "subsampling": 0, **pins},
|
||||
seed=None,
|
||||
expected="unresolved",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-forged",
|
||||
arm="wrong_key",
|
||||
state="forged",
|
||||
path=forged_path,
|
||||
reference=clean_path,
|
||||
transform_name="pixelseal-embed-foreign-message",
|
||||
transform_revision=f"videoseal@{pixelseal_oracle.SOURCE_COMMIT}",
|
||||
parameters={**pins, "foreign_message": list(foreign), "foreign_message_seed": 8},
|
||||
seed=8,
|
||||
expected="not_detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
del model
|
||||
gc.collect()
|
||||
return rows
|
||||
|
||||
|
||||
def _extract_audio(ffmpeg: str, source: Path, target: Path) -> None:
|
||||
run_ffmpeg(
|
||||
ffmpeg,
|
||||
[
|
||||
"-i",
|
||||
str(source),
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
str(perth_oracle.SAMPLE_RATE),
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
str(target),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _build_perth_rows(
|
||||
output_dir: Path,
|
||||
artifacts: Path,
|
||||
ffmpeg: str,
|
||||
) -> tuple[list[dict[str, object]], dict[str, object]]:
|
||||
if not perth_oracle.available():
|
||||
failure = perth_oracle.import_failure()
|
||||
raise SystemExit(f"Perth development stack is unavailable: {failure}")
|
||||
model = perth_oracle.load_model()
|
||||
pins = {
|
||||
"source_commit": perth_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": perth_oracle.CHECKPOINT_SHA256,
|
||||
"implicit_payload": True,
|
||||
}
|
||||
source_revision = f"{RECIPE_VERSION}@sha256:{sha256_file(Path(__file__))}"
|
||||
rows: list[dict[str, object]] = []
|
||||
samantha_marked_path: Path | None = None
|
||||
for name in AUDIO_CARRIERS:
|
||||
executed_seed = carrier_seed(name)
|
||||
clean_samples = synth_carrier(name, AUDIO_DURATION_S, seed=executed_seed)
|
||||
clean_path = artifacts / f"perth-{name}-clean.wav"
|
||||
clean_path.write_bytes(wav_pcm16_bytes(clean_samples, perth_oracle.SAMPLE_RATE))
|
||||
marked_path = artifacts / f"perth-{name}-marked.wav"
|
||||
marked = np.asarray(perth_oracle.embed(model, clean_samples), dtype=np.float32)
|
||||
marked_path.write_bytes(wav_pcm16_bytes(marked, perth_oracle.SAMPLE_RATE))
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": f"perth-{name}",
|
||||
"media_type": "audio",
|
||||
"adapter": "perth",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"perth-{name}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="synthesize-carrier",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"carrier": name, "duration_s": AUDIO_DURATION_S},
|
||||
seed=executed_seed,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"perth-{name}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="perth-embed",
|
||||
transform_revision=f"resemble-ai/Perth@{perth_oracle.SOURCE_COMMIT}",
|
||||
parameters=pins,
|
||||
seed=None,
|
||||
# Perth is speech-trained; these deliberately out-of-domain
|
||||
# carriers measure dependence without asserting detection.
|
||||
expected="unresolved",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
raw_speech = {voice: synthesize_speech(voice, artifacts) for voice in SPEECH_VOICES}
|
||||
assert_distinct_voices({voice: path.read_bytes() for voice, path in raw_speech.items()})
|
||||
tts = speech_provenance()
|
||||
for voice, aiff in raw_speech.items():
|
||||
pair_id = f"perth-speech-{voice.casefold()}"
|
||||
clean_samples = decode_audio_f32(ffmpeg, aiff)
|
||||
clean_path = artifacts / f"{pair_id}-clean.wav"
|
||||
clean_path.write_bytes(wav_pcm16_bytes(clean_samples, perth_oracle.SAMPLE_RATE))
|
||||
marked = np.asarray(perth_oracle.embed(model, clean_samples), dtype=np.float32)
|
||||
marked_path = artifacts / f"{pair_id}-marked.wav"
|
||||
marked_path.write_bytes(wav_pcm16_bytes(marked, perth_oracle.SAMPLE_RATE))
|
||||
if voice == "Samantha":
|
||||
samantha_marked_path = marked_path
|
||||
common = {
|
||||
"root": output_dir,
|
||||
"pair_id": pair_id,
|
||||
"media_type": "audio",
|
||||
"adapter": "perth",
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
provenance = {**tts, "voice": voice, "duration_s": round(clean_samples.shape[0] / perth_oracle.SAMPLE_RATE, 6)}
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-clean",
|
||||
arm="matched_negative",
|
||||
state="clean",
|
||||
path=clean_path,
|
||||
reference=None,
|
||||
transform_name="synthesize-speech",
|
||||
transform_revision=RECIPE_VERSION,
|
||||
parameters={"tts": provenance},
|
||||
seed=None,
|
||||
expected="not_detected",
|
||||
),
|
||||
_row(
|
||||
**common,
|
||||
case_id=f"{pair_id}-marked",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=marked_path,
|
||||
reference=clean_path,
|
||||
transform_name="perth-embed",
|
||||
transform_revision=f"resemble-ai/Perth@{perth_oracle.SOURCE_COMMIT}",
|
||||
parameters={**pins, "tts": provenance},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
mark, mark_x, mark_y = sora_like_mark()
|
||||
frames = [stamped_frame(index, mark, mark_x, mark_y) for index in range(FRAME_COUNT)]
|
||||
if samantha_marked_path is None:
|
||||
raise RuntimeError("the configured speech carriers must include Samantha")
|
||||
source_video = artifacts / "perth-video-source.mp4"
|
||||
encode_video(
|
||||
ffmpeg,
|
||||
source_video,
|
||||
frames,
|
||||
samantha_marked_path,
|
||||
audio_codec="aac",
|
||||
audio_rate=perth_oracle.SAMPLE_RATE,
|
||||
)
|
||||
cleaned_video = artifacts / "perth-video-cleaned.mp4"
|
||||
result = remove_video_visible(source_video, cleaned_video, mark="sora", backend="cv2")
|
||||
if result.output is None:
|
||||
raise SystemExit("visible-video cleaning did not produce an output")
|
||||
source_packets = aac_packet_bytes(ffmpeg, source_video)
|
||||
cleaned_packets = aac_packet_bytes(ffmpeg, cleaned_video)
|
||||
source_audio = artifacts / "perth-video-source-audio.wav"
|
||||
cleaned_audio = artifacts / "perth-video-cleaned-audio.wav"
|
||||
_extract_audio(ffmpeg, source_video, source_audio)
|
||||
_extract_audio(ffmpeg, cleaned_video, cleaned_audio)
|
||||
packet_identity = source_packets == cleaned_packets
|
||||
rows.extend(
|
||||
(
|
||||
_row(
|
||||
root=output_dir,
|
||||
case_id="perth-video-source-audio",
|
||||
pair_id="perth-video-pass-through",
|
||||
media_type="audio",
|
||||
adapter="perth",
|
||||
arm="positive",
|
||||
state="marked",
|
||||
path=source_audio,
|
||||
reference=None,
|
||||
source_revision=source_revision,
|
||||
transform_name="mux-aac",
|
||||
transform_revision=ffmpeg_version(ffmpeg),
|
||||
parameters={"codec": "aac", "bitrate": "128k", **pins},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
_row(
|
||||
root=output_dir,
|
||||
case_id="perth-video-cleaned-audio",
|
||||
pair_id="perth-video-pass-through",
|
||||
media_type="audio",
|
||||
adapter="perth",
|
||||
arm="positive",
|
||||
state="attacked",
|
||||
path=cleaned_audio,
|
||||
reference=source_audio,
|
||||
source_revision=source_revision,
|
||||
transform_name="remove-video-visible-audio-pass-through",
|
||||
transform_revision=source_revision,
|
||||
parameters={
|
||||
"audio_packets_identical": packet_identity,
|
||||
"visual_removed_frames": result.removed_frames,
|
||||
**pins,
|
||||
},
|
||||
seed=None,
|
||||
expected="detected",
|
||||
),
|
||||
)
|
||||
)
|
||||
del model
|
||||
gc.collect()
|
||||
return rows, {
|
||||
"audio_packets_identical": packet_identity,
|
||||
"source_audio_packet_sha256": hashlib.sha256(source_packets).hexdigest(),
|
||||
"cleaned_audio_packet_sha256": hashlib.sha256(cleaned_packets).hexdigest(),
|
||||
"visual_removed_frames": result.removed_frames,
|
||||
}
|
||||
|
||||
|
||||
def _cuda_available() -> bool:
|
||||
try:
|
||||
import torch
|
||||
|
||||
return bool(torch.cuda.is_available())
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def build_and_run(output_dir: Path) -> Path:
|
||||
"""Create the study artifacts, manifest, metadata, and kernel results."""
|
||||
if output_dir.exists():
|
||||
raise FileExistsError(f"output directory already exists: {output_dir}")
|
||||
ffmpeg, _ffprobe = require_tools()
|
||||
output_dir.mkdir(parents=True)
|
||||
artifacts = output_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
rows = _build_pixelseal_rows(output_dir, artifacts)
|
||||
perth_rows, pass_through = _build_perth_rows(output_dir, artifacts, ffmpeg)
|
||||
rows.extend(perth_rows)
|
||||
manifest = output_dir / "manifest.jsonl"
|
||||
write_jsonl(manifest, rows)
|
||||
metadata = {
|
||||
"schema_version": 1,
|
||||
"recipe": RECIPE_VERSION,
|
||||
"pixelseal": {
|
||||
"source_commit": pixelseal_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": pixelseal_oracle.CHECKPOINT_SHA256,
|
||||
"removal_profiles": {
|
||||
"status": "not_run" if not _cuda_available() else "available_not_automatic",
|
||||
"reason": "qwen-zimage, sdxl-zimage, and chroma-zimage require an NVIDIA CUDA device",
|
||||
},
|
||||
},
|
||||
"perth": {
|
||||
"source_commit": perth_oracle.SOURCE_COMMIT,
|
||||
"checkpoint_sha256": perth_oracle.CHECKPOINT_SHA256,
|
||||
"message": "not_applicable_fixed_implicit_mark",
|
||||
"video_visible_path": pass_through,
|
||||
},
|
||||
}
|
||||
(output_dir / "study-metadata.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
results = output_dir / "results.jsonl"
|
||||
run_benchmark(manifest, results)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
try:
|
||||
results = build_and_run(args.output_dir.resolve())
|
||||
except (FileNotFoundError, OSError, RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
log.info("Wrote study results to %s", results)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+214
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Revision-pinned local PixelSeal oracle for development-only benchmarks.
|
||||
|
||||
Meta publishes PixelSeal in the VideoSeal source tree, but not as a standalone
|
||||
TorchScript artifact. This loader uses the exact upstream source revision and
|
||||
official checkpoint, verifies both the model card and checkpoint digests, and
|
||||
builds only the image inference graph. It avoids the upstream convenience
|
||||
loader because that loader imports training datasets and writes downloads
|
||||
relative to the current working directory.
|
||||
|
||||
Nothing here ships in the installed CLI or public Python API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import logging
|
||||
import os
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE_COMMIT = "870ca7fb33578b90f14c602016b6c2788096226e"
|
||||
PACKAGE_NAME = "videoseal"
|
||||
PACKAGE_VERSION = "1.0"
|
||||
CHECKPOINT_URL = "https://dl.fbaipublicfiles.com/videoseal/pixelseal/checkpoint.pth"
|
||||
CHECKPOINT_SHA256 = "0c5665cff20eb6ce1b5aaa7d91c19dafb418bfee32d02dd3344e4ed60d9d75bd"
|
||||
MODEL_CARD_SHA256 = "96c4a448f7d81810780e5bde5a51b039db544b7ec2baf65a605302c1b17362c9"
|
||||
N_BITS = 256
|
||||
MESSAGE_SEED = 7
|
||||
DETECTION_BIT_ACCURACY_THRESHOLD = 0.9
|
||||
CACHE_DIR = Path(
|
||||
os.environ.get(
|
||||
"PIXELSEAL_CACHE_DIR",
|
||||
str(Path.home() / ".cache" / "remove-ai-watermarks-dev" / "pixelseal"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PixelSealReading:
|
||||
"""Decoded bits and matched-message accuracy for one image."""
|
||||
|
||||
bit_accuracy: float
|
||||
decoded_bits: tuple[int, ...]
|
||||
|
||||
@property
|
||||
def detected(self) -> bool:
|
||||
"""Apply this adapter's explicit matched-message rule."""
|
||||
return self.bit_accuracy >= DETECTION_BIT_ACCURACY_THRESHOLD
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return "".join(f"{byte:02x}" for byte in _bytes(self.decoded_bits))
|
||||
|
||||
|
||||
def _bytes(bits: Sequence[int]) -> bytes:
|
||||
return bytes(sum(bit << (7 - index) for index, bit in enumerate(bits[i : i + 8])) for i in range(0, len(bits), 8))
|
||||
|
||||
|
||||
def message_bits(seed: int = MESSAGE_SEED) -> tuple[int, ...]:
|
||||
"""The oracle's fixed 256-bit message, deterministic across processes."""
|
||||
import numpy as np
|
||||
|
||||
rng = np.random.default_rng(seed)
|
||||
return tuple(int(bit) for bit in rng.integers(0, 2, N_BITS))
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def import_failure() -> Exception | None:
|
||||
"""Return the missing or incompatible dependency error, if any."""
|
||||
try:
|
||||
import omegaconf # noqa: F401
|
||||
import timm # noqa: F401
|
||||
import torch # noqa: F401
|
||||
import torchvision # noqa: F401
|
||||
import videoseal # noqa: F401
|
||||
|
||||
if importlib.metadata.version(PACKAGE_NAME) != PACKAGE_VERSION:
|
||||
raise RuntimeError(
|
||||
f"{PACKAGE_NAME} must be installed from source commit {SOURCE_COMMIT} "
|
||||
f"(package version {PACKAGE_VERSION})"
|
||||
)
|
||||
except (ImportError, importlib.metadata.PackageNotFoundError, RuntimeError) as exc:
|
||||
return exc
|
||||
return None
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Whether the pinned PixelSeal source stack is importable."""
|
||||
return import_failure() is None
|
||||
|
||||
|
||||
def checkpoint_path() -> Path:
|
||||
"""Return the pinned local checkpoint, downloading and verifying it once."""
|
||||
target = CACHE_DIR / "checkpoint.pth"
|
||||
if target.is_file() and _sha256_file(target) == CHECKPOINT_SHA256:
|
||||
return target
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
temporary = CACHE_DIR / "checkpoint.pth.part"
|
||||
log.info("Downloading pinned PixelSeal checkpoint from %s", CHECKPOINT_URL)
|
||||
urllib.request.urlretrieve(CHECKPOINT_URL, temporary)
|
||||
digest = _sha256_file(temporary)
|
||||
if digest != CHECKPOINT_SHA256:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"pinned PixelSeal checkpoint digest mismatch: {digest} != {CHECKPOINT_SHA256}")
|
||||
temporary.replace(target)
|
||||
return target
|
||||
|
||||
|
||||
def load_model() -> object:
|
||||
"""Build PixelSeal from the pinned official source and weights on CPU."""
|
||||
failure = import_failure()
|
||||
if failure is not None:
|
||||
raise RuntimeError(
|
||||
"PixelSeal oracle requires facebookresearch/videoseal at source commit "
|
||||
f"{SOURCE_COMMIT}, plus torch, torchvision, timm, and omegaconf"
|
||||
) from failure
|
||||
|
||||
import torch
|
||||
import videoseal
|
||||
from omegaconf import OmegaConf
|
||||
from videoseal.augmentation.augmenter import get_dummy_augmenter
|
||||
from videoseal.models import Videoseal, build_embedder, build_extractor
|
||||
from videoseal.modules.jnd import JND
|
||||
|
||||
package_root = Path(videoseal.__file__).resolve().parent
|
||||
card_path = package_root / "cards" / "pixelseal.yaml"
|
||||
if not card_path.is_file() or _sha256_file(card_path) != MODEL_CARD_SHA256:
|
||||
raise RuntimeError("installed videoseal source does not carry the pinned PixelSeal model card")
|
||||
config = OmegaConf.load(card_path)
|
||||
args = config.args
|
||||
embedder = build_embedder(
|
||||
config.embedder.model,
|
||||
config.embedder.params,
|
||||
args.nbits,
|
||||
args.hidden_size_multiplier,
|
||||
)
|
||||
extractor = build_extractor(
|
||||
config.extractor.model,
|
||||
config.extractor.params,
|
||||
args.img_size_proc,
|
||||
args.nbits,
|
||||
)
|
||||
model = Videoseal(
|
||||
embedder,
|
||||
extractor,
|
||||
get_dummy_augmenter(),
|
||||
attenuation=JND(in_channels=1, out_channels=1),
|
||||
scaling_w=args.scaling_w,
|
||||
scaling_i=args.scaling_i,
|
||||
img_size=args.img_size_proc,
|
||||
chunk_size=args.videowam_chunk_size,
|
||||
step_size=args.videowam_step_size,
|
||||
)
|
||||
checkpoint = torch.load(checkpoint_path(), map_location="cpu", weights_only=True)
|
||||
loaded = model.load_state_dict(checkpoint["model"], strict=False)
|
||||
if loaded.missing_keys or loaded.unexpected_keys:
|
||||
raise RuntimeError(
|
||||
f"PixelSeal checkpoint/source mismatch: missing={loaded.missing_keys}, unexpected={loaded.unexpected_keys}"
|
||||
)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def _image_tensor(image: object) -> object:
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
array = np.asarray(image)
|
||||
if array.ndim != 3 or array.shape[2] != 3:
|
||||
raise ValueError(f"PixelSeal expects HxWx3 RGB pixels, got {array.shape}")
|
||||
values = array.astype(np.float32) / 255.0 if array.dtype == np.uint8 else np.asarray(array, dtype=np.float32)
|
||||
if not np.isfinite(values).all() or float(values.min()) < 0.0 or float(values.max()) > 1.0:
|
||||
raise ValueError("PixelSeal RGB pixels must be finite and in [0, 1]")
|
||||
return torch.from_numpy(np.ascontiguousarray(values)).permute(2, 0, 1).unsqueeze(0)
|
||||
|
||||
|
||||
def embed(model: object, image: object, message: Sequence[int]) -> object:
|
||||
"""Embed one message into an RGB image, returning float RGB in [0, 1]."""
|
||||
import torch
|
||||
|
||||
tensor = _image_tensor(image)
|
||||
msg = torch.tensor([list(message)], dtype=torch.float32)
|
||||
with torch.inference_mode():
|
||||
marked = model.embed(tensor, msgs=msg, is_video=False)["imgs_w"]
|
||||
return marked[0].permute(1, 2, 0).numpy()
|
||||
|
||||
|
||||
def read(model: object, image: object, *, message: Sequence[int] | None = None) -> PixelSealReading:
|
||||
"""Decode one image and compare its bits with the oracle's fixed message."""
|
||||
import torch
|
||||
|
||||
expected = message_bits() if message is None else tuple(message)
|
||||
tensor = _image_tensor(image)
|
||||
with torch.inference_mode():
|
||||
predictions = model.detect(tensor)["preds"]
|
||||
decoded = tuple(int(bit) for bit in (predictions[0, 1:] > 0).to(torch.int64).tolist())
|
||||
accuracy = sum(int(bit == want) for bit, want in zip(decoded, expected, strict=True)) / N_BITS
|
||||
return PixelSealReading(bit_accuracy=accuracy, decoded_bits=decoded)
|
||||
@@ -161,7 +161,10 @@ SURFACES: dict[str, OracleSurface] = {
|
||||
url="https://gemini.google.com/",
|
||||
media_types=("image", "video", "audio"),
|
||||
project_evidence=("image", "video"),
|
||||
prompt="Was this image/video/audio created or edited by Google AI?",
|
||||
prompt=(
|
||||
"Does this image/video/audio contain a SynthID watermark? Use the SynthID verification tool only; "
|
||||
"do not infer from visible content, logos, text, or metadata."
|
||||
),
|
||||
clean_result_note="Use not_detected only for an explicit SynthID-negative result; preserve unclear results.",
|
||||
account_slots=True,
|
||||
network_slots=True,
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
"""Measure same-encoder re-watermarking on the project's open image marks.
|
||||
|
||||
This development-only study applies message A to publication-cleared images,
|
||||
serializes the result as PNG, then applies message B with the same encoder to
|
||||
the decoded A-marked artifact. It measures both messages after each write and
|
||||
keeps clean-to-output and marginal attack fidelity separate.
|
||||
|
||||
The default cohort is twelve images: both providers from six preselected
|
||||
content strata in the tracked engine-selection matrix. Generated artifacts
|
||||
and reports belong outside the repository under ``.local-eval/``.
|
||||
|
||||
uv run --extra dev --extra trustmark python \
|
||||
scripts/rewatermarking_study.py \
|
||||
--output-dir /path/to/.local-eval/rewatermarking-2026-09-27
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageOps
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "src"))
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import engine_selection_manifest # noqa: E402
|
||||
import videoseal_oracle # noqa: E402
|
||||
from engine_selection_manifest import ContentFixture # noqa: E402
|
||||
from watermark_benchmark import repository_state, sha256_file # noqa: E402
|
||||
from watermark_benchmark_cohort import trustmark_runtime # noqa: E402
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUDY_VERSION = "rewatermarking-study-v1"
|
||||
DEFAULT_SIZE = 512
|
||||
DEFAULT_PAIR_IDS = ("00", "02", "07", "08", "11", "17")
|
||||
DWT_MATCH_THRESHOLD = 44 / 48
|
||||
TRUSTMARK_BITS = 61
|
||||
|
||||
|
||||
def bit_accuracy(decoded: Sequence[int], expected: Sequence[int]) -> float:
|
||||
"""Return the exact matching-bit fraction for equal-length messages."""
|
||||
if not decoded or len(decoded) != len(expected):
|
||||
raise ValueError("bit strings must be non-empty and have equal length")
|
||||
return sum(int(actual == wanted) for actual, wanted in zip(decoded, expected, strict=True)) / len(expected)
|
||||
|
||||
|
||||
def image_fidelity(
|
||||
reference: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
artifact: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
) -> dict[str, Any]:
|
||||
"""Return the benchmark kernel's decoded 8-bit image-fidelity metrics."""
|
||||
if reference.shape != artifact.shape:
|
||||
raise ValueError(f"image shape mismatch: {reference.shape} != {artifact.shape}")
|
||||
if reference.dtype != np.uint8 or artifact.dtype != np.uint8:
|
||||
raise ValueError("image fidelity requires uint8 decoded pixels")
|
||||
delta = artifact.astype(np.float64) - reference.astype(np.float64)
|
||||
absolute = np.abs(delta)
|
||||
mse = float(np.mean(np.square(delta)))
|
||||
changed = np.any(reference != artifact, axis=-1)
|
||||
identical = mse == 0.0
|
||||
return {
|
||||
"mae_8bit": float(np.mean(absolute)),
|
||||
"changed_fraction": float(np.mean(changed)),
|
||||
"psnr_db": None if identical else 20.0 * math.log10(255.0) - 10.0 * math.log10(mse),
|
||||
"psnr_status": "unbounded_identical" if identical else "measured",
|
||||
}
|
||||
|
||||
|
||||
def seeded_bits(seed: int, count: int) -> tuple[int, ...]:
|
||||
"""Return a deterministic binary message without touching global RNG state."""
|
||||
return tuple(int(bit) for bit in np.random.default_rng(seed).integers(0, 2, count))
|
||||
|
||||
|
||||
def message_sha256(bits: Sequence[int]) -> str:
|
||||
return hashlib.sha256("".join(str(bit) for bit in bits).encode()).hexdigest()
|
||||
|
||||
|
||||
def _standardize(carrier: ContentFixture, size: int) -> Image.Image:
|
||||
with Image.open(carrier.path) as source:
|
||||
return ImageOps.fit(
|
||||
source.convert("RGB"),
|
||||
(size, size),
|
||||
method=Image.Resampling.LANCZOS,
|
||||
centering=(0.5, 0.5),
|
||||
)
|
||||
|
||||
|
||||
def _save_png(image: Image.Image | np.ndarray[Any, Any], path: Path) -> np.ndarray[Any, np.dtype[np.uint8]]:
|
||||
if isinstance(image, Image.Image):
|
||||
output = image.convert("RGB")
|
||||
else:
|
||||
pixels = np.asarray(image)
|
||||
if pixels.dtype != np.uint8:
|
||||
pixels = np.rint(np.clip(pixels, 0.0, 1.0) * 255.0).astype(np.uint8)
|
||||
output = Image.fromarray(pixels, "RGB")
|
||||
output.save(path, format="PNG")
|
||||
with Image.open(path) as decoded:
|
||||
return np.asarray(decoded.convert("RGB"), dtype=np.uint8).copy()
|
||||
|
||||
|
||||
def _artifact_record(path: Path) -> dict[str, str]:
|
||||
return {"path": path.name, "sha256": sha256_file(path)}
|
||||
|
||||
|
||||
def _dwt_messages() -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
from remove_ai_watermarks.invisible_watermark import _BITS_48
|
||||
|
||||
return tuple(int(bit) for bit in format(_BITS_48["FLUX.2 (Black Forest Labs)"], "048b")), tuple(
|
||||
int(bit) for bit in format(_BITS_48["Stable Diffusion XL"], "048b")
|
||||
)
|
||||
|
||||
|
||||
def _dwt_embed(
|
||||
pixels: np.ndarray[Any, np.dtype[np.uint8]], message: Sequence[int]
|
||||
) -> np.ndarray[Any, np.dtype[np.uint8]]:
|
||||
import cv2
|
||||
from imwatermark import WatermarkEncoder
|
||||
|
||||
cv2.setNumThreads(2)
|
||||
encoder = WatermarkEncoder()
|
||||
encoder.set_watermark("bits", list(message))
|
||||
marked = encoder.encode(cv2.cvtColor(pixels, cv2.COLOR_RGB2BGR), "dwtDct")
|
||||
return cv2.cvtColor(np.asarray(marked, dtype=np.uint8), cv2.COLOR_BGR2RGB)
|
||||
|
||||
|
||||
def _dwt_decode(pixels: np.ndarray[Any, np.dtype[np.uint8]]) -> tuple[int, ...]:
|
||||
import cv2
|
||||
from imwatermark import WatermarkDecoder
|
||||
|
||||
decoder = WatermarkDecoder("bits", 48)
|
||||
decoded = decoder.decode(cv2.cvtColor(pixels, cv2.COLOR_RGB2BGR), "dwtDct")
|
||||
return tuple(int(bit) for bit in decoded)
|
||||
|
||||
|
||||
def _videoseal_embed(
|
||||
model: object,
|
||||
pixels: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
message: Sequence[int],
|
||||
) -> np.ndarray[Any, np.dtype[np.float32]]:
|
||||
return np.asarray(videoseal_oracle.embed_image(model, pixels.astype(np.float32) / 255.0, message))
|
||||
|
||||
|
||||
def _videoseal_decode(model: object, pixels: np.ndarray[Any, np.dtype[np.uint8]]) -> tuple[int, ...]:
|
||||
return videoseal_oracle.decode_image(model, pixels.astype(np.float32) / 255.0)
|
||||
|
||||
|
||||
def _trustmark_decode(runtime: Any, pixels: np.ndarray[Any, np.dtype[np.uint8]]) -> tuple[tuple[int, ...], bool, int]:
|
||||
payload, present, schema = runtime.decode(Image.fromarray(pixels, "RGB"), "binary")
|
||||
bits = tuple(int(bit) for bit in payload) if payload and set(payload) <= {"0", "1"} else ()
|
||||
return bits, bool(present), int(schema)
|
||||
|
||||
|
||||
def _case_row(
|
||||
*,
|
||||
scheme: str,
|
||||
carrier: ContentFixture,
|
||||
message_a: Sequence[int],
|
||||
message_b: Sequence[int],
|
||||
clean: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
marked_path: Path,
|
||||
rewatermarked_path: Path,
|
||||
marked: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
rewatermarked: np.ndarray[Any, np.dtype[np.uint8]],
|
||||
decoded_marked: Sequence[int],
|
||||
decoded_rewatermarked: Sequence[int],
|
||||
marked_detected: bool,
|
||||
rewatermarked_detected_a: bool,
|
||||
rewatermarked_detected_b: bool,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"scheme": scheme,
|
||||
"carrier": carrier.name,
|
||||
"pair_id": carrier.pair_id,
|
||||
"provider": carrier.provider,
|
||||
"content_stratum": carrier.content_stratum,
|
||||
"source_sha256": carrier.sha256,
|
||||
"reuse_basis": carrier.reuse_basis,
|
||||
"messages": {
|
||||
"a_sha256": message_sha256(message_a),
|
||||
"b_sha256": message_sha256(message_b),
|
||||
"hamming_fraction": 1.0 - bit_accuracy(message_a, message_b),
|
||||
},
|
||||
"marked": {
|
||||
"bit_accuracy_vs_a": bit_accuracy(decoded_marked, message_a),
|
||||
"bit_accuracy_vs_b": bit_accuracy(decoded_marked, message_b),
|
||||
"detected_as_a": marked_detected,
|
||||
"artifact": _artifact_record(marked_path),
|
||||
"fidelity_from_clean": image_fidelity(clean, marked),
|
||||
},
|
||||
"rewatermarked": {
|
||||
"bit_accuracy_vs_a": bit_accuracy(decoded_rewatermarked, message_a),
|
||||
"bit_accuracy_vs_b": bit_accuracy(decoded_rewatermarked, message_b),
|
||||
"detected_as_a": rewatermarked_detected_a,
|
||||
"detected_as_b": rewatermarked_detected_b,
|
||||
"artifact": _artifact_record(rewatermarked_path),
|
||||
"fidelity_from_clean": image_fidelity(clean, rewatermarked),
|
||||
"fidelity_from_marked": image_fidelity(marked, rewatermarked),
|
||||
},
|
||||
"attack_success": marked_detected and not rewatermarked_detected_a and rewatermarked_detected_b,
|
||||
}
|
||||
|
||||
|
||||
def _run_dwt(carrier: ContentFixture, clean_path: Path, artifacts: Path) -> list[dict[str, Any]]:
|
||||
message_flux, message_sdxl = _dwt_messages()
|
||||
with Image.open(clean_path) as image:
|
||||
clean = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for victim_name, message_a, attacker_name, message_b in (
|
||||
("flux", message_flux, "sdxl", message_sdxl),
|
||||
("sdxl", message_sdxl, "flux", message_flux),
|
||||
):
|
||||
marked_path = artifacts / f"{carrier.name}--dwt-{victim_name}--marked-a.png"
|
||||
marked = _save_png(_dwt_embed(clean, message_a), marked_path)
|
||||
rewatermarked_path = artifacts / f"{carrier.name}--dwt-{victim_name}-to-{attacker_name}--rewatermarked-b.png"
|
||||
rewatermarked = _save_png(_dwt_embed(marked, message_b), rewatermarked_path)
|
||||
decoded_marked = _dwt_decode(marked)
|
||||
decoded_rewatermarked = _dwt_decode(rewatermarked)
|
||||
rows.append(
|
||||
_case_row(
|
||||
scheme=f"dwt-dct-{victim_name}-to-{attacker_name}",
|
||||
carrier=carrier,
|
||||
message_a=message_a,
|
||||
message_b=message_b,
|
||||
clean=clean,
|
||||
marked_path=marked_path,
|
||||
rewatermarked_path=rewatermarked_path,
|
||||
marked=marked,
|
||||
rewatermarked=rewatermarked,
|
||||
decoded_marked=decoded_marked,
|
||||
decoded_rewatermarked=decoded_rewatermarked,
|
||||
marked_detected=bit_accuracy(decoded_marked, message_a) >= DWT_MATCH_THRESHOLD,
|
||||
rewatermarked_detected_a=bit_accuracy(decoded_rewatermarked, message_a) >= DWT_MATCH_THRESHOLD,
|
||||
rewatermarked_detected_b=bit_accuracy(decoded_rewatermarked, message_b) >= DWT_MATCH_THRESHOLD,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _run_trustmark(runtime: Any, carrier: ContentFixture, clean_path: Path, artifacts: Path) -> dict[str, Any]:
|
||||
message_a = seeded_bits(7, TRUSTMARK_BITS)
|
||||
message_b = seeded_bits(8, TRUSTMARK_BITS)
|
||||
with Image.open(clean_path) as image:
|
||||
clean_image = image.convert("RGB")
|
||||
clean = np.asarray(clean_image, dtype=np.uint8)
|
||||
marked_image = runtime.encode(clean_image, "".join(str(bit) for bit in message_a), MODE="binary")
|
||||
marked_path = artifacts / f"{carrier.name}--trustmark--marked-a.png"
|
||||
marked = _save_png(marked_image, marked_path)
|
||||
rewatermarked_image = runtime.encode(
|
||||
Image.fromarray(marked, "RGB"),
|
||||
"".join(str(bit) for bit in message_b),
|
||||
MODE="binary",
|
||||
)
|
||||
rewatermarked_path = artifacts / f"{carrier.name}--trustmark--rewatermarked-b.png"
|
||||
rewatermarked = _save_png(rewatermarked_image, rewatermarked_path)
|
||||
decoded_marked, marked_present, marked_schema = _trustmark_decode(runtime, marked)
|
||||
decoded_rewatermarked, rewatermarked_present, rewatermarked_schema = _trustmark_decode(runtime, rewatermarked)
|
||||
marked_detected = marked_present and marked_schema == 1 and decoded_marked == message_a
|
||||
detected_a = rewatermarked_present and rewatermarked_schema == 1 and decoded_rewatermarked == message_a
|
||||
detected_b = rewatermarked_present and rewatermarked_schema == 1 and decoded_rewatermarked == message_b
|
||||
return _case_row(
|
||||
scheme="trustmark-p-schema1",
|
||||
carrier=carrier,
|
||||
message_a=message_a,
|
||||
message_b=message_b,
|
||||
clean=clean,
|
||||
marked_path=marked_path,
|
||||
rewatermarked_path=rewatermarked_path,
|
||||
marked=marked,
|
||||
rewatermarked=rewatermarked,
|
||||
decoded_marked=decoded_marked,
|
||||
decoded_rewatermarked=decoded_rewatermarked,
|
||||
marked_detected=marked_detected,
|
||||
rewatermarked_detected_a=detected_a,
|
||||
rewatermarked_detected_b=detected_b,
|
||||
)
|
||||
|
||||
|
||||
def _run_videoseal(model: object, carrier: ContentFixture, clean_path: Path, artifacts: Path) -> dict[str, Any]:
|
||||
message_a = videoseal_oracle.message_bits(7)
|
||||
message_b = videoseal_oracle.message_bits(8)
|
||||
with Image.open(clean_path) as image:
|
||||
clean = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
marked_path = artifacts / f"{carrier.name}--videoseal-image--marked-a.png"
|
||||
marked = _save_png(_videoseal_embed(model, clean, message_a), marked_path)
|
||||
rewatermarked_path = artifacts / f"{carrier.name}--videoseal-image--rewatermarked-b.png"
|
||||
rewatermarked = _save_png(_videoseal_embed(model, marked, message_b), rewatermarked_path)
|
||||
decoded_marked = _videoseal_decode(model, marked)
|
||||
decoded_rewatermarked = _videoseal_decode(model, rewatermarked)
|
||||
return _case_row(
|
||||
scheme="videoseal-image-256b",
|
||||
carrier=carrier,
|
||||
message_a=message_a,
|
||||
message_b=message_b,
|
||||
clean=clean,
|
||||
marked_path=marked_path,
|
||||
rewatermarked_path=rewatermarked_path,
|
||||
marked=marked,
|
||||
rewatermarked=rewatermarked,
|
||||
decoded_marked=decoded_marked,
|
||||
decoded_rewatermarked=decoded_rewatermarked,
|
||||
marked_detected=bit_accuracy(decoded_marked, message_a) >= videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD,
|
||||
rewatermarked_detected_a=(
|
||||
bit_accuracy(decoded_rewatermarked, message_a) >= videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD
|
||||
),
|
||||
rewatermarked_detected_b=(
|
||||
bit_accuracy(decoded_rewatermarked, message_b) >= videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def summarize(rows: Sequence[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Aggregate only validated positive controls, while retaining all counts."""
|
||||
summary: dict[str, Any] = {}
|
||||
for scheme in sorted({str(row["scheme"]) for row in rows}):
|
||||
group = [row for row in rows if row["scheme"] == scheme]
|
||||
controls = [row for row in group if row["marked"]["detected_as_a"]]
|
||||
successes = [row for row in controls if row["attack_success"]]
|
||||
|
||||
def values(path: tuple[str, ...], source: Sequence[dict[str, Any]] = controls) -> list[float]:
|
||||
found: list[float] = []
|
||||
for row in source:
|
||||
value: Any = row
|
||||
for key in path:
|
||||
value = value[key]
|
||||
if value is not None:
|
||||
found.append(float(value))
|
||||
return found
|
||||
|
||||
def distribution(numbers: Sequence[float]) -> dict[str, float] | None:
|
||||
if not numbers:
|
||||
return None
|
||||
return {
|
||||
"min": min(numbers),
|
||||
"median": statistics.median(numbers),
|
||||
"max": max(numbers),
|
||||
"mean": statistics.fmean(numbers),
|
||||
}
|
||||
|
||||
summary[scheme] = {
|
||||
"cases": len(group),
|
||||
"valid_positive_controls": len(controls),
|
||||
"attack_successes": len(successes),
|
||||
"attack_success_rate_on_valid_controls": len(successes) / len(controls) if controls else None,
|
||||
"marked_bit_accuracy_vs_a": distribution(values(("marked", "bit_accuracy_vs_a"))),
|
||||
"rewatermarked_bit_accuracy_vs_a": distribution(values(("rewatermarked", "bit_accuracy_vs_a"))),
|
||||
"rewatermarked_bit_accuracy_vs_b": distribution(values(("rewatermarked", "bit_accuracy_vs_b"))),
|
||||
"rewatermarked_psnr_from_clean": distribution(values(("rewatermarked", "fidelity_from_clean", "psnr_db"))),
|
||||
"attack_psnr_from_marked": distribution(values(("rewatermarked", "fidelity_from_marked", "psnr_db"))),
|
||||
}
|
||||
return summary
|
||||
|
||||
|
||||
def _contact_sheet(rows: Sequence[dict[str, Any]], artifacts: Path, output: Path) -> None:
|
||||
selected: list[dict[str, Any]] = []
|
||||
for scheme in sorted({str(row["scheme"]) for row in rows}):
|
||||
candidates = [row for row in rows if row["scheme"] == scheme and row["marked"]["detected_as_a"]]
|
||||
if candidates:
|
||||
selected.append(min(candidates, key=lambda row: row["rewatermarked"]["fidelity_from_clean"]["psnr_db"]))
|
||||
if not selected:
|
||||
return
|
||||
thumb = 256
|
||||
label_height = 36
|
||||
sheet = Image.new("RGB", (thumb * 3, (thumb + label_height) * len(selected)), "white")
|
||||
draw = ImageDraw.Draw(sheet)
|
||||
for row_index, row in enumerate(selected):
|
||||
clean_path = artifacts / f"{row['carrier']}--clean.png"
|
||||
paths = (
|
||||
clean_path,
|
||||
artifacts / row["marked"]["artifact"]["path"],
|
||||
artifacts / row["rewatermarked"]["artifact"]["path"],
|
||||
)
|
||||
top = row_index * (thumb + label_height)
|
||||
for column, path in enumerate(paths):
|
||||
with Image.open(path) as image:
|
||||
thumbnail = image.convert("RGB").resize((thumb, thumb), Image.Resampling.LANCZOS)
|
||||
sheet.paste(thumbnail, (column * thumb, top))
|
||||
psnr = row["rewatermarked"]["fidelity_from_clean"]["psnr_db"]
|
||||
draw.text((4, top + thumb + 4), f"{row['scheme']} | {row['carrier']} | total PSNR {psnr:.2f} dB", fill="black")
|
||||
sheet.save(output, format="PNG")
|
||||
|
||||
|
||||
def run_study(output_dir: Path, *, pair_ids: Sequence[str], size: int, torch_threads: int) -> Path:
|
||||
if output_dir.exists():
|
||||
raise FileExistsError(f"refusing to overwrite existing output directory {output_dir}")
|
||||
carriers = engine_selection_manifest.load_content_manifest(
|
||||
REPO / "data" / "evaluations" / "engine-selection" / "content-manifest.csv"
|
||||
)
|
||||
selected = [carrier for carrier in carriers if carrier.pair_id in pair_ids]
|
||||
expected = len(pair_ids) * len({carrier.provider for carrier in carriers})
|
||||
if len(selected) != expected:
|
||||
raise ValueError(f"selected {len(selected)} carriers, expected {expected} for pair ids {pair_ids}")
|
||||
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(torch_threads)
|
||||
output_dir.mkdir(parents=True)
|
||||
artifacts = output_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
videoseal_model = videoseal_oracle.load_model()
|
||||
trustmark = trustmark_runtime()
|
||||
rows: list[dict[str, Any]] = []
|
||||
for index, carrier in enumerate(selected, start=1):
|
||||
log.info("Carrier %s/%s: %s", index, len(selected), carrier.name)
|
||||
clean_path = artifacts / f"{carrier.name}--clean.png"
|
||||
_save_png(_standardize(carrier, size), clean_path)
|
||||
rows.extend(_run_dwt(carrier, clean_path, artifacts))
|
||||
rows.append(_run_trustmark(trustmark, carrier, clean_path, artifacts))
|
||||
rows.append(_run_videoseal(videoseal_model, carrier, clean_path, artifacts))
|
||||
|
||||
cases_path = output_dir / "cases.jsonl"
|
||||
cases_path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8")
|
||||
summary = summarize(rows)
|
||||
repo_state = repository_state()
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"study_version": STUDY_VERSION,
|
||||
"repository_head": repo_state["commit"],
|
||||
"tracked_tree_dirty": repo_state["dirty"],
|
||||
"study_script_sha256": sha256_file(Path(__file__)),
|
||||
"source_manifest_sha256": sha256_file(
|
||||
REPO / "data" / "evaluations" / "engine-selection" / "content-manifest.csv"
|
||||
),
|
||||
"videoseal_source_commit": videoseal_oracle.SOURCE_COMMIT,
|
||||
"videoseal_jit_sha256": videoseal_oracle.JIT_SHA256,
|
||||
"pair_ids": list(pair_ids),
|
||||
"carriers": len(selected),
|
||||
"size": [size, size],
|
||||
"torch_threads": torch_threads,
|
||||
"case_rows_sha256": sha256_file(cases_path),
|
||||
"summary": summary,
|
||||
}
|
||||
report_path = output_dir / "report.json"
|
||||
report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
_contact_sheet(rows, artifacts, output_dir / "contact-sheet-worst-valid.png")
|
||||
log.info("Wrote %s", report_path)
|
||||
return report_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--size", type=int, default=DEFAULT_SIZE)
|
||||
parser.add_argument("--pair-id", action="append", dest="pair_ids")
|
||||
parser.add_argument("--torch-threads", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
if args.size < 256 or args.size % 16:
|
||||
parser.error("--size must be at least 256 and divisible by 16")
|
||||
if args.torch_threads < 1:
|
||||
parser.error("--torch-threads must be positive")
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
try:
|
||||
run_study(
|
||||
args.output_dir.resolve(),
|
||||
pair_ids=tuple(args.pair_ids or DEFAULT_PAIR_IDS),
|
||||
size=args.size,
|
||||
torch_threads=args.torch_threads,
|
||||
)
|
||||
except (FileExistsError, ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Prepare native-resolution, hash-bound SynthID re-watermarking candidates.
|
||||
|
||||
This local-only script first removes every detected visible AI mark and AI
|
||||
metadata, then makes one cleaned pixel control and one VideoSeal image-mode
|
||||
overlay for each of two publication-cleared, historically provider-verified
|
||||
SynthID originals. It does not contact a provider oracle.
|
||||
|
||||
uv run --extra dev python scripts/rewatermarking_synthid_candidates.py \
|
||||
--output-dir /path/to/.local-eval/rewatermarking-synthid-2026-09-27
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import videoseal_oracle # noqa: E402
|
||||
from watermark_benchmark import sha256_file # noqa: E402
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SELECTED = {
|
||||
"openai": "05b836ecfe40fd689177fda74384ae4fdcc446505bbc4281cd3cbb6523eb669e",
|
||||
"gemini": "4affd7f27767a445db6abf741355743ba8d95108ad922c9fff045feed8492236",
|
||||
}
|
||||
MESSAGE_SEED = 8
|
||||
|
||||
|
||||
def selected_sources() -> dict[str, Path]:
|
||||
"""Resolve only the two cleared and historically oracle-positive originals."""
|
||||
manifest = REPO / "data" / "synthid" / "manifest.csv"
|
||||
with manifest.open(newline="", encoding="utf-8") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
sources: dict[str, Path] = {}
|
||||
for provider, expected_hash in SELECTED.items():
|
||||
matches = [row for row in rows if row["sha256"] == expected_hash]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"expected exactly one manifest row for {provider}")
|
||||
row = matches[0]
|
||||
required_verifier = "openai-verify" if provider == "openai" else "gemini-app"
|
||||
if row["label"] != "pos" or row["verified_via"] != required_verifier:
|
||||
raise ValueError(f"{provider} source lacks the required positive-oracle provenance")
|
||||
source = manifest.parent / "originals" / row["filename"]
|
||||
if not source.is_file() or sha256_file(source) != expected_hash:
|
||||
raise ValueError(f"{provider} source bytes differ from the manifest")
|
||||
sources[provider] = source
|
||||
return sources
|
||||
|
||||
|
||||
def bit_accuracy(decoded: tuple[int, ...], expected: tuple[int, ...]) -> float:
|
||||
if len(decoded) != len(expected):
|
||||
raise ValueError("decoded message length differs from the expected message")
|
||||
return sum(int(actual == wanted) for actual, wanted in zip(decoded, expected, strict=True)) / len(expected)
|
||||
|
||||
|
||||
def overlay(model: object, pixels: np.ndarray[Any, Any], message: tuple[int, ...]) -> np.ndarray[Any, Any]:
|
||||
"""Embed VideoSeal B in image mode and return 8-bit pixels."""
|
||||
marked = videoseal_oracle.embed_image(model, pixels.astype(np.float32) / 255.0, message)
|
||||
return np.rint(np.clip(marked, 0.0, 1.0) * 255.0).astype(np.uint8)
|
||||
|
||||
|
||||
def decoded_accuracy(model: object, pixels: np.ndarray[Any, Any], message: tuple[int, ...]) -> float:
|
||||
decoded = videoseal_oracle.decode_image(model, pixels.astype(np.float32) / 255.0)
|
||||
return bit_accuracy(decoded, message)
|
||||
|
||||
|
||||
def cleaned_control(
|
||||
source: Path,
|
||||
output: Path,
|
||||
*,
|
||||
required_visible_key: str | None = None,
|
||||
) -> tuple[np.ndarray[Any, Any], dict[str, Any]]:
|
||||
"""Remove visible AI marks and metadata before any SynthID oracle input."""
|
||||
from remove_ai_watermarks.api import remove_visible_detailed
|
||||
|
||||
with Image.open(source) as image:
|
||||
original = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
report = remove_visible_detailed(
|
||||
source,
|
||||
output,
|
||||
sensitivity="auto",
|
||||
backend="cv2",
|
||||
strip_metadata=True,
|
||||
write_noop=True,
|
||||
)
|
||||
if report.status in {"partial", "unvalidated"}:
|
||||
raise RuntimeError(f"visible-mark cleanup was not validated: {report.status}")
|
||||
removed_keys = [mark.key for mark in report.marks]
|
||||
if required_visible_key is not None and required_visible_key not in removed_keys:
|
||||
raise RuntimeError(f"required visible mark was not removed: {required_visible_key}")
|
||||
with Image.open(output) as image:
|
||||
cleaned = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
if cleaned.shape != original.shape:
|
||||
raise ValueError("visible-mark cleanup changed image dimensions")
|
||||
return cleaned, {
|
||||
"status": report.status,
|
||||
"removed_keys": removed_keys,
|
||||
"removed_labels": report.labels,
|
||||
"changed_pixel_fraction": float(np.mean(np.any(original != cleaned, axis=-1))),
|
||||
"output_sha256": sha256_file(output),
|
||||
}
|
||||
|
||||
|
||||
def run(output_dir: Path, *, torch_threads: int) -> Path:
|
||||
if output_dir.exists():
|
||||
raise FileExistsError(f"refusing to overwrite existing output directory {output_dir}")
|
||||
sources = selected_sources()
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(torch_threads)
|
||||
model = videoseal_oracle.load_model()
|
||||
message = videoseal_oracle.message_bits(MESSAGE_SEED)
|
||||
output_dir.mkdir(parents=True)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for provider, source in sources.items():
|
||||
log.info("Preparing %s candidate", provider)
|
||||
control = output_dir / f"{provider}-pixel-control.png"
|
||||
clean, visible_preflight = cleaned_control(
|
||||
source,
|
||||
control,
|
||||
required_visible_key="gemini" if provider == "gemini" else None,
|
||||
)
|
||||
candidate = output_dir / f"{provider}-videoseal-overlay.png"
|
||||
marked = overlay(model, clean, message)
|
||||
if marked.shape != clean.shape:
|
||||
raise ValueError(f"{provider} overlay changed image dimensions")
|
||||
Image.fromarray(marked, "RGB").save(candidate)
|
||||
with Image.open(candidate) as image:
|
||||
marked = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
accuracy = decoded_accuracy(model, marked, message)
|
||||
if accuracy < videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD:
|
||||
raise RuntimeError(f"{provider} VideoSeal message B did not survive serialization: {accuracy:.4f}")
|
||||
delta = np.subtract(marked, clean, dtype=np.float32)
|
||||
np.square(delta, out=delta)
|
||||
mse = float(np.mean(delta, dtype=np.float64))
|
||||
if mse <= 0.0:
|
||||
raise RuntimeError(f"{provider} overlay did not change pixels")
|
||||
rows.append(
|
||||
{
|
||||
"provider": provider,
|
||||
"source_sha256": sha256_file(source),
|
||||
"control": {"path": control.name, "sha256": sha256_file(control)},
|
||||
"candidate": {"path": candidate.name, "sha256": sha256_file(candidate)},
|
||||
"visible_preflight": visible_preflight,
|
||||
"size": [int(clean.shape[1]), int(clean.shape[0])],
|
||||
"videoseal_b_bit_accuracy": accuracy,
|
||||
"psnr_from_control_db": 20.0 * math.log10(255.0) - 10.0 * math.log10(mse),
|
||||
"changed_pixel_fraction": float(np.mean(np.any(clean != marked, axis=-1))),
|
||||
}
|
||||
)
|
||||
report = {
|
||||
"source_manifest_sha256": sha256_file(REPO / "data" / "synthid" / "manifest.csv"),
|
||||
"study_script_sha256": sha256_file(Path(__file__)),
|
||||
"videoseal_jit_sha256": videoseal_oracle.JIT_SHA256,
|
||||
"message_seed": MESSAGE_SEED,
|
||||
"message_sha256": hashlib.sha256("".join(str(bit) for bit in message).encode()).hexdigest(),
|
||||
"rows": rows,
|
||||
"note": "Local candidate preparation only; no production SynthID verdict has been requested.",
|
||||
}
|
||||
report_path = output_dir / "report.json"
|
||||
report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return report_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", required=True, type=Path)
|
||||
parser.add_argument("--torch-threads", default=2, type=int)
|
||||
args = parser.parse_args()
|
||||
if args.torch_threads < 1:
|
||||
parser.error("--torch-threads must be positive")
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
try:
|
||||
report_path = run(args.output_dir.resolve(), torch_threads=args.torch_threads)
|
||||
except (FileExistsError, ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
log.info("Wrote %s", report_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Measure VideoSeal message replacement on decoded, saved MP4 artifacts.
|
||||
|
||||
The pair of carriers is the synthetic benchmark gradient and a committed,
|
||||
publication-cleared video fixture. Each clip is encoded before it is used by
|
||||
the next stage, so detector and fidelity measurements refer to the retained
|
||||
MP4 bytes. Outputs belong outside the repository under ``.local-eval/``.
|
||||
|
||||
uv run --extra dev python scripts/rewatermarking_video_study.py \
|
||||
--output-dir /path/to/.local-eval/rewatermarking-video-2026-09-27
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "src"))
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import videoseal_oracle # noqa: E402
|
||||
from videoseal_temporal_study import real_carrier_frames # noqa: E402
|
||||
from watermark_benchmark import decode_video_artifact, repository_state, sha256_file # noqa: E402
|
||||
from watermark_benchmark_video_cohort import encode_clip, synth_carrier # noqa: E402
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def bit_accuracy(decoded: Sequence[int], expected: Sequence[int]) -> float:
|
||||
"""Return matching-bit fraction for equal-length messages."""
|
||||
if not decoded or len(decoded) != len(expected):
|
||||
raise ValueError("bit strings must be non-empty and have equal length")
|
||||
return sum(int(bit == want) for bit, want in zip(decoded, expected, strict=True)) / len(expected)
|
||||
|
||||
|
||||
def mean_video_psnr(reference: np.ndarray[Any, Any], artifact: np.ndarray[Any, Any]) -> float | None:
|
||||
"""Match the benchmark kernel's mean of per-frame decoded RGB PSNR."""
|
||||
if reference.shape != artifact.shape or reference.ndim != 4 or reference.shape[-1] != 3:
|
||||
raise ValueError("video frames must share a (T, H, W, 3) shape")
|
||||
per_frame_mse = np.empty(reference.shape[0], dtype=np.float64)
|
||||
for index, (reference_frame, artifact_frame) in enumerate(zip(reference, artifact, strict=True)):
|
||||
delta = np.subtract(artifact_frame, reference_frame, dtype=np.float32)
|
||||
np.square(delta, out=delta)
|
||||
per_frame_mse[index] = np.mean(delta, dtype=np.float64)
|
||||
finite = per_frame_mse[per_frame_mse > 0.0]
|
||||
if finite.size == 0:
|
||||
return None
|
||||
return float(np.mean(10.0 * np.log10(1.0 / finite)))
|
||||
|
||||
|
||||
def _measure(
|
||||
model: object,
|
||||
frames: np.ndarray[Any, Any],
|
||||
message_a: Sequence[int],
|
||||
message_b: Sequence[int],
|
||||
) -> dict[str, Any]:
|
||||
reading = videoseal_oracle.read(model, frames, message=message_a)
|
||||
accuracy_b = bit_accuracy(reading.decoded_bits, message_b)
|
||||
return {
|
||||
"bit_accuracy_vs_a": reading.bit_accuracy,
|
||||
"bit_accuracy_vs_b": accuracy_b,
|
||||
"detected_as_a": reading.bit_accuracy >= videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD,
|
||||
"detected_as_b": accuracy_b >= videoseal_oracle.DETECTION_BIT_ACCURACY_THRESHOLD,
|
||||
}
|
||||
|
||||
|
||||
def run_study(output_dir: Path, *, torch_threads: int = 2) -> Path:
|
||||
if output_dir.exists():
|
||||
raise FileExistsError(f"refusing to overwrite existing output directory {output_dir}")
|
||||
ffmpeg = shutil.which("ffmpeg")
|
||||
if ffmpeg is None:
|
||||
raise RuntimeError("ffmpeg is required for the video study")
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(torch_threads)
|
||||
model = videoseal_oracle.load_model()
|
||||
message_a = videoseal_oracle.message_bits(7)
|
||||
message_b = videoseal_oracle.message_bits(8)
|
||||
output_dir.mkdir(parents=True)
|
||||
carriers = {
|
||||
"moving_gradient": (
|
||||
synth_carrier("moving_gradient")[:64],
|
||||
{"source": "deterministic synthetic benchmark carrier"},
|
||||
),
|
||||
"real_sora": real_carrier_frames("sora"),
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for name, (source_frames, provenance) in carriers.items():
|
||||
log.info("Measuring %s", name)
|
||||
case_dir = output_dir / name
|
||||
case_dir.mkdir()
|
||||
clean_path = encode_clip(ffmpeg, case_dir / "clean.mp4", source_frames)
|
||||
clean, clean_sha256 = decode_video_artifact(clean_path)
|
||||
marked_path = encode_clip(
|
||||
ffmpeg,
|
||||
case_dir / "marked-a.mp4",
|
||||
np.asarray(videoseal_oracle.embed(model, clean, message_a)),
|
||||
)
|
||||
marked, marked_sha256 = decode_video_artifact(marked_path)
|
||||
rewatermarked_path = encode_clip(
|
||||
ffmpeg,
|
||||
case_dir / "rewatermarked-b.mp4",
|
||||
np.asarray(videoseal_oracle.embed(model, marked, message_b)),
|
||||
)
|
||||
rewatermarked, rewatermarked_sha256 = decode_video_artifact(rewatermarked_path)
|
||||
baseline = _measure(model, clean, message_a, message_b)
|
||||
marked_reading = _measure(model, marked, message_a, message_b)
|
||||
rewatermarked_reading = _measure(model, rewatermarked, message_a, message_b)
|
||||
row = {
|
||||
"carrier": name,
|
||||
"provenance": provenance,
|
||||
"frame_shape": list(clean.shape),
|
||||
"clean": {"sha256": clean_sha256, **baseline},
|
||||
"marked": {
|
||||
"sha256": marked_sha256,
|
||||
**marked_reading,
|
||||
"mean_psnr_from_clean_db": mean_video_psnr(clean, marked),
|
||||
},
|
||||
"rewatermarked": {
|
||||
"sha256": rewatermarked_sha256,
|
||||
**rewatermarked_reading,
|
||||
"mean_psnr_from_clean_db": mean_video_psnr(clean, rewatermarked),
|
||||
"mean_psnr_from_marked_db": mean_video_psnr(marked, rewatermarked),
|
||||
},
|
||||
"attack_success": (
|
||||
marked_reading["detected_as_a"]
|
||||
and not rewatermarked_reading["detected_as_a"]
|
||||
and rewatermarked_reading["detected_as_b"]
|
||||
),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
cases_path = output_dir / "cases.jsonl"
|
||||
cases_path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8")
|
||||
report = {
|
||||
"repository_head": repository_state()["commit"],
|
||||
"study_script_sha256": sha256_file(Path(__file__)),
|
||||
"case_rows_sha256": sha256_file(cases_path),
|
||||
"videoseal_jit_sha256": videoseal_oracle.JIT_SHA256,
|
||||
"torch_threads": torch_threads,
|
||||
"carriers": len(rows),
|
||||
"attack_successes": sum(int(row["attack_success"]) for row in rows),
|
||||
"mean_psnr_from_clean_db": statistics.fmean(row["rewatermarked"]["mean_psnr_from_clean_db"] for row in rows),
|
||||
}
|
||||
report_path = output_dir / "report.json"
|
||||
report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
log.info("Wrote %s", report_path)
|
||||
return report_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--torch-threads", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
if args.torch_threads < 1:
|
||||
parser.error("--torch-threads must be positive")
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
try:
|
||||
run_study(args.output_dir.resolve(), torch_threads=args.torch_threads)
|
||||
except (FileExistsError, ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||
parser.error(str(exc))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -144,6 +144,33 @@ def embed(model: object, frames: object, message: Sequence[int]) -> object:
|
||||
return marked.permute(0, 2, 3, 1).numpy()
|
||||
|
||||
|
||||
def embed_image(model: object, pixels: object, message: Sequence[int]) -> object:
|
||||
"""Embed a message into one RGB image in [0, 1] (H, W, 3)."""
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
array = torch.from_numpy(np.ascontiguousarray(pixels, dtype=np.float32))
|
||||
tensor = array.permute(2, 0, 1).unsqueeze(0)
|
||||
msg = torch.tensor([list(message)], dtype=torch.float32)
|
||||
with torch.no_grad():
|
||||
marked = model.embed(tensor, msg, is_video=False)
|
||||
return marked[0].permute(1, 2, 0).numpy()
|
||||
|
||||
|
||||
def decode_image(model: object, pixels: object) -> tuple[int, ...]:
|
||||
"""Decode the message bits from one RGB image."""
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
array = torch.from_numpy(np.ascontiguousarray(pixels, dtype=np.float32))
|
||||
tensor = array.permute(2, 0, 1).unsqueeze(0)
|
||||
with torch.no_grad():
|
||||
predictions = model.detect(tensor, is_video=False)
|
||||
if isinstance(predictions, dict):
|
||||
predictions = predictions["preds"]
|
||||
return tuple(int(bit) for bit in (predictions[:, 1:] > 0).to(torch.int64).view(-1).tolist())
|
||||
|
||||
|
||||
def _aggregate_bit_preds(bit_preds: object, aggregation: str) -> object:
|
||||
"""Aggregate per-frame bit logits exactly as upstream ``extract_message``.
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
@@ -71,7 +72,9 @@ MEDIA_TYPES: tuple[Media, ...] = ("image", "audio", "video")
|
||||
ADAPTER_MEDIA: dict[str, Media] = {
|
||||
"dwt-dct": "image",
|
||||
"trustmark": "image",
|
||||
"pixelseal": "image",
|
||||
"audioseal": "audio",
|
||||
"perth": "audio",
|
||||
"videoseal": "video",
|
||||
}
|
||||
_FIELDS = {
|
||||
@@ -131,6 +134,8 @@ class DetectorOutcome:
|
||||
label: str | None
|
||||
error: str | None = None
|
||||
temporal: tuple[float, ...] | None = None
|
||||
message_bit_accuracy: float | None = None
|
||||
score: float | None = None
|
||||
|
||||
|
||||
class DetectorAdapter(Protocol):
|
||||
@@ -173,11 +178,66 @@ class VideoSealAdapter:
|
||||
def detect(self, path: Path, frames: NDArray[Any]) -> DetectorOutcome:
|
||||
from videoseal_oracle import read
|
||||
|
||||
reading = read(_videoseal_model(), frames)
|
||||
reading = read(_oracle_model("videoseal_oracle"), frames)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label=reading.label if reading.detected else None,
|
||||
temporal=reading.per_frame_bit_accuracy,
|
||||
message_bit_accuracy=reading.bit_accuracy,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerthAdapter:
|
||||
"""Adapter for Perth's implicit presence score."""
|
||||
|
||||
name: str
|
||||
source_file: Path
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
"""Probe the optional stack only when a Perth row is evaluated."""
|
||||
from perth_oracle import available
|
||||
|
||||
return available()
|
||||
|
||||
def detect(self, path: Path, samples: NDArray[Any]) -> DetectorOutcome:
|
||||
from perth_oracle import read
|
||||
|
||||
reading = read(_oracle_model("perth_oracle"), samples)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label="perth-v1" if reading.detected else None,
|
||||
score=reading.confidence,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PixelSealAdapter:
|
||||
"""Adapter for PixelSeal's fixed-message image verifier."""
|
||||
|
||||
name: str
|
||||
source_file: Path
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
"""Probe the optional stack only when a PixelSeal row is evaluated."""
|
||||
from pixelseal_oracle import available
|
||||
|
||||
return available()
|
||||
|
||||
def detect(self, path: Path, image: NDArray[Any]) -> DetectorOutcome:
|
||||
import cv2
|
||||
from pixelseal_oracle import read
|
||||
|
||||
from remove_ai_watermarks.image_io import to_bgr
|
||||
|
||||
rgb = cv2.cvtColor(to_bgr(image), cv2.COLOR_BGR2RGB)
|
||||
reading = read(_oracle_model("pixelseal_oracle"), rgb)
|
||||
return DetectorOutcome(
|
||||
status="detected" if reading.detected else "not_detected",
|
||||
label=reading.label if reading.detected else None,
|
||||
message_bit_accuracy=reading.bit_accuracy,
|
||||
)
|
||||
|
||||
|
||||
@@ -409,11 +469,15 @@ def default_adapters() -> dict[str, DetectorAdapter]:
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
import audioseal_oracle
|
||||
import perth_oracle
|
||||
import pixelseal_oracle
|
||||
import videoseal_oracle
|
||||
|
||||
dwt_source = Path(invisible_watermark.__file__).resolve()
|
||||
trustmark_source = Path(trustmark_detector.__file__).resolve()
|
||||
audio_oracle_source = Path(audioseal_oracle.__file__).resolve()
|
||||
perth_oracle_source = Path(perth_oracle.__file__).resolve()
|
||||
pixel_oracle_source = Path(pixelseal_oracle.__file__).resolve()
|
||||
video_oracle_source = Path(videoseal_oracle.__file__).resolve()
|
||||
return {
|
||||
"dwt-dct": FunctionAdapter(
|
||||
@@ -428,12 +492,20 @@ def default_adapters() -> dict[str, DetectorAdapter]:
|
||||
available=trustmark_detector.is_available(),
|
||||
detector=_detect_trustmark,
|
||||
),
|
||||
"pixelseal": PixelSealAdapter(
|
||||
name="pixelseal",
|
||||
source_file=pixel_oracle_source,
|
||||
),
|
||||
"audioseal": FunctionAdapter(
|
||||
name="audioseal",
|
||||
source_file=audio_oracle_source,
|
||||
available=audioseal_oracle.available(),
|
||||
detector=_detect_audioseal,
|
||||
),
|
||||
"perth": PerthAdapter(
|
||||
name="perth",
|
||||
source_file=perth_oracle_source,
|
||||
),
|
||||
"videoseal": VideoSealAdapter(
|
||||
name="videoseal",
|
||||
source_file=video_oracle_source,
|
||||
@@ -585,13 +657,13 @@ def _audioseal_detector() -> object:
|
||||
|
||||
|
||||
@cache
|
||||
def _videoseal_model() -> object:
|
||||
def _oracle_model(module_name: str) -> object:
|
||||
"""Load and cache one development-only oracle model."""
|
||||
scripts_dir = Path(__file__).resolve().parent
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
from videoseal_oracle import load_model
|
||||
|
||||
return load_model()
|
||||
module = importlib.import_module(module_name)
|
||||
return module.load_model()
|
||||
|
||||
|
||||
def _image_fidelity(
|
||||
@@ -745,6 +817,10 @@ def _detection_record(
|
||||
"max_bit_accuracy": max(values) if values else None,
|
||||
"per_frame_bit_accuracy": [round(value, 6) for value in values],
|
||||
}
|
||||
if outcome.message_bit_accuracy is not None:
|
||||
record["message_bit_accuracy"] = float(outcome.message_bit_accuracy)
|
||||
if outcome.score is not None:
|
||||
record["score"] = float(outcome.score)
|
||||
if outcome.error is not None:
|
||||
record["error"] = outcome.error
|
||||
return record
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "remove-ai-watermarks",
|
||||
"displayName": "Remove AI Watermarks",
|
||||
"version": "1.0.9",
|
||||
"version": "1.0.10",
|
||||
"description": "Identify and remove AI provenance marks from images and video the user generated or edited, using the remove-ai-watermarks CLI.",
|
||||
"author": {
|
||||
"name": "wiltodelta",
|
||||
|
||||
@@ -5,7 +5,7 @@ license: Apache-2.0
|
||||
compatibility: Python 3.11-3.14. Invisible image removal needs NVIDIA CUDA. Some video writes need ffmpeg. Installer may be uv, pipx, or pip.
|
||||
metadata:
|
||||
author: wiltodelta
|
||||
version: "1.0.9"
|
||||
version: "1.0.10"
|
||||
homepage: https://raiw.cc
|
||||
repository: https://github.com/wiltodelta/remove-ai-watermarks
|
||||
---
|
||||
|
||||
@@ -22,7 +22,7 @@ remove-ai-watermarks visible image.png --keep-metadata -o clean.png
|
||||
|
||||
Image mark keys: `gemini`, `doubao`, `jimeng`, `qwen`, `wan`, `kling`, `yuanbao`,
|
||||
`samsung`, `runninghub`, `baidu`, `liblib`, `liblib_pill`, `microsoft`,
|
||||
`generic_ai_label`, `openart`, `jimeng_pill`. Default
|
||||
`generic_ai_label`, `openart`, `jimeng_pill`. `wan` is new (since CLI 0.43.0). Default
|
||||
`--mark auto` removes every selected match. `microsoft` is the top-right AI
|
||||
badge, separate from the invisible InvisMark watermark on the same vendor's
|
||||
images. `jimeng_pill` is a weak detector and needs corroboration. The compact
|
||||
@@ -125,7 +125,7 @@ remove-ai-watermarks video batch ./videos --mode all
|
||||
```
|
||||
|
||||
Video mark keys: `sora`, `veo`, `seedance`, `doubao`, `dola`, `hailuo`,
|
||||
`vidu`, `kling`. Default `--mark auto` picks the first temporally stable match in that
|
||||
`vidu`, `kling`; `vidu` is new (since CLI 0.43.0). Default `--mark auto` picks the first temporally stable match in that
|
||||
order.
|
||||
|
||||
`video invisible` and `video all --invisible` accept MP4, MOV, and M4V only.
|
||||
|
||||
@@ -32,7 +32,7 @@ _GPU_MARKERS = ("GPU ", "gpu:")
|
||||
# told an agent its CLI was current right before it typed a value that build
|
||||
# rejects. tests/test_agent_skill.py pins it against the package version and
|
||||
# against every "(since CLI X.Y.Z)" annotation in the skill.
|
||||
MIN_CLI_VERSION = (0, 40, 5)
|
||||
MIN_CLI_VERSION = (0, 43, 0)
|
||||
# Markers a build without the pixel stack prints. The first is the current CLI's own
|
||||
# install hint; the rest are what older builds emit before the guard existed, and the
|
||||
# Homebrew formula ships exactly such a build.
|
||||
|
||||
@@ -35,7 +35,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.42.0"
|
||||
__version__ = "0.43.0"
|
||||
|
||||
__all__ = [
|
||||
"BatchItemResult",
|
||||
|
||||
@@ -151,10 +151,7 @@ def test_temporal_study_measures_only_saved_decodes(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(study, "encode_clip", encode)
|
||||
monkeypatch.setattr(study, "apply_scale", lambda _f, _p, d: encode(_f, d / "scale.mp4", raw))
|
||||
monkeypatch.setattr(study, "apply_fps_half", lambda _f, _p, d: encode(_f, d / "fps.mp4", raw))
|
||||
import watermark_benchmark as benchmark
|
||||
|
||||
monkeypatch.setattr(benchmark, "_decode_video", lambda path: decoded[path])
|
||||
monkeypatch.setattr(study, "_decode_video", lambda path: decoded[path])
|
||||
monkeypatch.setattr(study, "decode_video_artifact", lambda path: (decoded[path], "f" * 64))
|
||||
monkeypatch.setattr(study, "read", read)
|
||||
monkeypatch.setattr(study, "read_aggregation_matrix", matrix)
|
||||
monkeypatch.setattr(study, "render_matrix", lambda rows: "")
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Tests for the pinned Perth oracle's pure contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import perth_oracle # noqa: E402
|
||||
|
||||
|
||||
def test_detection_rule_is_upstream_round_at_one_half() -> None:
|
||||
assert perth_oracle.PerthReading(0.499).detected is False
|
||||
assert perth_oracle.PerthReading(0.5).detected is False
|
||||
assert perth_oracle.PerthReading(0.501).detected is True
|
||||
|
||||
|
||||
def test_checkpoint_and_source_are_immutable_pins() -> None:
|
||||
assert len(perth_oracle.SOURCE_COMMIT) == 40
|
||||
assert len(perth_oracle.CHECKPOINT_SHA256) == 64
|
||||
assert all(char in "0123456789abcdef" for char in perth_oracle.CHECKPOINT_SHA256)
|
||||
|
||||
|
||||
def test_perth_has_no_message_payload_contract() -> None:
|
||||
reading = perth_oracle.PerthReading(0.75)
|
||||
|
||||
assert not hasattr(reading, "decoded_bits")
|
||||
assert not hasattr(perth_oracle, "message_bits")
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for the pinned PixelSeal oracle's pure helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import pixelseal_oracle # noqa: E402
|
||||
|
||||
|
||||
class TestMessage:
|
||||
def test_message_is_deterministic_and_256_bits(self) -> None:
|
||||
first = pixelseal_oracle.message_bits()
|
||||
second = pixelseal_oracle.message_bits()
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 256
|
||||
assert set(first) <= {0, 1}
|
||||
|
||||
def test_seed_changes_the_message(self) -> None:
|
||||
assert pixelseal_oracle.message_bits(seed=8) != pixelseal_oracle.message_bits(seed=7)
|
||||
|
||||
|
||||
class TestReading:
|
||||
def test_label_is_hex_of_decoded_bits(self) -> None:
|
||||
reading = pixelseal_oracle.PixelSealReading(1.0, tuple([1] * 8 + [0] * 8))
|
||||
|
||||
assert reading.label == "ff00"
|
||||
|
||||
def test_detection_rule_is_the_documented_adapter_threshold(self) -> None:
|
||||
assert pixelseal_oracle.PixelSealReading(0.899, ()).detected is False
|
||||
assert pixelseal_oracle.PixelSealReading(0.9, ()).detected is True
|
||||
|
||||
|
||||
class TestPins:
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[pixelseal_oracle.CHECKPOINT_SHA256, pixelseal_oracle.MODEL_CARD_SHA256],
|
||||
)
|
||||
def test_digests_are_sha256(self, value: str) -> None:
|
||||
assert len(value) == 64
|
||||
assert all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
def test_source_commit_is_pinned(self) -> None:
|
||||
assert len(pixelseal_oracle.SOURCE_COMMIT) == 40
|
||||
|
||||
|
||||
def test_cache_dir_env_override_is_honored(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
original_cache = pixelseal_oracle.CACHE_DIR
|
||||
try:
|
||||
with monkeypatch.context() as environment:
|
||||
environment.setenv("PIXELSEAL_CACHE_DIR", str(tmp_path / "cache"))
|
||||
importlib.reload(pixelseal_oracle)
|
||||
assert tmp_path / "cache" == pixelseal_oracle.CACHE_DIR
|
||||
finally:
|
||||
importlib.reload(pixelseal_oracle)
|
||||
assert original_cache == pixelseal_oracle.CACHE_DIR
|
||||
@@ -30,6 +30,10 @@ def test_catalog_covers_every_project_vendor_oracle_surface() -> None:
|
||||
assert oracles.SURFACES["gemini-web"].automation == "real_browser"
|
||||
assert oracles.SURFACES["meta-web"].automation == "playwright"
|
||||
assert oracles.SURFACES["gemini-web"].media_types == ("image", "video", "audio")
|
||||
assert oracles.SURFACES["gemini-web"].prompt == (
|
||||
"Does this image/video/audio contain a SynthID watermark? Use the SynthID verification tool only; "
|
||||
"do not infer from visible content, logos, text, or metadata."
|
||||
)
|
||||
|
||||
|
||||
def test_default_slot_config_is_repository_local() -> None:
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Pure-logic guards for the development-only re-watermarking study."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks import api
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
|
||||
import rewatermarking_study as study
|
||||
import rewatermarking_synthid_candidates as synthid_candidates
|
||||
import rewatermarking_video_study as video_study
|
||||
|
||||
|
||||
def test_bit_accuracy_requires_equal_nonempty_messages() -> None:
|
||||
assert study.bit_accuracy((0, 1, 1, 0), (0, 0, 1, 0)) == pytest.approx(0.75)
|
||||
with pytest.raises(ValueError, match="non-empty"):
|
||||
study.bit_accuracy((), ())
|
||||
with pytest.raises(ValueError, match="equal length"):
|
||||
study.bit_accuracy((0,), (0, 1))
|
||||
|
||||
|
||||
def test_image_fidelity_uses_decoded_8bit_pixels() -> None:
|
||||
reference = np.zeros((2, 2, 3), dtype=np.uint8)
|
||||
artifact = np.full((2, 2, 3), 10, dtype=np.uint8)
|
||||
metrics = study.image_fidelity(reference, artifact)
|
||||
assert metrics["mae_8bit"] == pytest.approx(10.0)
|
||||
assert metrics["changed_fraction"] == pytest.approx(1.0)
|
||||
assert metrics["psnr_db"] == pytest.approx(28.1308036087)
|
||||
|
||||
|
||||
def test_summary_excludes_failed_positive_controls() -> None:
|
||||
def row(*, scheme: str, control: bool, success: bool, psnr: float) -> dict[str, object]:
|
||||
return {
|
||||
"scheme": scheme,
|
||||
"marked": {"detected_as_a": control, "bit_accuracy_vs_a": 1.0 if control else 0.5},
|
||||
"rewatermarked": {
|
||||
"bit_accuracy_vs_a": 0.5,
|
||||
"bit_accuracy_vs_b": 1.0,
|
||||
"fidelity_from_clean": {"psnr_db": psnr},
|
||||
"fidelity_from_marked": {"psnr_db": psnr + 1},
|
||||
},
|
||||
"attack_success": success,
|
||||
}
|
||||
|
||||
summary = study.summarize(
|
||||
[
|
||||
row(scheme="example", control=True, success=True, psnr=40.0),
|
||||
row(scheme="example", control=False, success=True, psnr=10.0),
|
||||
]
|
||||
)["example"]
|
||||
assert summary["cases"] == 2
|
||||
assert summary["valid_positive_controls"] == 1
|
||||
assert summary["attack_successes"] == 1
|
||||
assert summary["rewatermarked_psnr_from_clean"]["median"] == pytest.approx(40.0)
|
||||
|
||||
|
||||
def test_video_fidelity_uses_mean_per_frame_psnr() -> None:
|
||||
reference = np.zeros((2, 2, 2, 3), dtype=np.float32)
|
||||
artifact = np.stack(
|
||||
(
|
||||
np.full((2, 2, 3), 0.1, dtype=np.float32),
|
||||
np.full((2, 2, 3), 0.2, dtype=np.float32),
|
||||
)
|
||||
)
|
||||
assert video_study.mean_video_psnr(reference, artifact) == pytest.approx(16.9897, abs=1e-4)
|
||||
assert video_study.mean_video_psnr(reference, reference) is None
|
||||
with pytest.raises(ValueError, match="share"):
|
||||
video_study.mean_video_psnr(reference, artifact[:1])
|
||||
|
||||
|
||||
def test_synthid_candidate_sources_are_manifest_pinned_and_positive() -> None:
|
||||
sources = synthid_candidates.selected_sources()
|
||||
assert set(sources) == {"openai", "gemini"}
|
||||
for provider, source in sources.items():
|
||||
assert source.is_file()
|
||||
assert synthid_candidates.sha256_file(source) == synthid_candidates.SELECTED[provider]
|
||||
|
||||
|
||||
def test_synthid_candidate_bit_accuracy() -> None:
|
||||
assert synthid_candidates.bit_accuracy((1, 0, 1), (1, 1, 1)) == pytest.approx(2 / 3)
|
||||
with pytest.raises(ValueError, match="length"):
|
||||
synthid_candidates.bit_accuracy((1,), (1, 0))
|
||||
|
||||
|
||||
def test_synthid_control_removes_visible_mark_and_metadata_before_oracle(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.png"
|
||||
output = tmp_path / "control.png"
|
||||
original = np.full((8, 9, 3), 80, dtype=np.uint8)
|
||||
cleaned = original.copy()
|
||||
cleaned[6:, 7:] = 70
|
||||
Image.fromarray(original, "RGB").save(source)
|
||||
|
||||
def fake_remove(
|
||||
actual_source: Path,
|
||||
actual_output: Path,
|
||||
**kwargs: object,
|
||||
) -> SimpleNamespace:
|
||||
assert actual_source == source
|
||||
assert actual_output == output
|
||||
assert kwargs == {
|
||||
"sensitivity": "auto",
|
||||
"backend": "cv2",
|
||||
"strip_metadata": True,
|
||||
"write_noop": True,
|
||||
}
|
||||
Image.fromarray(cleaned, "RGB").save(actual_output)
|
||||
return SimpleNamespace(
|
||||
status="cleaned",
|
||||
marks=(SimpleNamespace(key="gemini"),),
|
||||
labels=["Google Gemini visible watermark (sparkle)"],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(api, "remove_visible_detailed", fake_remove)
|
||||
|
||||
pixels, preflight = synthid_candidates.cleaned_control(source, output, required_visible_key="gemini")
|
||||
|
||||
assert np.array_equal(pixels, cleaned)
|
||||
assert preflight["status"] == "cleaned"
|
||||
assert preflight["removed_keys"] == ["gemini"]
|
||||
assert preflight["changed_pixel_fraction"] == pytest.approx(4 / 72)
|
||||
|
||||
|
||||
def test_synthid_control_rejects_unvalidated_visible_cleanup(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
source = tmp_path / "source.png"
|
||||
output = tmp_path / "control.png"
|
||||
Image.new("RGB", (8, 8), (80, 80, 80)).save(source)
|
||||
monkeypatch.setattr(
|
||||
api,
|
||||
"remove_visible_detailed",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(status="partial", marks=(), labels=[]),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not validated"):
|
||||
synthid_candidates.cleaned_control(source, output, required_visible_key="gemini")
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
@@ -96,3 +97,27 @@ class TestAggregationFormulas:
|
||||
|
||||
with pytest.raises(ValueError, match="unknown aggregation"):
|
||||
videoseal_oracle._aggregate_bit_preds(torch.zeros(2, 4), "median")
|
||||
|
||||
|
||||
class TestImageMode:
|
||||
def test_image_helpers_preserve_layout_and_decode_message_columns(self) -> None:
|
||||
torch = pytest.importorskip("torch")
|
||||
message = (1, 0, 1, 0)
|
||||
|
||||
class Model:
|
||||
def embed(self, tensor: object, bits: object, *, is_video: bool) -> object:
|
||||
assert is_video is False
|
||||
torch.testing.assert_close(bits, torch.tensor([message], dtype=torch.float32))
|
||||
return tensor
|
||||
|
||||
def detect(self, tensor: object, *, is_video: bool) -> object:
|
||||
assert is_video is False
|
||||
assert tuple(tensor.shape) == (1, 3, 2, 4)
|
||||
return {"preds": torch.tensor([[9.0, 1.0, -1.0, 1.0, -1.0]])}
|
||||
|
||||
pixels = np.arange(24, dtype=np.float32).reshape(2, 4, 3) / 24.0
|
||||
|
||||
marked = videoseal_oracle.embed_image(Model(), pixels, message)
|
||||
|
||||
np.testing.assert_allclose(marked, pixels)
|
||||
assert videoseal_oracle.decode_image(Model(), marked) == message
|
||||
@@ -393,10 +393,12 @@ def test_jsonl_writer_refuses_overwrite_and_emits_strict_json(tmp_path: Path) ->
|
||||
def test_builtin_adapters_name_their_exact_source_files() -> None:
|
||||
adapters = MODULE.default_adapters()
|
||||
|
||||
assert set(adapters) == {"dwt-dct", "trustmark", "audioseal", "videoseal"}
|
||||
assert set(adapters) == {"dwt-dct", "trustmark", "pixelseal", "audioseal", "perth", "videoseal"}
|
||||
assert adapters["dwt-dct"].source_file.name == "invisible_watermark.py"
|
||||
assert adapters["trustmark"].source_file.name == "trustmark_detector.py"
|
||||
assert adapters["pixelseal"].source_file.name == "pixelseal_oracle.py"
|
||||
assert adapters["audioseal"].source_file.name == "audioseal_oracle.py"
|
||||
assert adapters["perth"].source_file.name == "perth_oracle.py"
|
||||
assert adapters["videoseal"].source_file.name == "videoseal_oracle.py"
|
||||
|
||||
|
||||
@@ -741,6 +743,20 @@ def test_detection_record_omits_temporal_for_non_temporal_adapters() -> None:
|
||||
assert "temporal" not in MODULE._detection_record(outcome, "not_detected", None)
|
||||
|
||||
|
||||
def test_detection_record_carries_message_accuracy_and_presence_score() -> None:
|
||||
outcome = MODULE.DetectorOutcome(
|
||||
status="detected",
|
||||
label="fixed-message",
|
||||
message_bit_accuracy=0.996,
|
||||
score=0.75,
|
||||
)
|
||||
|
||||
record = MODULE._detection_record(outcome, "detected", 2.0)
|
||||
|
||||
assert record["message_bit_accuracy"] == 0.996
|
||||
assert record["score"] == 0.75
|
||||
|
||||
|
||||
def test_run_benchmark_records_videoseal_temporal_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
pytest.skip("video rows decode through the system ffmpeg")
|
||||
|
||||
Reference in new issue
Block a user