From 5bfed00553ae19a36fc133efd755e39e5f885451 Mon Sep 17 00:00:00 2001 From: test-user Date: Wed, 27 May 2026 18:15:48 -0700 Subject: [PATCH] feat(metadata): blank AI-label XMP inside the HEIF/AVIF meta box (v0.6.9) HEIF/AVIF store XMP as a meta-box `mime` item whose bytes live in mdat/idat, out of reach of the top-level uuid/jumb box stripper. An AI-label XMP packet there (TC260 AIGC, IPTC "Made with AI", IPTC 2025.1) was therefore left in place. isobmff.blank_ai_xmp_packets locates each XMP packet by its delimiters and, if it carries an AI marker (_AI_LABEL_MARKERS), overwrites it with spaces of the SAME length. Equal length means no box size or iloc offset shifts -- the coded image stays bit-for-bit intact, the item stays structurally valid, only the AI label content is destroyed. Plain (non-AI) XMP is left alone, mirroring the top-level XMP-uuid content match. Wired into remove_ai_metadata's ISOBMFF branch after strip_c2pa_boxes. Chosen over exiftool (a non-bundled binary dep) to stay pure-Python and droplet-compatible; over full iinf/iloc surgery to avoid offset-rewrite corruption risk. The AI labels we target are all XMP, so this closes the practical gap. An Exif *item* inside the meta box (rare) still needs iinf/iloc surgery or exiftool -- documented. 4 new tests (TestMetaBoxXmpBlanking): AI packet blanked (same length, marker gone, surrounding image bytes intact), plain XMP preserved, no-packet no-op, and end-to-end remove_ai_metadata on a .heic. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 4 +- README.md | 4 +- pyproject.toml | 2 +- src/remove_ai_watermarks/__init__.py | 2 +- src/remove_ai_watermarks/metadata.py | 13 ++++- src/remove_ai_watermarks/noai/isobmff.py | 41 ++++++++++++++-- tests/test_metadata.py | 62 ++++++++++++++++++++++++ uv.lock | 2 +- 8 files changed, 118 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cd3783a..f4f7f87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ Who embeds what, and whether it is locally detectable (so we know which gaps are - **No detectable signal on download (correctly reported `unknown`):** **Recraft** (PNG export is a re-encoded design export — strips everything), **Krea hosting FLUX 2** (no imwatermark despite FLUX — the host omits the encoder, same as Stability's hosted SDXL), and Midjourney (embeds nothing). Lesson: the imwatermark detector only fires on *pristine* output from a pipeline that runs the encoder (diffusers default, official BFL), not from re-hosts (Krea/Stability) or re-encoded exports (Recraft/Canva). - **Invisible but NOT locally detectable (proprietary, API/oracle only — same wall as SynthID):** Amazon Titan Image Generator + Nova Canvas (Bedrock `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), NVIDIA Cosmos (SynthID video). No local detector possible; treat like SynthID. - **C2PA 2.4 "Durable Content Credentials" (April 2026; verified against the spec) raise the bar for metadata stripping.** 2.4 defines soft bindings (an invisible watermark or a content fingerprint) plus a server-side manifest repository and a new `c2pa.repository-receipt` assertion. Per the spec: "if a C2PA manifest is removed from an asset, but a copy of that manifest remains in a provenance store elsewhere, the manifest and asset may be matched using available soft bindings." So our local `metadata --remove` deletes the *embedded* manifest, but a fingerprint/watermark soft binding can still re-link the image to its manifest in a repository server-side. Stripping the file is becoming necessary-but-not-sufficient against durable provenance. (Our parsers target the stable embedded-manifest format documented in C2PA 2.1 §11; that format is unchanged in 2.4 -- the new pieces are repository/soft-binding infra, not the on-file box layout, so no parser change is implied.) Spec: https://spec.c2pa.org/specifications/specifications/2.4/specs/C2PA_Specification.html We now READ the soft-binding `alg` (`C2PA_SOFT_BINDINGS` / `soft_binding_vendors_in`) to name the forensic-watermark vendor, and locally DECODE the one open scheme, Adobe TrustMark (`trustmark_detector`); the rest (Digimarc/Imatag/Steg.AI/...) stay name-only (proprietary decoders). -- **Built 2026-05-26 (this batch):** soft-binding `alg` vendor detection; IPTC Photo Metadata 2025.1 AI-disclosure fields (`AISystemUsed` etc.); **video C2PA metadata** detect + strip for MP4/MOV/M4V (free — `isobmff.py` is format-agnostic, MP4 is ISOBMFF); Adobe TrustMark open decoder. NOT done (out of cheap reach, per the feasibility review): visible video-logo removal (needs a video frame pipeline) and audio (SynthID/ElevenLabs/Resemble/Suno all oracle-only or unmarked). **Box detection window — now handled (v0.6.8):** detection no longer relies on a fixed first-MB read. `metadata.scan_head(path, size)` reads the first `size` bytes and, for ISOBMFF, appends the payloads of late provenance boxes found by `isobmff.scan_c2pa_region` (a file-seeking top-level box walker that skips past `mdat` by size without reading it), so a C2PA/AIGC/IPTC manifest placed AFTER a large `mdat` in a streaming/non-faststart MP4 is now caught. Every C2PA/marker byte scan (`has_ai_metadata`, `aigc_label`, `iptc_ai_system`, `synthid_source`, `exif_generator` XMP, `get_ai_metadata` soft-binding, and `identify`) goes through `scan_head`; it is behavior-neutral for non-ISOBMFF inputs (exactly `f.read(size)`). The remaining gap is EXIF/XMP stored as items *inside the `meta` box* (still needs meta-box surgery / exiftool). +- **Built 2026-05-26 (this batch):** soft-binding `alg` vendor detection; IPTC Photo Metadata 2025.1 AI-disclosure fields (`AISystemUsed` etc.); **video C2PA metadata** detect + strip for MP4/MOV/M4V (free — `isobmff.py` is format-agnostic, MP4 is ISOBMFF); Adobe TrustMark open decoder. NOT done (out of cheap reach, per the feasibility review): visible video-logo removal (needs a video frame pipeline) and audio (SynthID/ElevenLabs/Resemble/Suno all oracle-only or unmarked). **Box detection window — now handled (v0.6.8):** detection no longer relies on a fixed first-MB read. `metadata.scan_head(path, size)` reads the first `size` bytes and, for ISOBMFF, appends the payloads of late provenance boxes found by `isobmff.scan_c2pa_region` (a file-seeking top-level box walker that skips past `mdat` by size without reading it), so a C2PA/AIGC/IPTC manifest placed AFTER a large `mdat` in a streaming/non-faststart MP4 is now caught. Every C2PA/marker byte scan (`has_ai_metadata`, `aigc_label`, `iptc_ai_system`, `synthid_source`, `exif_generator` XMP, `get_ai_metadata` soft-binding, and `identify`) goes through `scan_head`; it is behavior-neutral for non-ISOBMFF inputs (exactly `f.read(size)`). **Meta-box XMP removal — now handled (v0.6.9):** an AI-label XMP packet stored as a meta-box `mime` item (HEIF/AVIF; out of reach of the top-level box stripper) is blanked in place by `isobmff.blank_ai_xmp_packets` — it locates the packet by its `` delimiters and, if it carries an AI marker (`_AI_LABEL_MARKERS`), overwrites it with spaces of the SAME length, so box sizes / `iloc` offsets stay valid and the coded image is untouched (selective: plain non-AI XMP is left alone, mirroring the top-level uuid logic). Wired into `remove_ai_metadata`'s ISOBMFF branch after `strip_c2pa_boxes`. The remaining gap is an `Exif` meta-box *item* (rare; the AI labels are XMP) — still needs `iinf`/`iloc` surgery or exiftool. - **Regulatory driver (context, not a code change):** AI-content labeling mandates are expanding, which pushes more generators toward exactly the C2PA + watermark signals we read. The full per-jurisdiction table lives in README "## Legal" -- keep it there, not duplicated here. Newly added + primary-source verified 2026-05-26: **EU AI Act Article 50** machine-readable marking applicable **2026-08-02** (verified against the article text); **South Korea AI Framework Act Art. 31(3)** in force since **22 January 2026** (verified via Kim & Chang + FPF/Korea Times; Enforcement Decree accepts an invisible-watermark label); **California AB 853** (amends the CA AI Transparency Act) latent-disclosure duty operative **2026-08-02**, requiring a disclosure "permanent or extraordinarily difficult to remove" (verified against the leginfo bill text -- this is the exact disclosure our tool strips); **India IT Amendment Rules 2026** in force **2026-02-20** (verified via Chambers), which prominently-label + permanent-provenance-id all synthetic media AND **expressly prohibit removing/suppressing the label or metadata** -- the first major all-content removal ban outside China. **Removal liability (README "## Legal" disclaimer):** the tool is lawful general-purpose software; liability sits with the remover and is intent-gated -- downstream acts (fraud/deception/IP), plus US DMCA 17 USC 1202 (removing copyright-management info to conceal infringement), plus the removal-as-such bans in China + India. When extending the README table, verify each date/article against the statute/bill text before committing, not against search summaries. ## Known limitations @@ -63,7 +63,7 @@ Who embeds what, and whether it is locally detectable (so we know which gaps are - Pyright first run is slow (2-3 min) due to ML deps (torch/diffusers/transformers stubs); full-project `uv run pyright` can stall for many minutes — scope it to changed files. - `ultralytics` monkey-patches `PIL.Image.open` and tries to autoload `pi_heif`. When `pi_heif` is missing, opening files raises `ModuleNotFoundError`, not `UnidentifiedImageError`. Code that opens user-supplied or unknown-format files should `except Exception`, not just `OSError`/`UnidentifiedImageError`. - **rich `console.print` parses `[word]` as a style tag and silently drops unknown ones.** A literal bracketed token in a print string disappears: `pip install 'remove-ai-watermarks[gpu]'` rendered as `...remove-ai-watermarks'` (the `[gpu]` extra eaten), which sent users a broken install command (surfaced via #19). Escape the literal bracket as `\[gpu]` (in a normal Python string that is `"\\[gpu]"`) in any rich string carrying user-facing brackets. Regression-guarded by `tests/test_cli.py::TestGpuHintMarkup`. -- Metadata detection for AVIF/HEIF/JPEG-XL relies on a binary scan for `C2PA_UUID` + `IPTC_AI_MARKERS`, plus EXIF `Software` / XMP `CreatorTool` generator tags via `metadata.exif_generator` (validated with synthesized AVIF/JPEG fixtures + an XMP raw-scan fixture). C2PA removal in those containers is implemented via `noai/isobmff.py` (top-level ``uuid`` / ``jumb`` box stripper, no re-encoding), which now also drops a top-level XMP ``uuid`` box that carries an AI label (matched by AI-marker content, not by the XMP UUID, so byte-order-robust) and covers MP4/MOV/M4V/M4A by content sniff. **Non-ISOBMFF audio/video removal is via ffmpeg** (`_FFMPEG_STRIP_EXTS` -> `_strip_with_ffmpeg`): WebM/Matroska (EBML), MP3 (ID3), WAV/FLAC/OGG (RIFF/Vorbis) are stripped losslessly with `ffmpeg -map_metadata -1 -map_chapters -1 -c copy` (codec data untouched). Requires ffmpeg on PATH; raises `RuntimeError` if absent or if ffmpeg can't parse the file. Verified end-to-end (a real ffmpeg-made WAV/MP3 with a `title=Suno AI` tag -> tag gone, audio bytes preserved). **Still NOT built (deliberate):** EXIF/XMP stored as *items inside the ``meta`` box* (typical for AVIF/HEIF images) needs meta-box surgery (iinf/iloc edit + mdat splice) with corruption risk -- exiftool would do it but is a non-installed binary dep, so it stays a documented gap. **Audio watermark DETECTION (Resemble PerTh) was evaluated and NOT built (2026-05-26):** `resemble-perth`'s `PerthImplicitWatermarker.get_watermark()` returns a raw bit-array with **no presence/confidence flag** (clean audio decodes to arbitrary bits too), so reliably distinguishing watermarked-from-clean needs either Resemble's fixed payload or a confidence API -- neither is public, and there's no real Resemble sample to calibrate against. Same wall-class as the SynthID pixel detector: the decode exists, reliable presence-detection does not. (perth's top-level `PerthImplicitWatermarker` is also gated to None unless `librosa` is importable.) +- Metadata detection for AVIF/HEIF/JPEG-XL relies on a binary scan for `C2PA_UUID` + `IPTC_AI_MARKERS`, plus EXIF `Software` / XMP `CreatorTool` generator tags via `metadata.exif_generator` (validated with synthesized AVIF/JPEG fixtures + an XMP raw-scan fixture). C2PA removal in those containers is implemented via `noai/isobmff.py` (top-level ``uuid`` / ``jumb`` box stripper, no re-encoding), which now also drops a top-level XMP ``uuid`` box that carries an AI label (matched by AI-marker content, not by the XMP UUID, so byte-order-robust) and covers MP4/MOV/M4V/M4A by content sniff. **Non-ISOBMFF audio/video removal is via ffmpeg** (`_FFMPEG_STRIP_EXTS` -> `_strip_with_ffmpeg`): WebM/Matroska (EBML), MP3 (ID3), WAV/FLAC/OGG (RIFF/Vorbis) are stripped losslessly with `ffmpeg -map_metadata -1 -map_chapters -1 -c copy` (codec data untouched). Requires ffmpeg on PATH; raises `RuntimeError` if absent or if ffmpeg can't parse the file. Verified end-to-end (a real ffmpeg-made WAV/MP3 with a `title=Suno AI` tag -> tag gone, audio bytes preserved). **Meta-box XMP now handled (`isobmff.blank_ai_xmp_packets`, v0.6.9):** an AI-label XMP packet stored as a meta-box `mime` item (AVIF/HEIF) is blanked in place (overwritten with spaces of the same length, so `iloc` offsets and the coded image stay valid). **Still NOT built:** an `Exif` *item* inside the `meta` box (rare -- AI labels are XMP) needs full `iinf`/`iloc` surgery (offset rewrite) with corruption risk -- exiftool (R/W/C for HEIC/AVIF EXIF+XMP, verified on exiftool.org 2026-05-27) would do it but is a non-installed binary dep, so it stays a documented gap. **Audio watermark DETECTION (Resemble PerTh) was evaluated and NOT built (2026-05-26):** `resemble-perth`'s `PerthImplicitWatermarker.get_watermark()` returns a raw bit-array with **no presence/confidence flag** (clean audio decodes to arbitrary bits too), so reliably distinguishing watermarked-from-clean needs either Resemble's fixed payload or a confidence API -- neither is public, and there's no real Resemble sample to calibrate against. Same wall-class as the SynthID pixel detector: the decode exists, reliable presence-detection does not. (perth's top-level `PerthImplicitWatermarker` is also gated to None unless `librosa` is importable.) - **SynthID detection is metadata-only.** There is no reliable *local* detector of the SynthID *pixel* watermark — Google's decoder is proprietary, no public spec or API (only a waitlisted portal). Authoritative confirmation: Google DeepMind's own paper "SynthID-Image: Image watermarking at internet scale" (Gowal et al., arXiv:2510.09263) states the verification service is restricted to "trusted testers" and does not release detector weights or a reproducible algorithm — so a local pixel detector is infeasible by design, not just unbuilt. https://arxiv.org/abs/2510.09263 We detect SynthID by its C2PA companion (`synthid_source` / `SYNTHID_C2PA_ISSUERS`), which is reliable while the manifest is intact but says nothing once C2PA is stripped. **Surface-dependent blind spot (verified 2026-05-24):** the same Google model emits different metadata per surface -- the Gemini *app* wraps outputs in Google C2PA, but the *API/playground* (AI Studio, Nano Banana / gemini-2.5-flash-image) emits the SynthID *pixel* watermark (confirmed via the Gemini-app oracle) + the visible sparkle but **no C2PA/IPTC at all**, so `synthid_source` returns None despite SynthID being present. Only the pixel oracle or the visible-sparkle detector catches those. (Meta AI is another surface mismatch: it writes the IPTC `digitalSourceType=trainedAlgorithmicMedia` marker, not C2PA and not SynthID.) Google→SynthID is long-standing; OpenAI→SynthID is confirmed by OpenAI's Help Center (ChatGPT/Codex/API "include both C2PA metadata and SynthID watermarks", updated 2026-05-21) but time-gated (pre-rollout OpenAI images carry C2PA without SynthID), so the OpenAI verdict is hedged "likely". Oracles: Gemini app "Verify with SynthID" (Google), openai.com/verify (OpenAI). The spectral phase-coherence approach from `github.com/aloshdenny/reverse-SynthID` was evaluated (May 2026) and **does not work for real-content detection**: on its own shipped codebook + validation set, watermarked and cleaned images were indistinguishable (conf within noise, cleaned often higher); it only fires on pure-black 1024x1024 reference images at exact resolution (the controlled case it was calibrated on). The README's "90% / conf=0.91" reproduces only in that lab condition. Do not build a production detector on it; if revisited, it is experimental/diagnostic only and needs a per-resolution, per-model reference corpus. A from-scratch gpt-image pilot (2026-05-24) confirmed this independently: 5 independent solid-black gpt-image outputs share a near-identical fixed signature (pairwise residual correlation **0.92**, avg-template retains 97% energy), so the watermark/carrier IS strongly present and consistent on flat content — but the carrier frequencies extracted from it do NOT discriminate real content (carrier-to-random ratio: cleaned 1.86 > watermarked 1.53; a non-gpt-image image scored highest at 3.67). The signature drowns in content texture. Net: a perfectly consistent solid-color signature still yields no real-content pixel detector with magnitude/carrier methods. A corpus discrimination test (2026-05-24, `scripts/synthid_pixel_probe.py`, raw zero-mean residual NCC) independently re-confirms this: at matched resolution, SynthID positives do NOT cluster apart from negatives (within-Gemini 0.07; at 1024 px pos-vs-neg >= pos-vs-pos). The only high correlations were near-duplicate *content* (5 ChatGPT renders of one prompt at ~0.92, while a distinct ChatGPT image scored ~0 against them) — content, not a carrier. The probe is solid-fills-only and EXPERIMENTAL/DIAGNOSTIC; do not use it on real content. **Correction (deeper re-examination 2026-05-25):** the carrier IS real on solid fills — the earlier "no carrier" was a *method* artifact of using spatial / FFT-magnitude NCC, which can't see it. The carrier is a fixed *phase* at specific low frequencies, so the right metric is **per-bin phase coherence**. On 8 white `gemini-2.5-flash-image` fills (generated via the reverse-SynthID trick: identity-edit prompt "Recreate this image exactly as it is" on a synthetic pure-white PNG — this bypasses the recitation block that rejects text prompts for pure colors), phase coherence at the white carriers `(0,±7..±12,±20..±23)` = **0.86** vs **0.31** random; single-image leave-one-out phase-match **+0.83** vs real photos **-0.24**. (Black `2.5-flash` fills clip to std≈0 — SynthID can't push values below 0, so no carrier in black; the repo's dark carriers come from nano-banana-pro.) **But it does not generalize:** (a) carriers are model-version + resolution + color specific — the repo's v4 codebook (built for `gemini-3.1-flash-image-preview` + `nano-banana-pro-preview`) scores ~0.527 on my 2.5-flash white fills, indistinguishable from negatives (~0.50), i.e. carriers shift across model versions and need a per-model codebook; (b) on real content (30 `2.5-flash` images) the carrier collapses — set phase coherence at carriers 0.37 ≈ random 0.42, and the repo's v4 detector gives content 0.518 ≈ negatives 0.504 (no separation; a faint +0.24 single-image lean is likely a brightness confound). Net: the spectral/phase approach is a real *controlled-fill* characterizer, NOT an arbitrary-real-content detector, and is brittle to model version. Metadata proxy + visible sparkle + online oracles remain the ceiling for real content. - **External AI-vs-real classifier models are out of scope (decided 2026-05-24).** Generic HuggingFace detectors (`Organika/sdxl-detector` Swin Transformer, `umm-maybe/AI-image-detector`, and fine-tunes) exist and report ~0.98 on their *own* SDXL-vs-real validation sets, but they are per-generator and the model cards themselves note degraded accuracy off-distribution; they are untested on gpt-image / Gemini Nano Banana (the metadata-stripped surfaces we care about), and our own light SDXL pass would likely defeat them the same way it defeats SynthID. Detection here stays local + signal-based (metadata + visible sparkle); do not add a bundled classifier dependency. - **SynthID v2 vs default pipeline:** the SDXL-based default profile (since May 2026) defeats SynthID v2. **Verified end-to-end (May 2026):** local SDXL run on a Gemini 3 Pro output, checked via the Gemini app's "Verify with SynthID" feature, returned "no SynthID watermark detected". Also confirmed against **OpenAI's** SynthID (2026-05-23): a fresh ChatGPT/gpt-image output read "SynthID detected" on openai.com/verify before the local SDXL run and "SynthID not detected" after (corpus regression chain: pos `4ef377bd` -> cleaned `47188e88`). The same configuration is used in raiw-app production (`fal-ai/fast-sdxl/image-to-image`, strength 0.05, steps 50, guidance 7.5, no pre-downscale). fal's own `llms.txt` for `fast-sdxl` names the base checkpoint as `stabilityai/stable-diffusion-xl-base-1.0` (verified 2026-05-25) -- the exact checkpoint the local CLI defaults to (`DEFAULT_MODEL_ID`). So the local `invisible` default is weight-for-weight identical to prod; "fast-sdxl" is fal's optimized serving, not different weights. After the native-resolution fix the local pipeline matches prod on weights + strength + steps + guidance + resolution. SD-1.5 dreamshaper at 768 px was previously the default and does NOT defeat v2 — verified empirically against the same feature (strength 0.04, 0.10, and elastic warp α∈{5,8} all flagged positive). That SD-1.5 path was removed; only `default` (SDXL) and `ctrlregen` profiles remain. **Scope of the claim: defeating the SynthID verifier is NOT the same as forensic invisibility.** "Removing the Watermark Is Not Enough: Forensic Stealth in Generative-AI Watermark Removal" (arXiv:2605.09203, 2026-05) shows that six removal attacks across four families (UnMarker, CtrlRegen+, WatermarkAttacker, etc.) all leave forensic traces: independent detectors flag *removal-processed* images vs genuinely-clean ones at **>98% TPR at 1% FPR**. So our SDXL pass makes the oracle read "SynthID not detected," but the output can still be classifiable as "an image that went through a removal pipeline." Do not over-claim "indistinguishable from a real photo." https://arxiv.org/abs/2605.09203 diff --git a/README.md b/README.md index f489481..e81852e 100644 --- a/README.md +++ b/README.md @@ -324,10 +324,10 @@ Tracked but not yet implemented: - **Grow the SynthID reference corpus** (`data/synthid_corpus/`) with oracle-labeled samples per model and resolution (Gemini app for Google, openai.com/verify for OpenAI). Prerequisite for any pixel-detector attempt and for an automated removal-regression set. - **Real non-PNG C2PA fixtures**. SynthID-source detection for JPEG / WebP / AVIF is currently covered only by synthetic byte blobs; replace with real vendor-emitted files to ground the binary-scan path. - **Maintenance debt**. Clear strict-pyright debt in `remove_ai_metadata` / `cli.py` (untyped piexif / PIL / click / rich) so `maintain.sh` can finish green. (`uv-secure` is already clean since `idna` was bumped to 3.16.) -- **AVIF / HEIF EXIF/XMP inside the `meta` box**. Removal already strips top-level C2PA `uuid` / JUMBF `jumb` boxes and any AI-labelled top-level XMP `uuid` box, and non-ISOBMFF audio/video (WebM, MP3, WAV, FLAC, OGG) is stripped losslessly via ffmpeg. Still open: EXIF/XMP stored as *items inside the `meta` box* (typical for AVIF/HEIF stills) — needs `meta`-box surgery (iinf/iloc + mdat splice) or `exiftool` (a non-bundled binary dependency). +- **AVIF / HEIF meta-box XMP** — *shipped (v0.6.9)*. An AI-label XMP packet stored as a `meta`-box `mime` item (HEIF/AVIF, out of reach of the top-level box stripper) is now blanked in place: located by its `` delimiters and, if it carries an AI marker, overwritten with spaces of the same length, so box sizes / `iloc` offsets stay valid and the coded image is untouched. Still open: an `Exif` *item* inside the `meta` box (rare — AI labels are XMP) needs full `iinf`/`iloc` surgery (offset rewrite) or `exiftool` (a non-bundled binary dependency). - **Multi-signal contradiction reporting ("Integrity Clash")** — *shipped (v0.6.7)*. `identify` now surfaces contradictions between independent provenance signals (two different AI vendors named by separate stamps, or camera-capture C2PA credentials next to AI-generation markers) as `integrity_clashes` (shown in red in the table view and in `--json`), rather than collapsing to a single verdict. Inspired by [arXiv:2603.02378](https://arxiv.org/abs/2603.02378). - **More C2PA device signers**. Leica, Nikon, Google Pixel, Sony, and Truepic are mapped (each verified against a real signed file). Canon and Samsung Galaxy (AI-edit) are deferred until a real signed sample surfaces — no public direct-download C2PA file exists for them today (upload-to-verify / news-agency-licensed only). -- **C2PA detection window for streaming MP4** — *shipped (v0.6.8)*. Detection no longer relies on a fixed first-MB read: for ISOBMFF containers it walks the top-level boxes (seeking past `mdat` by size) to find a C2PA / AIGC / IPTC manifest placed after the media data, so a streaming / non-faststart MP4 is caught. The remaining gap is EXIF/XMP stored as items *inside the `meta` box* (needs meta-box surgery or `exiftool`). +- **C2PA detection window for streaming MP4** — *shipped (v0.6.8)*. Detection no longer relies on a fixed first-MB read: for ISOBMFF containers it walks the top-level boxes (seeking past `mdat` by size) to find a C2PA / AIGC / IPTC manifest placed after the media data, so a streaming / non-faststart MP4 is caught. - **Resemble PerTh audio detection** — evaluated, not feasible with the public API: `get_watermark()` returns a raw bit array with no presence/confidence flag, so watermarked vs. clean audio can't be reliably separated without Resemble's fixed payload or a confidence service. Same wall as the SynthID pixel detector. - **Video pipeline (`noai-video`)**: per-frame inpainting and tracking for Sora 2 dynamic logo, Veo 3.1 badge, Kling, Runway. Separate package, not folded into this repo. diff --git a/pyproject.toml b/pyproject.toml index b121918..12ec91b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.6.8" +version = "0.6.9" description = "Remove visible and invisible AI watermarks from images (Gemini / Nano Banana, ChatGPT, Stable Diffusion)" readme = "README.md" requires-python = ">=3.10" diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index 189296f..52b451e 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -1,3 +1,3 @@ """Remove-AI-Watermarks: Unified tool for removing visible and invisible AI watermarks.""" -__version__ = "0.6.8" +__version__ = "0.6.9" diff --git a/src/remove_ai_watermarks/metadata.py b/src/remove_ai_watermarks/metadata.py index 546a8bd..d2f44d9 100644 --- a/src/remove_ai_watermarks/metadata.py +++ b/src/remove_ai_watermarks/metadata.py @@ -575,16 +575,25 @@ def remove_ai_metadata( # codestream bit-for-bit. MP4/MOV/M4A are ISOBMFF too, so the same top-level # uuid/jumb box walker applies. Route by suffix OR by an ``ftyp`` content # sniff, so a correctly-shaped container is handled whatever its extension. - from remove_ai_watermarks.noai.isobmff import is_isobmff, strip_c2pa_boxes + from remove_ai_watermarks.noai.isobmff import blank_ai_xmp_packets, is_isobmff, strip_c2pa_boxes with open(source_path, "rb") as f: head = f.read(12) if source_path.suffix.lower() in _ISOBMFF_EXTS or is_isobmff(head): data = source_path.read_bytes() + # Top-level uuid/jumb boxes (C2PA + AI-label XMP), then AI-label XMP that + # lives inside a meta-box ``mime`` item (HEIF/AVIF) -- blanked in place so + # box sizes and iloc offsets stay valid and the coded image is untouched. cleaned, stripped = strip_c2pa_boxes(data) + cleaned, blanked = blank_ai_xmp_packets(cleaned) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_bytes(cleaned) - logger.info("Stripped %d AI-provenance box(es) → %s", stripped, output_path) + logger.info( + "Stripped %d AI-provenance box(es), blanked %d meta-box XMP packet(s) → %s", + stripped, + blanked, + output_path, + ) return output_path # Non-ISOBMFF audio/video (WebM/Matroska EBML, MP3 ID3, WAV/FLAC/OGG): the diff --git a/src/remove_ai_watermarks/noai/isobmff.py b/src/remove_ai_watermarks/noai/isobmff.py index 105847d..a86f238 100644 --- a/src/remove_ai_watermarks/noai/isobmff.py +++ b/src/remove_ai_watermarks/noai/isobmff.py @@ -17,6 +17,7 @@ Reference: ISO/IEC 14496-12 (ISOBMFF) and C2PA 2.1 spec §11. from __future__ import annotations +import re import struct from typing import TYPE_CHECKING @@ -43,6 +44,12 @@ C2PA_BOX_TYPES: frozenset[bytes] = frozenset({b"uuid", b"jumb"}) # XMP (copyright, camera info) is kept. _AI_LABEL_MARKERS: tuple[bytes, ...] = AIGC_MARKERS + IPTC_AI_MARKERS + IPTC_AI_FIELD_MARKERS +# Adobe XMP packet delimiters (XMP spec part 3). In HEIF/AVIF the XMP packet +# sits inside a ``meta``-box ``mime`` item whose bytes live in ``mdat`` / ``idat``, +# out of reach of the top-level box stripper, so an AI-label packet there is +# blanked in place (see ``blank_ai_xmp_packets``). +_XMP_PACKET_RE = re.compile(rb"<\?xpacket begin=.*?<\?xpacket end=[^>]*?\?>", re.DOTALL) + def _iter_top_level_boxes(data: bytes) -> Iterator[tuple[int, int, bytes, int]]: """Yield ``(start, end, type, payload_offset)`` for each top-level box. @@ -145,9 +152,10 @@ def strip_c2pa_boxes(data: bytes) -> tuple[bytes, int]: All other boxes (incl. ``mdat`` / codestream) are emitted verbatim, so pixel and audio data is preserved bit-for-bit. Non-ISOBMFF input is returned unchanged. Despite the name this also covers MP4/MOV/M4A video and audio - (all ISOBMFF). NOTE: EXIF/XMP stored as *items inside the ``meta`` box* - (typical for AVIF/HEIF images) is not removed -- that needs meta-box surgery - and is a documented limitation. + (all ISOBMFF). NOTE: this drops only top-level boxes. An AI-label XMP packet + stored as an *item inside the ``meta`` box* (typical for AVIF/HEIF) is handled + separately by :func:`blank_ai_xmp_packets`; an ``Exif`` meta-box item is still + not removed (would need meta-box surgery) and remains a documented limitation. """ if not is_isobmff(data): return data, 0 @@ -167,3 +175,30 @@ def strip_c2pa_boxes(data: bytes) -> tuple[bytes, int]: continue out.extend(data[start:end]) return bytes(out), stripped + + +def blank_ai_xmp_packets(data: bytes) -> tuple[bytes, int]: + """Overwrite (with spaces, in place) any XMP packet carrying an AI-label + marker; return ``(data, blanked_count)``. + + HEIF/AVIF store XMP as a ``meta``-box ``mime`` item whose bytes live in + ``mdat`` / ``idat``, which ``strip_c2pa_boxes`` cannot remove without + meta-box surgery (``iinf`` / ``iloc`` rewrite). Instead, the XMP packet is + located by its ```` delimiters and, when it carries + an AI-label marker (TC260 AIGC / IPTC / IPTC-2025.1), overwritten with spaces. + Because the replacement is the **same length**, every box size and ``iloc`` + offset stays valid and the coded image data is untouched -- only the AI label + content is destroyed. Packets without an AI marker (plain copyright / camera + XMP) are left intact, mirroring the top-level XMP-``uuid`` content match. + """ + blanked = 0 + + def _scrub(match: re.Match[bytes]) -> bytes: + nonlocal blanked + packet = match.group() + if any(marker in packet for marker in _AI_LABEL_MARKERS): + blanked += 1 + return b" " * len(packet) + return packet + + return _XMP_PACKET_RE.sub(_scrub, data), blanked diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 842e603..245c515 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -712,6 +712,68 @@ class TestLateProvenanceBox: assert has_ai_metadata(p) is True +_AI_XMP = ( + b'' + b'{"Label":"1"}' + b'' +) +_PLAIN_XMP = ( + b'' + b"(c) me" + b'' +) + + +class TestMetaBoxXmpBlanking: + """HEIF/AVIF store XMP as a meta-box ``mime`` item (bytes in mdat/idat), out of + reach of the top-level box stripper. An AI-label XMP packet there is blanked + in place (same length -> iloc offsets and image data stay intact).""" + + def test_blanks_ai_packet_only(self): + from remove_ai_watermarks.noai.isobmff import blank_ai_xmp_packets + + before, after = b"IMG_BEFORE" * 4, b"IMG_AFTER" * 4 + data = before + _AI_XMP + after + _PLAIN_XMP + out, n = blank_ai_xmp_packets(data) + assert n == 1 + assert len(out) == len(data) # same length -> no offset shifts + assert b"TC260:AIGC" not in out # AI label destroyed + assert before in out # surrounding (image) bytes intact + assert after in out + assert b"dc:rights" in out # plain XMP left alone + + def test_no_packet_is_noop(self): + from remove_ai_watermarks.noai.isobmff import blank_ai_xmp_packets + + data = b"just some mdat bytes, no xmp here" + assert blank_ai_xmp_packets(data) == (data, 0) + + def test_plain_xmp_untouched(self): + from remove_ai_watermarks.noai.isobmff import blank_ai_xmp_packets + + out, n = blank_ai_xmp_packets(_PLAIN_XMP) + assert n == 0 + assert out == _PLAIN_XMP + + def test_remove_ai_metadata_blanks_meta_box_xmp(self, tmp_path: Path): + # End-to-end: a HEIF with an AI XMP packet inside mdat is cleaned without + # touching the surrounding (coded image) bytes or the file length. + heic_ftyp = b"\x00\x00\x00\x18ftypheic\x00\x00\x00\x00heicmif1" + img = b"CODEDIMAGE" * 8 + mdat = _box(b"mdat", img + _AI_XMP + img) + src = tmp_path / "ai.heic" + src.write_bytes(heic_ftyp + mdat) + assert has_ai_metadata(src) is True + + out = tmp_path / "clean.heic" + remove_ai_metadata(src, out) + res = out.read_bytes() + assert len(res) == src.stat().st_size # length preserved + assert b"TC260:AIGC" not in res + assert img in res # coded image bytes intact + assert has_ai_metadata(out) is False + + class TestIsobmffMetadataRemoval: """Container-level AI-provenance stripping across ISOBMFF image/video/audio.""" diff --git a/uv.lock b/uv.lock index 0b13897..3ee2e1a 100644 --- a/uv.lock +++ b/uv.lock @@ -2865,7 +2865,7 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.6.8" +version = "0.6.9" source = { editable = "." } dependencies = [ { name = "click" },