mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-07-30 19:21:37 +02:00
feat: SDXL default; AVIF/HEIF/JPEG-XL C2PA stripping
SD-1.5 dreamshaper at 768 px did not defeat SynthID v2 on Gemini 3 Pro outputs (verified May 2026 via Gemini app's "Verify with SynthID"). Switch the default invisible engine to SDXL at 1024 px, matching the raiw-app production config (strength 0.05, steps 50). Drop the SD-1.5 pipeline. Metadata layer: add C2PA UUID and IPTC AI marker byte-scan detection across all formats, plus an ISOBMFF box walker (noai/isobmff.py) that strips top-level C2PA uuid and JUMBF jumb boxes from AVIF/HEIF/JPEG-XL containers without re-encoding. README gets a Legal table and a Threat-model section about SynthID v2's 136-bit payload. CLAUDE.md tracks the SD-1.5 regression as historical context. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
89b7633e5c
commit
f2fc5e09ab
@@ -197,9 +197,14 @@ def cmd_visible(
|
||||
@click.option(
|
||||
"-o", "--output", type=click.Path(path_type=Path), default=None, help="Output path (default: <source>_clean.<ext>)."
|
||||
)
|
||||
@click.option("--strength", type=float, default=0.02, help="Denoising strength (0.0-1.0). Default: 0.02.")
|
||||
@click.option("--steps", type=int, default=100, help="Number of denoising steps. Default: 100.")
|
||||
@click.option("--pipeline", type=click.Choice(["default", "ctrlregen"]), default="default", help="Pipeline profile.")
|
||||
@click.option("--strength", type=float, default=0.05, help="Denoising strength (0.0-1.0). Default: 0.05.")
|
||||
@click.option("--steps", type=int, default=50, help="Number of denoising steps. Default: 50.")
|
||||
@click.option(
|
||||
"--pipeline",
|
||||
type=click.Choice(["default", "ctrlregen"]),
|
||||
default="default",
|
||||
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
|
||||
)
|
||||
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
|
||||
@click.option("--seed", type=int, default=None, help="Random seed for reproducibility.")
|
||||
@click.option("--hf-token", type=str, default=None, help="HuggingFace API token.")
|
||||
@@ -334,13 +339,13 @@ def cmd_metadata(
|
||||
@click.option(
|
||||
"--inpaint-method", type=click.Choice(["ns", "telea", "gaussian"]), default="ns", help="Inpainting method."
|
||||
)
|
||||
@click.option("--strength", type=float, default=0.02, help="Invisible watermark denoising strength (0.0-1.0).")
|
||||
@click.option("--steps", type=int, default=100, help="Number of denoising steps for invisible removal.")
|
||||
@click.option("--strength", type=float, default=0.05, help="Invisible watermark denoising strength (0.0-1.0).")
|
||||
@click.option("--steps", type=int, default=50, help="Number of denoising steps for invisible removal.")
|
||||
@click.option(
|
||||
"--pipeline",
|
||||
type=click.Choice(["default", "ctrlregen"]),
|
||||
default="default",
|
||||
help="Pipeline profile for invisible removal.",
|
||||
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
|
||||
)
|
||||
@click.option("--model", type=str, default=None, help="HuggingFace model ID for invisible removal.")
|
||||
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
|
||||
@@ -600,7 +605,12 @@ def _process_batch_image(
|
||||
@click.option(
|
||||
"--humanize", type=float, default=0.0, help="Analog Humanizer film grain intensity (0 = off, typical: 2.0-6.0)."
|
||||
)
|
||||
@click.option("--pipeline", type=click.Choice(["default", "ctrlregen"]), default="default", help="Pipeline profile.")
|
||||
@click.option(
|
||||
"--pipeline",
|
||||
type=click.Choice(["default", "ctrlregen"]),
|
||||
default="default",
|
||||
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
|
||||
)
|
||||
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
|
||||
@click.option("--seed", type=int, default=None, help="Random seed for reproducibility.")
|
||||
@click.option("--hf-token", type=str, default=None, help="HuggingFace API token.")
|
||||
@@ -667,7 +677,7 @@ def cmd_batch(
|
||||
seed=seed,
|
||||
hf_token=hf_token,
|
||||
humanize=humanize,
|
||||
)
|
||||
)
|
||||
processed += 1
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -52,7 +52,10 @@ class InvisibleEngine:
|
||||
to break watermark patterns, and reconstructs via reverse diffusion.
|
||||
"""
|
||||
|
||||
DEFAULT_MODEL_ID = "Lykon/dreamshaper-8"
|
||||
# SDXL base is the default since May 2026: empirically defeats SynthID v2
|
||||
# at strength=0.05 / steps=50 / native ~1024px. See CLAUDE.md "Known
|
||||
# limitations" for the regression evidence ruling out SD-1.5 pipelines.
|
||||
DEFAULT_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
|
||||
CTRLREGEN_MODEL_ID = "yepengliu/ctrlregen"
|
||||
|
||||
def __init__(
|
||||
@@ -68,7 +71,8 @@ class InvisibleEngine:
|
||||
Args:
|
||||
model_id: HuggingFace model ID. None = use default for pipeline.
|
||||
device: Device for inference (auto/cpu/mps/cuda). None = auto.
|
||||
pipeline: Pipeline profile ("default" or "ctrlregen").
|
||||
pipeline: Pipeline profile. "default" (SDXL base, defeats SynthID
|
||||
v2) or "ctrlregen" (CtrlRegen).
|
||||
hf_token: HuggingFace API token.
|
||||
progress_callback: Optional callback for progress messages.
|
||||
"""
|
||||
@@ -123,7 +127,9 @@ class InvisibleEngine:
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
max_dimension = 768
|
||||
# SDXL is trained at 1024px and degrades both quality and watermark-removal
|
||||
# efficacy below that.
|
||||
max_dimension = 1024
|
||||
image = Image.open(image_path)
|
||||
image = ImageOps.exif_transpose(image)
|
||||
orig_size = image.size # (width, height)
|
||||
|
||||
@@ -60,6 +60,19 @@ AI_KEYWORDS: tuple[str, ...] = (
|
||||
"c2pa",
|
||||
)
|
||||
|
||||
# C2PA UUID used in ISOBMFF (AVIF, HEIF, MP4) ``uuid`` boxes.
|
||||
# Reference: https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html
|
||||
C2PA_UUID: bytes = bytes.fromhex("d8fec3d61b0e483c92975828877ec481")
|
||||
|
||||
# IPTC ``digitalSourceType`` values (IPTC 2025.1) that flag AI provenance.
|
||||
# Used by Instagram, Facebook, X (Twitter) to show "Made with AI" labels.
|
||||
IPTC_AI_MARKERS: tuple[bytes, ...] = (
|
||||
b"trainedAlgorithmicMedia",
|
||||
b"compositeSynthetic",
|
||||
b"algorithmicMedia",
|
||||
b"compositeWithTrainedAlgorithmicMedia",
|
||||
)
|
||||
|
||||
STANDARD_METADATA_KEYS: frozenset[str] = frozenset(
|
||||
[
|
||||
"Author",
|
||||
@@ -95,25 +108,38 @@ def has_ai_metadata(image_path: Path) -> bool:
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(image_path) as img:
|
||||
for key in img.info:
|
||||
if _is_ai_key(key):
|
||||
return True
|
||||
# PIL may not handle AVIF/HEIF/JPEG-XL without the optional plugins
|
||||
# (ultralytics also monkey-patches Image.open in a way that can raise
|
||||
# ModuleNotFoundError when pi_heif autoload fails), so any open failure
|
||||
# falls through to the binary scan.
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
for key in img.info:
|
||||
if _is_ai_key(key):
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("PIL could not open %s for metadata scan: %s", image_path, exc)
|
||||
|
||||
# Check C2PA
|
||||
# Check C2PA — via the official ``c2pa`` lib if available, otherwise via a
|
||||
# binary scan that also catches AVIF/HEIF/JPEG-XL containers (PIL doesn't
|
||||
# expose their metadata uniformly).
|
||||
try:
|
||||
from c2pa import has_c2pa_metadata
|
||||
|
||||
if has_c2pa_metadata(image_path):
|
||||
return True
|
||||
except ImportError:
|
||||
# Try simple binary scan (read only first 512KB to avoid OOM on huge files)
|
||||
with open(image_path, "rb") as f:
|
||||
data = f.read(512 * 1024)
|
||||
if b"c2pa" in data.lower() or b"C2PA" in data:
|
||||
return True
|
||||
pass
|
||||
|
||||
return False
|
||||
# Binary scan covers C2PA (PNG caBX, JPEG APP11, AVIF/HEIF/JXL uuid boxes)
|
||||
# and IPTC AI markers in XMP. Read only the first 512KB to bound memory.
|
||||
with open(image_path, "rb") as f:
|
||||
data = f.read(512 * 1024)
|
||||
if b"c2pa" in data.lower() or b"C2PA" in data:
|
||||
return True
|
||||
if C2PA_UUID in data:
|
||||
return True
|
||||
return any(marker in data for marker in IPTC_AI_MARKERS)
|
||||
|
||||
|
||||
def _scan_png_c2pa_chunk(image_path: Path) -> dict[str, str]:
|
||||
@@ -244,6 +270,19 @@ def remove_ai_metadata(
|
||||
if output_path is None:
|
||||
output_path = source_path
|
||||
|
||||
# AVIF/HEIF/JPEG-XL: strip C2PA boxes at the container level without
|
||||
# re-encoding. Avoids needing PIL plugins (pillow-heif / pillow-jxl) and
|
||||
# preserves pixel data bit-for-bit.
|
||||
if source_path.suffix.lower() in (".avif", ".heif", ".heic", ".jxl"):
|
||||
from remove_ai_watermarks.noai.isobmff import strip_c2pa_boxes
|
||||
|
||||
data = source_path.read_bytes()
|
||||
cleaned, stripped = strip_c2pa_boxes(data)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(cleaned)
|
||||
logger.info("Stripped %d C2PA box(es) → %s", stripped, output_path)
|
||||
return output_path
|
||||
|
||||
# Read image and filter metadata
|
||||
with Image.open(source_path) as img:
|
||||
img = img.copy()
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Minimal ISOBMFF box walker for stripping C2PA from AVIF / HEIF / MP4 / JPEG-XL.
|
||||
|
||||
The ISO Base Media File Format wraps content in nested ``[size:4][type:4][...]``
|
||||
boxes. C2PA stores its manifest in a top-level ``uuid`` box keyed by the
|
||||
C2PA UUID; JPEG-XL uses a ``jumb`` box (JUMBF) instead. To strip provenance
|
||||
without re-encoding the image, we walk the top-level box list, drop boxes that
|
||||
carry C2PA, and emit the rest verbatim. The codestream (``mdat`` for ISOBMFF,
|
||||
``jxlc`` / ``jxlp`` for JPEG-XL) is untouched, so pixel data is preserved
|
||||
bit-for-bit.
|
||||
|
||||
This file intentionally avoids dependencies on format-specific libraries
|
||||
(pillow-heif, pillow-jxl, pymp4) so it works on systems where they aren't
|
||||
installed.
|
||||
|
||||
Reference: ISO/IEC 14496-12 (ISOBMFF) and C2PA 2.1 spec §11.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
from remove_ai_watermarks.metadata import C2PA_UUID
|
||||
|
||||
# Top-level box types that carry C2PA payload. ``uuid`` boxes are checked
|
||||
# against ``C2PA_UUID`` before being stripped; ``jumb`` boxes are always
|
||||
# stripped (JPEG-XL uses them exclusively for JUMBF).
|
||||
C2PA_BOX_TYPES: frozenset[bytes] = frozenset({b"uuid", b"jumb"})
|
||||
|
||||
|
||||
def _iter_top_level_boxes(data: bytes) -> Iterator[tuple[int, int, bytes, int]]:
|
||||
"""Yield ``(start, end, type, payload_offset)`` for each top-level box.
|
||||
|
||||
Handles all three ISOBMFF box-size encodings:
|
||||
- ``size > 1``: 32-bit size field is the total box length.
|
||||
- ``size == 1``: 64-bit ``largesize`` follows after the type field.
|
||||
- ``size == 0``: box runs to end of file.
|
||||
"""
|
||||
pos = 0
|
||||
n = len(data)
|
||||
while pos + 8 <= n:
|
||||
size32 = struct.unpack_from(">I", data, pos)[0]
|
||||
box_type = data[pos + 4 : pos + 8]
|
||||
if size32 == 1:
|
||||
if pos + 16 > n:
|
||||
return
|
||||
size = struct.unpack_from(">Q", data, pos + 8)[0]
|
||||
payload_off = pos + 16
|
||||
elif size32 == 0:
|
||||
size = n - pos
|
||||
payload_off = pos + 8
|
||||
else:
|
||||
size = size32
|
||||
payload_off = pos + 8
|
||||
if size < (payload_off - pos) or pos + size > n:
|
||||
return
|
||||
yield pos, pos + size, box_type, payload_off
|
||||
pos += size
|
||||
|
||||
|
||||
def is_isobmff(data: bytes) -> bool:
|
||||
"""Cheap sniff: ISOBMFF files start with an ``ftyp`` box."""
|
||||
return len(data) >= 8 and data[4:8] == b"ftyp"
|
||||
|
||||
|
||||
def strip_c2pa_boxes(data: bytes) -> tuple[bytes, int]:
|
||||
"""Return ``(cleaned_bytes, stripped_count)``.
|
||||
|
||||
Walks top-level boxes; drops any ``uuid`` box whose UUID equals
|
||||
``C2PA_UUID`` and any ``jumb`` box (JPEG-XL JUMBF container). All other
|
||||
boxes are emitted verbatim. If the input is not ISOBMFF-shaped, returns
|
||||
it unchanged.
|
||||
"""
|
||||
if not is_isobmff(data):
|
||||
return data, 0
|
||||
|
||||
out = bytearray()
|
||||
stripped = 0
|
||||
for start, end, box_type, payload_off in _iter_top_level_boxes(data):
|
||||
if box_type in C2PA_BOX_TYPES:
|
||||
if box_type == b"uuid":
|
||||
# uuid boxes carry the 16-byte UUID immediately after the type.
|
||||
if payload_off + 16 <= end and data[payload_off : payload_off + 16] == C2PA_UUID:
|
||||
stripped += 1
|
||||
continue
|
||||
else: # b"jumb"
|
||||
stripped += 1
|
||||
continue
|
||||
out.extend(data[start:end])
|
||||
return bytes(out), stripped
|
||||
@@ -5,7 +5,7 @@ Pure configuration and lookup functions with no ML dependencies.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DEFAULT_MODEL_ID = "Lykon/dreamshaper-8"
|
||||
DEFAULT_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
|
||||
CTRLREGEN_MODEL_ID = "yepengliu/ctrlregen"
|
||||
|
||||
LOW_STRENGTH = 0.04
|
||||
|
||||
Reference in New Issue
Block a user