feat: SDXL default; AVIF/HEIF/JPEG-XL C2PA stripping

SD-1.5 dreamshaper at 768 px did not defeat SynthID v2 on Gemini 3 Pro
outputs (verified May 2026 via Gemini app's "Verify with SynthID"). Switch
the default invisible engine to SDXL at 1024 px, matching the raiw-app
production config (strength 0.05, steps 50). Drop the SD-1.5 pipeline.

Metadata layer: add C2PA UUID and IPTC AI marker byte-scan detection
across all formats, plus an ISOBMFF box walker (noai/isobmff.py) that
strips top-level C2PA uuid and JUMBF jumb boxes from AVIF/HEIF/JPEG-XL
containers without re-encoding.

README gets a Legal table and a Threat-model section about SynthID v2's
136-bit payload. CLAUDE.md tracks the SD-1.5 regression as historical
context.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
test-user
2026-05-17 12:54:37 -07:00
co-authored by Claude Opus 4.7
parent 89b7633e5c
commit f2fc5e09ab
10 changed files with 298 additions and 41 deletions
+18 -8
View File
@@ -197,9 +197,14 @@ def cmd_visible(
@click.option(
"-o", "--output", type=click.Path(path_type=Path), default=None, help="Output path (default: <source>_clean.<ext>)."
)
@click.option("--strength", type=float, default=0.02, help="Denoising strength (0.0-1.0). Default: 0.02.")
@click.option("--steps", type=int, default=100, help="Number of denoising steps. Default: 100.")
@click.option("--pipeline", type=click.Choice(["default", "ctrlregen"]), default="default", help="Pipeline profile.")
@click.option("--strength", type=float, default=0.05, help="Denoising strength (0.0-1.0). Default: 0.05.")
@click.option("--steps", type=int, default=50, help="Number of denoising steps. Default: 50.")
@click.option(
"--pipeline",
type=click.Choice(["default", "ctrlregen"]),
default="default",
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
)
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
@click.option("--seed", type=int, default=None, help="Random seed for reproducibility.")
@click.option("--hf-token", type=str, default=None, help="HuggingFace API token.")
@@ -334,13 +339,13 @@ def cmd_metadata(
@click.option(
"--inpaint-method", type=click.Choice(["ns", "telea", "gaussian"]), default="ns", help="Inpainting method."
)
@click.option("--strength", type=float, default=0.02, help="Invisible watermark denoising strength (0.0-1.0).")
@click.option("--steps", type=int, default=100, help="Number of denoising steps for invisible removal.")
@click.option("--strength", type=float, default=0.05, help="Invisible watermark denoising strength (0.0-1.0).")
@click.option("--steps", type=int, default=50, help="Number of denoising steps for invisible removal.")
@click.option(
"--pipeline",
type=click.Choice(["default", "ctrlregen"]),
default="default",
help="Pipeline profile for invisible removal.",
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
)
@click.option("--model", type=str, default=None, help="HuggingFace model ID for invisible removal.")
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
@@ -600,7 +605,12 @@ def _process_batch_image(
@click.option(
"--humanize", type=float, default=0.0, help="Analog Humanizer film grain intensity (0 = off, typical: 2.0-6.0)."
)
@click.option("--pipeline", type=click.Choice(["default", "ctrlregen"]), default="default", help="Pipeline profile.")
@click.option(
"--pipeline",
type=click.Choice(["default", "ctrlregen"]),
default="default",
help="Pipeline profile (default=SDXL, ctrlregen=CtrlRegen).",
)
@click.option("--device", type=click.Choice(["auto", "cpu", "mps", "cuda"]), default="auto", help="Inference device.")
@click.option("--seed", type=int, default=None, help="Random seed for reproducibility.")
@click.option("--hf-token", type=str, default=None, help="HuggingFace API token.")
@@ -667,7 +677,7 @@ def cmd_batch(
seed=seed,
hf_token=hf_token,
humanize=humanize,
)
)
processed += 1
except Exception as e:
+9 -3
View File
@@ -52,7 +52,10 @@ class InvisibleEngine:
to break watermark patterns, and reconstructs via reverse diffusion.
"""
DEFAULT_MODEL_ID = "Lykon/dreamshaper-8"
# SDXL base is the default since May 2026: empirically defeats SynthID v2
# at strength=0.05 / steps=50 / native ~1024px. See CLAUDE.md "Known
# limitations" for the regression evidence ruling out SD-1.5 pipelines.
DEFAULT_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
CTRLREGEN_MODEL_ID = "yepengliu/ctrlregen"
def __init__(
@@ -68,7 +71,8 @@ class InvisibleEngine:
Args:
model_id: HuggingFace model ID. None = use default for pipeline.
device: Device for inference (auto/cpu/mps/cuda). None = auto.
pipeline: Pipeline profile ("default" or "ctrlregen").
pipeline: Pipeline profile. "default" (SDXL base, defeats SynthID
v2) or "ctrlregen" (CtrlRegen).
hf_token: HuggingFace API token.
progress_callback: Optional callback for progress messages.
"""
@@ -123,7 +127,9 @@ class InvisibleEngine:
from PIL import Image, ImageOps
max_dimension = 768
# SDXL is trained at 1024px and degrades both quality and watermark-removal
# efficacy below that.
max_dimension = 1024
image = Image.open(image_path)
image = ImageOps.exif_transpose(image)
orig_size = image.size # (width, height)
+50 -11
View File
@@ -60,6 +60,19 @@ AI_KEYWORDS: tuple[str, ...] = (
"c2pa",
)
# C2PA UUID used in ISOBMFF (AVIF, HEIF, MP4) ``uuid`` boxes.
# Reference: https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html
C2PA_UUID: bytes = bytes.fromhex("d8fec3d61b0e483c92975828877ec481")
# IPTC ``digitalSourceType`` values (IPTC 2025.1) that flag AI provenance.
# Used by Instagram, Facebook, X (Twitter) to show "Made with AI" labels.
IPTC_AI_MARKERS: tuple[bytes, ...] = (
b"trainedAlgorithmicMedia",
b"compositeSynthetic",
b"algorithmicMedia",
b"compositeWithTrainedAlgorithmicMedia",
)
STANDARD_METADATA_KEYS: frozenset[str] = frozenset(
[
"Author",
@@ -95,25 +108,38 @@ def has_ai_metadata(image_path: Path) -> bool:
"""
from PIL import Image
with Image.open(image_path) as img:
for key in img.info:
if _is_ai_key(key):
return True
# PIL may not handle AVIF/HEIF/JPEG-XL without the optional plugins
# (ultralytics also monkey-patches Image.open in a way that can raise
# ModuleNotFoundError when pi_heif autoload fails), so any open failure
# falls through to the binary scan.
try:
with Image.open(image_path) as img:
for key in img.info:
if _is_ai_key(key):
return True
except Exception as exc:
logger.debug("PIL could not open %s for metadata scan: %s", image_path, exc)
# Check C2PA
# Check C2PA — via the official ``c2pa`` lib if available, otherwise via a
# binary scan that also catches AVIF/HEIF/JPEG-XL containers (PIL doesn't
# expose their metadata uniformly).
try:
from c2pa import has_c2pa_metadata
if has_c2pa_metadata(image_path):
return True
except ImportError:
# Try simple binary scan (read only first 512KB to avoid OOM on huge files)
with open(image_path, "rb") as f:
data = f.read(512 * 1024)
if b"c2pa" in data.lower() or b"C2PA" in data:
return True
pass
return False
# Binary scan covers C2PA (PNG caBX, JPEG APP11, AVIF/HEIF/JXL uuid boxes)
# and IPTC AI markers in XMP. Read only the first 512KB to bound memory.
with open(image_path, "rb") as f:
data = f.read(512 * 1024)
if b"c2pa" in data.lower() or b"C2PA" in data:
return True
if C2PA_UUID in data:
return True
return any(marker in data for marker in IPTC_AI_MARKERS)
def _scan_png_c2pa_chunk(image_path: Path) -> dict[str, str]:
@@ -244,6 +270,19 @@ def remove_ai_metadata(
if output_path is None:
output_path = source_path
# AVIF/HEIF/JPEG-XL: strip C2PA boxes at the container level without
# re-encoding. Avoids needing PIL plugins (pillow-heif / pillow-jxl) and
# preserves pixel data bit-for-bit.
if source_path.suffix.lower() in (".avif", ".heif", ".heic", ".jxl"):
from remove_ai_watermarks.noai.isobmff import strip_c2pa_boxes
data = source_path.read_bytes()
cleaned, stripped = strip_c2pa_boxes(data)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(cleaned)
logger.info("Stripped %d C2PA box(es) → %s", stripped, output_path)
return output_path
# Read image and filter metadata
with Image.open(source_path) as img:
img = img.copy()
+93
View File
@@ -0,0 +1,93 @@
"""Minimal ISOBMFF box walker for stripping C2PA from AVIF / HEIF / MP4 / JPEG-XL.
The ISO Base Media File Format wraps content in nested ``[size:4][type:4][...]``
boxes. C2PA stores its manifest in a top-level ``uuid`` box keyed by the
C2PA UUID; JPEG-XL uses a ``jumb`` box (JUMBF) instead. To strip provenance
without re-encoding the image, we walk the top-level box list, drop boxes that
carry C2PA, and emit the rest verbatim. The codestream (``mdat`` for ISOBMFF,
``jxlc`` / ``jxlp`` for JPEG-XL) is untouched, so pixel data is preserved
bit-for-bit.
This file intentionally avoids dependencies on format-specific libraries
(pillow-heif, pillow-jxl, pymp4) so it works on systems where they aren't
installed.
Reference: ISO/IEC 14496-12 (ISOBMFF) and C2PA 2.1 spec §11.
"""
from __future__ import annotations
import struct
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterator
from remove_ai_watermarks.metadata import C2PA_UUID
# Top-level box types that carry C2PA payload. ``uuid`` boxes are checked
# against ``C2PA_UUID`` before being stripped; ``jumb`` boxes are always
# stripped (JPEG-XL uses them exclusively for JUMBF).
C2PA_BOX_TYPES: frozenset[bytes] = frozenset({b"uuid", b"jumb"})
def _iter_top_level_boxes(data: bytes) -> Iterator[tuple[int, int, bytes, int]]:
"""Yield ``(start, end, type, payload_offset)`` for each top-level box.
Handles all three ISOBMFF box-size encodings:
- ``size > 1``: 32-bit size field is the total box length.
- ``size == 1``: 64-bit ``largesize`` follows after the type field.
- ``size == 0``: box runs to end of file.
"""
pos = 0
n = len(data)
while pos + 8 <= n:
size32 = struct.unpack_from(">I", data, pos)[0]
box_type = data[pos + 4 : pos + 8]
if size32 == 1:
if pos + 16 > n:
return
size = struct.unpack_from(">Q", data, pos + 8)[0]
payload_off = pos + 16
elif size32 == 0:
size = n - pos
payload_off = pos + 8
else:
size = size32
payload_off = pos + 8
if size < (payload_off - pos) or pos + size > n:
return
yield pos, pos + size, box_type, payload_off
pos += size
def is_isobmff(data: bytes) -> bool:
"""Cheap sniff: ISOBMFF files start with an ``ftyp`` box."""
return len(data) >= 8 and data[4:8] == b"ftyp"
def strip_c2pa_boxes(data: bytes) -> tuple[bytes, int]:
"""Return ``(cleaned_bytes, stripped_count)``.
Walks top-level boxes; drops any ``uuid`` box whose UUID equals
``C2PA_UUID`` and any ``jumb`` box (JPEG-XL JUMBF container). All other
boxes are emitted verbatim. If the input is not ISOBMFF-shaped, returns
it unchanged.
"""
if not is_isobmff(data):
return data, 0
out = bytearray()
stripped = 0
for start, end, box_type, payload_off in _iter_top_level_boxes(data):
if box_type in C2PA_BOX_TYPES:
if box_type == b"uuid":
# uuid boxes carry the 16-byte UUID immediately after the type.
if payload_off + 16 <= end and data[payload_off : payload_off + 16] == C2PA_UUID:
stripped += 1
continue
else: # b"jumb"
stripped += 1
continue
out.extend(data[start:end])
return bytes(out), stripped
@@ -5,7 +5,7 @@ Pure configuration and lookup functions with no ML dependencies.
from __future__ import annotations
DEFAULT_MODEL_ID = "Lykon/dreamshaper-8"
DEFAULT_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
CTRLREGEN_MODEL_ID = "yepengliu/ctrlregen"
LOW_STRENGTH = 0.04