fix: metadata-strip parity, input robustness, and detection/clash coverage

Bug fixes (each with a regression test):
- metadata strip parity across every marker placement: IPTC digitalSourceType
  in XMP, the Samsung post-EOI trailer, the China TC260 AIGC block in EXIF
  UserComment, a bare AIGC block in a non-standard APP segment, and the ISOBMFF
  EXIF path (AIGC + xAI) are all now stripped -- anything a scanner flags, the
  strip reaches
- Samsung genAIType detected when its trailer sits past the 512 KB scan window
  (file-tail read on large photos)
- crashes on edge inputs: Gemini detector on images with a short side < 16px,
  footprint_mask on a zero-size ndarray, the humanizer on chromatic_shift >=
  width, and the CLI on unreadable/corrupt/empty input (clean error, not a
  traceback)
- WebP written losslessly (cv2 quality 101), not lossy at 100
- the IPTC digitalSourceType algorithmicMedia (procedural, not trained on
  sampled data) is no longer flagged as AI-generated, so clean procedural
  content is not scrubbed
- c2pa source-type: compositeWithTrainedAlgorithmicMedia is checked before the
  bare algorithmicMedia token, so an AI-enhanced composite is not misclassified

Detection:
- integrity-clash coverage now normalizes ByteDance / Canva / ElevenLabs /
  Black Forest Labs, so a transplanted manifest next to an independent
  conflicting stamp is caught; the generic China TC260 AIGC label is attributed
  to a co-present TC260 vendor, so a legit Doubao image (its own C2PA + TC260
  label) does not clash (corpus-validated: 0 new clashes on 5000 carriers)

CLI:
- batch exits non-zero (with a warning) when any image errors or a GPU-missing
  SynthID scrub is skipped, and copies the input through so the output dir stays
  complete -- it used to always exit 0 and could silently drop files

Perf:
- GeminiEngine reused as a process-wide singleton with a precomputed template
  ladder: -24% on the identify sparkle path, detection byte-identical

Internal: one shared _ai_exif_targets rule set feeds both EXIF scrubbers so
their coverage cannot drift; docs synced; maintain.sh hardened so the uv-secure
internal teardown crash no longer aborts the gate (still fails on a real finding).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Victor Kuznetsov
2026-07-13 10:49:24 +03:00
co-authored by Claude Opus 4.8
parent 190dc89d23
commit a4c901ff39
25 changed files with 894 additions and 97 deletions
@@ -315,6 +315,8 @@ class TextMarkEngine:
With ``force`` and no glyph found, falls back to the whole geometry box (the
``--no-detect`` path). The caller gates on detection.
"""
if image is None or image.size == 0:
return None # guard before to_bgr (cvtColor raises on an empty Mat); mirror detect()
image = image_io.to_bgr(image)
h, w = image.shape[:2]
if h < 32 or w < 64:
+55 -7
View File
@@ -595,6 +595,9 @@ def cmd_visible(
except RuntimeError as e: # e.g. a selected migan/lama backend whose extra is absent
console.print(f" Error: {e}")
raise SystemExit(1) from e
except (ValueError, OSError) as e: # unreadable / truncated / non-image input
console.print(f" Error: cannot read image {source.name}: {e}")
raise SystemExit(1) from e
elapsed = time.monotonic() - t0
h, w = result.shape[:2]
console.print(f" Input: {source.name} ({w}x{h})")
@@ -933,7 +936,11 @@ def cmd_metadata(
return
# Remove
out = remove_ai_metadata(source, output, keep_standard=keep_standard)
try:
out = remove_ai_metadata(source, output, keep_standard=keep_standard)
except (OSError, ValueError) as e: # unreadable / truncated / non-image (PIL raises OSError subclasses)
console.print(f" Error: cannot process {source.name}: {e}")
raise SystemExit(1) from e
console.print(f" AI metadata stripped -> {out}")
@@ -1246,6 +1253,14 @@ def cmd_all(
# ── Batch command ──
def _passthrough_copy(img_path: Path, out_path: Path) -> None:
"""Copy the input's pixels through to ``out_path`` unchanged (the invisible-mode skip
paths), so the output dir stays complete without touching the pixels."""
src_bgr, src_alpha = image_io.read_bgr_and_alpha(img_path)
if src_bgr is not None:
image_io.write_bgr_with_alpha(out_path, src_bgr, src_alpha)
def _process_batch_image(
ctx: click.Context,
img_path: Path,
@@ -1272,16 +1287,21 @@ def _process_batch_image(
tile_size: int = 1024,
tile_overlap: int = 128,
force: bool = False,
) -> None:
) -> bool:
"""Process a single image for batch mode.
Applies the requested watermark removal steps (visible, invisible,
metadata) to *img_path* and writes the result to *out_path*.
Returns True if the invisible (SynthID) scrub was skipped because the GPU deps
are missing while a signal was present -- so the batch caller can warn + exit
non-zero, mirroring the single ``all`` command.
Raises:
ValueError: If the image cannot be opened.
"""
saved_alpha: NDArray[Any] | None = None
synthid_skipped = False
if mode in ("visible", "all"):
# Always read the ORIGINAL source: the visible pass is the first step, so a
@@ -1341,13 +1361,22 @@ def _process_batch_image(
# visible-processed `out_path` whose C2PA is already gone.
vendor=vendor_for_strength(img_path),
)
elif not invisible_available() and not skip_no_signal:
# An invisible signal IS present but the GPU deps are missing, so the
# SynthID scrub cannot run. Mirror the single `all` command's loud skip:
# flag it for a batch-level warning + non-zero exit (a silently retained
# SynthID watermark is the #1 "it didn't work" report). For invisible mode
# nothing wrote out_path yet -> copy the input through so the output dir is
# complete with the pixels deliberately left intact (without this, a
# signal-bearing image in a GPU-less --mode invisible run got NO output).
synthid_skipped = True
if mode == "invisible" and not out_path.exists():
_passthrough_copy(img_path, out_path)
elif skip_no_signal and mode == "invisible" and not out_path.exists():
# No invisible target and the visible/all pass did not write out_path
# (invisible mode): copy the input through so the output dir is complete
# with the pixels deliberately left intact.
src_bgr, src_alpha = image_io.read_bgr_and_alpha(img_path)
if src_bgr is not None:
image_io.write_bgr_with_alpha(out_path, src_bgr, src_alpha)
_passthrough_copy(img_path, out_path)
if mode in ("metadata", "all"):
from remove_ai_watermarks.metadata import remove_ai_metadata
@@ -1361,6 +1390,8 @@ def _process_batch_image(
if final_bgr is not None:
image_io.write_bgr_with_alpha(out_path, final_bgr, saved_alpha)
return synthid_skipped
@main.command("batch")
@click.argument("directory", type=click.Path(exists=True, file_okay=False, path_type=Path))
@@ -1457,6 +1488,7 @@ def cmd_batch(
processed = 0
errors = 0
synthid_skipped_count = 0
with Progress(
SpinnerColumn(),
@@ -1473,7 +1505,7 @@ def cmd_batch(
progress.update(task, description=f"{img_path.name}")
try:
_process_batch_image(
if _process_batch_image(
ctx=ctx,
img_path=img_path,
out_path=out_path,
@@ -1499,7 +1531,8 @@ def cmd_batch(
tile_size=tile_size,
tile_overlap=tile_overlap,
force=force,
)
):
synthid_skipped_count += 1
processed += 1
except Exception as e:
@@ -1511,6 +1544,21 @@ def cmd_batch(
console.print(f"\n {processed} processed" + (f" {errors} errors" if errors else ""))
if synthid_skipped_count:
# Mirror the single `all` command: a silently retained SynthID watermark is the
# #1 "it didn't work" report, so make the skipped scrub impossible to miss.
console.print(
f"\n WARNING: the invisible (SynthID) watermark was NOT removed on "
f"{synthid_skipped_count} image(s) -- the GPU dependencies are not installed, "
f"so those outputs still carry the invisible watermark.\n"
f" Install the extra and rerun: pip install 'remove-ai-watermarks[gpu]'"
)
# Non-zero exit so a wrapping service detects an incomplete/failed run (batch used
# to always exit 0, hiding both per-image errors and skipped SynthID scrubs).
if errors or synthid_skipped_count:
raise SystemExit(1)
if __name__ == "__main__":
main()
+40 -4
View File
@@ -20,6 +20,7 @@ to DETECT and to shape the removal mask -- the old reverse-alpha pixel recovery
# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportUnknownParameterType=false, reportMissingTypeArgument=false, reportMissingTypeStubs=false, reportMissingImports=false, reportArgumentType=false, reportAssignmentType=false, reportReturnType=false, reportCallIssue=false, reportIndexIssue=false, reportOperatorIssue=false, reportOptionalMemberAccess=false, reportOptionalCall=false, reportOptionalSubscript=false, reportOptionalOperand=false, reportAttributeAccessIssue=false, reportPrivateImportUsage=false, reportPrivateUsage=false, reportInvalidTypeForm=false, reportConstantRedefinition=false, reportUnnecessaryComparison=false
from __future__ import annotations
import functools
import logging
from dataclasses import dataclass
from enum import Enum
@@ -126,6 +127,12 @@ def _load_embedded_asset(name: str) -> NDArray[Any]:
return img
# Single source of truth for the multi-scale template ladder (aggressively downscaled to
# slightly upscaled): the precomputed `_tmpl_cache` and the `_scan_scales` loop must use
# the SAME scales or a scan scale would miss the cache and KeyError.
_TEMPLATE_SCALES: tuple[int, ...] = tuple(range(16, 120, 2))
class GeminiEngine:
"""Detects and localizes the visible Gemini sparkle for the shared fill removal.
@@ -242,6 +249,16 @@ class GeminiEngine:
self._alpha_small = _calculate_alpha_map(bg_small)
self._alpha_large = _calculate_alpha_map(bg_large)
# Per-scale resized templates are constant (``_alpha_large`` never changes),
# so precompute the whole fixed 16..118 ladder once: ``_scan_scales`` runs it on
# every image (twice -- global + corner), and re-``resize``-ing the 96x96 source
# each time is pure repeated work. Prebuilt (not lazy) so the dict is read-only
# after construction and safe to share across threads via the module singleton.
self._tmpl_cache: dict[int, NDArray[Any]] = {
scale: cv2.resize(self._alpha_large, (scale, scale), interpolation=cv2.INTER_AREA)
for scale in _TEMPLATE_SCALES
}
logger.debug(
"Alpha maps loaded: small=%s, large=%s",
self._alpha_small.shape,
@@ -275,11 +292,10 @@ class GeminiEngine:
``_alpha_large`` is the high-quality source downscaled per scale; the range
covers aggressively downscaled to slightly upscaled logos.
"""
for scale in range(16, 120, 2):
for scale in _TEMPLATE_SCALES:
if scale > gray.shape[0] or scale > gray.shape[1]:
continue
tmpl = cv2.resize(self._alpha_large, (scale, scale), interpolation=cv2.INTER_AREA)
match_res = cv2.matchTemplate(gray, tmpl, cv2.TM_CCOEFF_NORMED)
match_res = cv2.matchTemplate(gray, self._tmpl_cache[scale], cv2.TM_CCOEFF_NORMED)
_, max_val, _, max_loc = cv2.minMaxLoc(match_res)
yield scale, float(max_val), max_loc
@@ -361,6 +377,12 @@ class GeminiEngine:
if promoted is not None:
candidates.append(promoted)
# No candidate at any scale: the search region is smaller than the 16px template
# floor (an image whose short side is < 16px), so nothing is detectable. Return
# the empty (detected=False) result rather than dereferencing candidates[0].
if not candidates:
return result
# Select the candidate with the highest full-fusion confidence (pre-FP-gate).
best_scale, pos_x, pos_y, best_raw_ncc = candidates[0]
grad_score, var_score, best_fused = 0.0, 0.0, -1.0
@@ -543,6 +565,8 @@ class GeminiEngine:
yield no mask (reported-removed-but-unchanged). Absent ``region``, direct callers
keep the detect-then-force behavior.
"""
if image is None or image.size == 0:
return None # guard before to_bgr (cvtColor raises on an empty Mat); mirror detect_watermark
image = image_io.to_bgr(image)
h, w = image.shape[:2]
if region is not None:
@@ -681,6 +705,18 @@ class GeminiEngine:
return float(np.median((hi - lo) / (hi + 1.0)))
@functools.lru_cache(maxsize=1)
def _shared_engine() -> GeminiEngine:
"""Process-wide default ``GeminiEngine`` singleton.
The engine holds only constant assets (embedded captures, alpha maps, the
precomputed template ladder) and takes the image as a method argument, so one
instance is reused across every ``detect_sparkle_confidence`` call instead of
reloading assets + recomputing alpha maps + rebuilding the template cache on
each of the ~34k images an ``identify`` batch scans. Output is identical."""
return GeminiEngine()
def detect_sparkle_confidence(image_path: Path, *, image: NDArray[Any] | None = None) -> float | None:
"""Visible-sparkle detection confidence for a file, for provenance use.
@@ -698,4 +734,4 @@ def detect_sparkle_confidence(image_path: Path, *, image: NDArray[Any] | None =
img = image if image is not None else image_io.imread(image_path)
if img is None:
return None
return float(GeminiEngine().detect_watermark(img).confidence)
return float(_shared_engine().detect_watermark(img).confidence)
+8 -5
View File
@@ -41,11 +41,14 @@ def apply_analog_humanizer(image: NDArray, grain_intensity: float = 4.0, chromat
# Shift R channel left, B channel right. np.roll is circular, so it wraps
# the opposite edge into a thin colored fringe at the L/R borders; replicate
# the original edge columns there to keep the intended offset interior-only.
if chromatic_shift > 0:
r = np.roll(r, -chromatic_shift, axis=1)
r[:, -chromatic_shift:] = r[:, -chromatic_shift - 1 : -chromatic_shift]
b = np.roll(b, chromatic_shift, axis=1)
b[:, :chromatic_shift] = b[:, chromatic_shift : chromatic_shift + 1]
# Clamp so the edge-replication slices below always have a source column: a shift
# >= width would leave them empty and crash the broadcast (r[:, -shift:] = (H, 0)).
shift = min(chromatic_shift, image.shape[1] - 1)
if shift > 0:
r = np.roll(r, -shift, axis=1)
r[:, -shift:] = r[:, -shift - 1 : -shift]
b = np.roll(b, shift, axis=1)
b[:, :shift] = b[:, shift : shift + 1]
merged = cv2.merge((b, g, r))
+37 -1
View File
@@ -299,6 +299,19 @@ _AI_VENDOR_TOKENS: tuple[tuple[str, str], ...] = (
("grok", "xAI"),
("aurora", "xAI"),
("xai", "xAI"),
# ByteDance family (all its brands normalize to one origin, mirroring constants.py):
# without these a transplanted ByteDance C2PA manifest next to an independent
# conflicting stamp went undetected by the clash check.
("bytedance", "ByteDance"),
("doubao", "ByteDance"),
("jimeng", "ByteDance"),
("dreamina", "ByteDance"),
("volcengine", "ByteDance"),
("volcano engine", "ByteDance"),
("canva", "Canva"),
("elevenlabs", "ElevenLabs"),
("eleven labs", "ElevenLabs"),
("black forest", "Black Forest Labs"),
)
@@ -328,6 +341,17 @@ def _vendor_of(text: str | None) -> str | None:
_C2PA_MANIFEST_SOURCE = "c2pa_manifest"
_CLASH_SOURCE: dict[str, str] = {"c2pa": _C2PA_MANIFEST_SOURCE, "synthid": _C2PA_MANIFEST_SOURCE}
# The generic China TC260 AIGC vendor label -- a COUNTRY-LEVEL regulatory "this is AI"
# stamp any Chinese generator applies to its own output, naming no specific vendor.
_GENERIC_AIGC_VENDOR = "China AIGC (TC260)"
# Vendors that apply the TC260 label to their OWN output. When one is co-attributed with
# the generic AIGC label, the label is that vendor's own stamp (not an independent
# competing origin), so the clash check attributes the AIGC label to it -- else a legit
# ByteDance/Doubao image (C2PA "ByteDance" + its own TC260 label) would false-clash once
# ByteDance normalizes via _vendor_of. Chinese generators only (Canva/BFL/ElevenLabs,
# also added to _vendor_of, are NOT TC260 appliers).
_TC260_VENDORS: frozenset[str] = frozenset({"ByteDance"})
def _integrity_clashes(
ai_vendors: dict[str, str], camera_label: str | None, *, camera_has_ai_marker: bool
@@ -352,6 +376,18 @@ def _integrity_clashes(
# families clash only when they belong to different provenance sources (see
# _CLASH_SOURCE) AND name different vendors -- so multiple vendors named within
# one C2PA manifest (c2pa issuer + synthid proxy) do not flag.
# The generic TC260 AIGC label is a Chinese regulatory "this is AI" stamp. When a
# Chinese TC260-applying vendor (ByteDance) is ALSO attributed, the label is that
# vendor's own stamp on its own output, so attribute it to that vendor -- a legit
# Doubao image carries BOTH a ByteDance C2PA manifest and its own TC260 label and
# must not clash. Against a NON-TC260 vendor (OpenAI, Google, ...) the label stays
# generic and still clashes as a laundering tell (a foreign-vendor image carrying a
# Chinese TC260 label names two different origins).
if ai_vendors.get("aigc") == _GENERIC_AIGC_VENDOR:
own = next((v for f, v in ai_vendors.items() if f != "aigc" and v in _TC260_VENDORS), None)
if own:
ai_vendors = {**ai_vendors, "aigc": own} # copy co-located with the relabel
source = {fam: _CLASH_SOURCE.get(fam, fam) for fam in ai_vendors}
independent_conflict = any(
source[a] != source[b] and ai_vendors[a] != ai_vendors[b] for a, b in itertools.combinations(ai_vendors, 2)
@@ -625,7 +661,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b
watermarks.append("China AIGC label (TC260 standard)")
if platform is None:
platform = "China AIGC-labeled generator (TC260; e.g. Doubao)"
ai_vendor_claims["aigc"] = "China AIGC (TC260)"
ai_vendor_claims["aigc"] = _GENERIC_AIGC_VENDOR
# ── Local diffusion parameters (Stable Diffusion / ComfyUI) ──────
local_keys = sorted(k for k in meta if k.lower() in _LOCAL_GEN_KEYS)
+5 -1
View File
@@ -145,7 +145,11 @@ def _encode_params(ext: str) -> list[int]:
params += [sf, sf444]
return params
if ext == ".webp":
return [cv2.IMWRITE_WEBP_QUALITY, 100]
# cv2 WebP: quality 1-100 is LOSSY; a value > 100 selects LOSSLESS mode.
# "work with originals" requires lossless so a mark-removal re-encode does not
# degrade the untouched pixels the fill composites over (regression: q100
# round-tripped a random image at maxdiff ~230, q101 at 0).
return [cv2.IMWRITE_WEBP_QUALITY, 101]
return []
+139 -31
View File
@@ -93,9 +93,17 @@ def c2pa_marker_in(data: bytes) -> bool:
IPTC_AI_MARKERS: tuple[bytes, ...] = (
b"trainedAlgorithmicMedia",
b"compositeSynthetic",
b"algorithmicMedia",
b"compositeWithTrainedAlgorithmicMedia",
)
# NOTE: bare ``algorithmicMedia`` is deliberately NOT here. That IPTC digitalSourceType
# means "created purely by an algorithm, NOT from sampled training data" (procedural /
# generative-code art) -- it is NOT AI/ML generation. Real "Made with AI" labels
# (Meta / Instagram / MidJourney) use ``trainedAlgorithmicMedia``. Including the bare
# token flagged clean procedural images as AI (is_ai=high + has_invisible_target=True ->
# a diffusion scrub of clean content), contradicting the c2pa layer, which sets
# source_type without ai_source for it (tests/test_noai.py::test_plain_algorithmic_media_not_flagged_ai).
# It is not a substring of the trained/composite tokens, so its removal does not affect
# their detection.
# IPTC Photo Metadata 2025.1 (published 2025-11-27) added explicit AI-disclosure
# XMP properties in the Iptc4xmpExt namespace. Their mere presence is an AI
@@ -521,18 +529,45 @@ _SAMSUNG_GENAI_RE = re.compile(rb'genAIType"\s*:\s*(-?\d+)')
_SAMSUNG_EDITOR_MARKER = b"PhotoEditor_Re_Edit_Data"
def _read_file_tail(image_path: Path, size: int) -> bytes:
"""Return the last ``size`` bytes of the file (or the whole file if smaller)."""
try:
file_size = image_path.stat().st_size
with open(image_path, "rb") as f:
if file_size > size:
f.seek(file_size - size)
return f.read()
except OSError:
return b""
def samsung_genai(image_path: Path) -> int | None:
"""Return Samsung's non-zero ``genAIType`` value if the image carries the
Galaxy AI editing marker, else None.
See the module note above ``_SAMSUNG_GENAI_RE``: detection is empirical and
gated on the ``PhotoEditor_Re_Edit_Data`` container so an incidental
``genAIType`` token cannot false-positive.
``genAIType`` token cannot false-positive. Galaxy AI appends the marker as a
trailer AFTER the JPEG EOI, so on a multi-MB phone photo it sits past the quick-
scan window; when the head misses it, also read the file tail (else detection
and removal disagree -- the strip reads the whole file and would drop a marker
detection never reported).
"""
head = scan_head(image_path, _QUICK_SCAN_BYTES)
if _SAMSUNG_EDITOR_MARKER not in head:
data = scan_head(image_path, _QUICK_SCAN_BYTES)
if _SAMSUNG_EDITOR_MARKER not in data:
# The marker is a post-EOI trailer, so only a file LARGER than the quick-scan
# window can hide it past the head (`scan_head` already read a smaller file
# whole). Gate the extra tail read on that — `samsung_genai` is on the identify
# hot path, so a redundant 512 KB re-read per small image is not free.
try:
oversize = image_path.stat().st_size > _QUICK_SCAN_BYTES
except OSError:
oversize = False
if oversize:
data = _read_file_tail(image_path, _QUICK_SCAN_BYTES)
if _SAMSUNG_EDITOR_MARKER not in data:
return None
m = _SAMSUNG_GENAI_RE.search(head)
m = _SAMSUNG_GENAI_RE.search(data)
if m is None:
return None
return int(m.group(1)) or None
@@ -709,47 +744,87 @@ def xai_signature(image_path: Path) -> bool:
)
def _scrub_ai_exif(exif_dict: dict[str, Any]) -> list[str]:
"""Delete AI-provenance tags from a piexif dict's ``0th`` IFD, in place.
def _is_aigc_exif_value(raw: object) -> bool:
"""Whether an EXIF tag value carries a China TC260 AIGC producer/service block.
Removes (a) the xAI/Grok signature pair (``ImageDescription`` "Signature: ..."
+ UUID ``Artist``) and (b) any ``Software`` / ``Make`` / ``Artist`` /
``ImageDescription`` tag whose value carries an ``AI_GENERATOR_TOKENS`` token
(Ideogram's ``Make``, Firefly's ``Software``, etc.). Mirrors the detection in
``xai_signature`` / ``exif_generator`` so removal scrubs exactly what
``identify`` flags, while leaving genuine camera/editor EXIF intact. Returns
the names of the removed tags (for logging).
Mirrors ``aigc_label``'s EXIF path: the ``{"AIGC":{...}}`` wrapper embedded in
``UserComment`` / ``ImageDescription`` by China-served generators (Doubao's
producer schema AND Tencent Cloud's service-provider schema, both keyed under
``_TC260_FIELDS``). Gated on both the ``AIGC`` marker and a TC260 field so a
coincidental token cannot false-drop a genuine caption/comment.
"""
if not isinstance(raw, (bytes, bytearray)):
return False
if b"AIGC" not in raw:
return False
text = bytes(raw).decode("latin-1", "ignore")
return any(field in text for field in _TC260_FIELDS)
def _ai_exif_targets(loaded: dict[str, Any]) -> list[tuple[str, int, bytes, str]]:
"""The SINGLE AI-EXIF rule set, as ``(ifd_key, tag, value_bytes, name)`` entries.
Shared by both EXIF scrubbers so their coverage cannot drift: the JPEG-path
:func:`_scrub_ai_exif` pops each tag, and the ISOBMFF-path
``isobmff.blank_ai_exif_tokens`` blanks each value's bytes in place. Covers
(a) the xAI/Grok ``Signature:`` + UUID-``Artist`` pair, (b) any ``Software`` /
``Make`` / ``Artist`` / ``ImageDescription`` tag carrying an ``AI_GENERATOR_TOKENS``
token, and (c) the China TC260 ``{"AIGC":{...}}`` block in ``ImageDescription``
(0th) or ``UserComment`` (Exif). De-duplicated by ``(ifd_key, tag)`` so a value
flagged by two rules is removed and named once. Mirrors the detection in
``xai_signature`` / ``exif_generator`` / ``aigc_label``; adding a new AI EXIF
placement here reaches BOTH containers.
"""
import piexif
from remove_ai_watermarks.noai.constants import AI_GENERATOR_TOKENS
ifd = exif_dict.get("0th")
if not ifd:
return []
ifd0: dict[int, Any] = loaded.get("0th") or {}
ifde: dict[int, Any] = loaded.get("Exif") or {}
seen: set[tuple[str, int]] = set()
targets: list[tuple[str, int, bytes, str]] = []
drop: dict[int, str] = {}
def add(ifd_key: str, ifd: dict[int, Any], tag: int, name: str) -> None:
value = ifd.get(tag)
if isinstance(value, bytes) and (ifd_key, tag) not in seen:
seen.add((ifd_key, tag))
targets.append((ifd_key, tag, value, name))
# (a) xAI / Grok: the Signature blob and the UUID Artist go together.
if _is_xai_signature_pair(
_exif_text(ifd, piexif.ImageIFD.ImageDescription), _exif_text(ifd, piexif.ImageIFD.Artist)
_exif_text(ifd0, piexif.ImageIFD.ImageDescription), _exif_text(ifd0, piexif.ImageIFD.Artist)
):
drop[piexif.ImageIFD.ImageDescription] = "ImageDescription"
drop[piexif.ImageIFD.Artist] = "Artist"
# (b) Known AI generator token in any of the text tags.
add("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription")
add("0th", ifd0, piexif.ImageIFD.Artist, "Artist")
# (b) known AI generator token in a 0th text tag.
for tag, name in (
(piexif.ImageIFD.Software, "Software"),
(piexif.ImageIFD.Make, "Make"),
(piexif.ImageIFD.Artist, "Artist"),
(piexif.ImageIFD.ImageDescription, "ImageDescription"),
):
if any(token in _exif_text(ifd, tag).lower() for token in AI_GENERATOR_TOKENS):
drop[tag] = name
if any(token in _exif_text(ifd0, tag).lower() for token in AI_GENERATOR_TOKENS):
add("0th", ifd0, tag, name)
# (c) TC260 AIGC block in ImageDescription (0th) or UserComment (Exif sub-IFD).
if _is_aigc_exif_value(ifd0.get(piexif.ImageIFD.ImageDescription)):
add("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription")
if _is_aigc_exif_value(ifde.get(piexif.ExifIFD.UserComment)):
add("Exif", ifde, piexif.ExifIFD.UserComment, "UserComment")
for tag in drop:
ifd.pop(tag, None)
return list(drop.values())
return targets
def _scrub_ai_exif(exif_dict: dict[str, Any]) -> list[str]:
"""Delete the AI-provenance EXIF tags (`_ai_exif_targets`) from a piexif dict's
``0th`` / ``Exif`` IFDs in place; return the removed tag names (for logging).
Genuine camera/editor EXIF is left intact."""
removed: list[str] = []
for ifd_key, tag, _value, name in _ai_exif_targets(exif_dict):
ifd = exif_dict.get(ifd_key)
if ifd is not None:
ifd.pop(tag, None)
removed.append(name)
return removed
def get_ai_metadata(image_path: Path) -> dict[str, str]:
@@ -879,17 +954,50 @@ def _jpeg_app_carries_ai(marker: int, payload: bytes) -> bool:
if marker == 0xEB: # APP11: C2PA / JUMBF manifest
return c2pa_marker_in(payload) or b"jumb" in payload[:256].lower()
if marker == 0xE1 and payload.startswith(b"http://ns.adobe.com/xap/"): # APP1 XMP
return c2pa_marker_in(payload) or any(m in payload for m in AIGC_MARKERS)
return (
c2pa_marker_in(payload)
or any(m in payload for m in AIGC_MARKERS)
or any(m in payload for m in IPTC_AI_MARKERS) # digitalSourceType in XMP, not only APP13
or any(m in payload for m in IPTC_AI_FIELD_MARKERS) # IPTC 2025.1 AI-disclosure fields
)
if marker == 0xED: # APP13: Photoshop / IPTC
return any(m in payload for m in IPTC_AI_MARKERS) or any(m in payload for m in IPTC_AI_FIELD_MARKERS)
# A bare / wrapped China TC260 AIGC block (``AIGC{...}`` or ``{"AIGC":{...}}``) that
# some China-served generators glue into a non-standard APP segment near the JFIF
# header. ``aigc_label`` detects it anywhere in the scan head, so removal must drop
# the carrying segment too (detection<->removal parity). Skip APP1-EXIF (0xE1
# ``Exif``): its camera tags are scrubbed tag-by-tag via piexif, and the AIGC-in-
# UserComment/ImageDescription placement is handled there, so it must not be dropped
# wholesale here.
if 0xE0 <= marker <= 0xEF and not (marker == 0xE1 and payload.startswith(b"Exif")):
return _is_aigc_exif_value(payload)
return False
def _strip_samsung_trailer(scan_and_tail: bytes) -> bytes:
"""Drop a Samsung Galaxy AI editing trailer appended AFTER the JPEG EOI.
Galaxy AI records its ``PhotoEditor_Re_Edit_Data`` (``genAIType``) blob as a
proprietary trailer past the final ``FFD9`` end-of-image, so the verbatim
scan copy in :func:`_strip_jpeg_metadata_lossless` would carry it through. If
the marker is present in the post-EOI trailer, truncate at EOI (the coded scan
is untouched, pixels stay bit-identical). A JPEG with no such trailer -- or a
non-Samsung trailer (e.g. an MPF multi-picture block) -- is returned unchanged.
"""
if _SAMSUNG_EDITOR_MARKER not in scan_and_tail:
return scan_and_tail
eoi = scan_and_tail.rfind(b"\xff\xd9")
if eoi == -1 or _SAMSUNG_EDITOR_MARKER not in scan_and_tail[eoi:]:
return scan_and_tail # marker not in the post-EOI trailer; leave the scan alone
return scan_and_tail[: eoi + 2]
def _strip_jpeg_metadata_lossless(source_path: Path, output_path: Path) -> bool:
"""Remove AI metadata from a JPEG WITHOUT re-encoding the DCT scan, so the pixels
stay bit-identical (the point of "work with originals" -- a metadata strip must not
degrade the image). Walks the marker segments up to SOS, drops the AI-bearing APP
segments (:func:`_jpeg_app_carries_ai`), copies the entropy-coded scan verbatim,
segments (:func:`_jpeg_app_carries_ai`), copies the entropy-coded scan verbatim
(minus a Samsung Galaxy AI trailer past EOI, via :func:`_strip_samsung_trailer`),
then scrubs AI EXIF tags in place via piexif (which rewrites only the APP1 EXIF,
leaving genuine camera EXIF and the scan untouched). Returns False if the bytes are
not a parseable JPEG, so the caller falls back to the near-lossless PIL re-save."""
@@ -905,7 +1013,7 @@ def _strip_jpeg_metadata_lossless(source_path: Path, output_path: Path) -> bool:
return False # malformed marker boundary: defer to the PIL re-encode fallback
marker = data[i + 1]
if marker in (0xDA, 0xD9): # SOS / EOI -> the coded scan follows; copy verbatim
out += data[i:]
out += _strip_samsung_trailer(data[i:])
break
if 0xD0 <= marker <= 0xD7 or marker == 0x01: # standalone markers carry no length
out += data[i : i + 2]
+5 -2
View File
@@ -374,12 +374,15 @@ def _populate_registry_fields(buf: bytes, c2pa_info: dict[str, Any]) -> bool:
c2pa_info["source_type"] = "trainedAlgorithmicMedia (AI-generated)"
c2pa_info["ai_source_kind"] = "generated"
ai_source = True
elif b"algorithmicMedia" in buf:
c2pa_info["source_type"] = "algorithmicMedia"
elif b"compositeWithTrainedAlgorithmicMedia" in buf:
# Checked BEFORE bare ``algorithmicMedia``: a manifest can carry both tokens
# (an AI-enhanced composite with a procedural ingredient), and the bare-token
# branch would otherwise fire first and misclassify the AI composite as non-AI.
c2pa_info["source_type"] = "compositeWithTrainedAlgorithmicMedia (AI-enhanced)"
c2pa_info["ai_source_kind"] = "enhanced"
ai_source = True
elif b"algorithmicMedia" in buf:
c2pa_info["source_type"] = "algorithmicMedia"
# SynthID pixel-watermark proxy: a C2PA manifest from a SynthID-using
# vendor (Google/OpenAI) on AI-generated content implies an invisible
+22 -25
View File
@@ -246,42 +246,39 @@ def blank_ai_exif_tokens(data: bytes) -> tuple[bytes, int]:
``remove_ai_metadata`` (a documented gap). This locates EXIF TIFF blocks by
their byte-order header, **validates each with piexif** (so a coincidental
II/MM run in pixel data is ignored -- it will not parse as a TIFF IFD), and
overwrites any value carrying an ``AI_GENERATOR_TOKENS`` token with spaces of
the SAME length. Because the replacement is same-length, every box size and
``iloc`` offset stays valid and the coded image is untouched -- only the AI tag
content is destroyed; camera/editor EXIF without an AI token is left intact
(mirrors ``metadata._scrub_ai_exif`` and ``blank_ai_xmp_packets``).
overwrites any AI value with spaces of the SAME length. Because the replacement
is same-length, every box size and ``iloc`` offset stays valid and the coded image
is untouched -- only the AI tag content is destroyed; camera/editor EXIF without an
AI token is left intact. This mirrors ``metadata._scrub_ai_exif`` in what it removes
-- generator tokens (``Software``/``Make``/``Artist``/``ImageDescription``), the
China TC260 ``{"AIGC":{...}}`` block (``ImageDescription``/``UserComment``), and the
xAI/Grok ``Signature:`` + UUID-``Artist`` pair -- since on the ISOBMFF path this is
the ONLY EXIF scrubber (``_scrub_ai_exif`` never runs there), so without parity a
HEIC/AVIF AIGC/xAI tag is detected but not removed.
"""
import piexif
from remove_ai_watermarks.noai.constants import AI_GENERATOR_TOKENS
# The AI-EXIF rule set is defined ONCE in metadata._ai_exif_targets and shared by both
# EXIF scrubbers (the JPEG _scrub_ai_exif pops the tag; here we blank the value bytes),
# so their coverage cannot drift. Imported lazily to avoid import-order coupling with
# metadata (which imports this module); a deliberate cross-module use, not an API leak.
from remove_ai_watermarks.metadata import _ai_exif_targets # pyright: ignore[reportPrivateUsage]
ai_tags = (
piexif.ImageIFD.Software,
piexif.ImageIFD.Make,
piexif.ImageIFD.Artist,
piexif.ImageIFD.ImageDescription,
)
out = bytearray(data)
blanked = 0
for header in _TIFF_HEADERS:
pos = data.find(header)
while pos != -1:
window = bytes(out[pos : pos + _EXIF_WINDOW])
ifd: dict[int, Any] = {}
try:
ifd = piexif.load(window).get("0th", {})
loaded: dict[str, Any] = piexif.load(window)
except Exception:
ifd = {}
for tag in ai_tags:
value = ifd.get(tag)
if not isinstance(value, bytes):
continue
if any(token in value.decode("latin1", "replace").lower() for token in AI_GENERATOR_TOKENS):
# Blank the value bytes in place, within this EXIF block only.
vpos = out.find(value, pos, pos + _EXIF_WINDOW)
if vpos != -1:
out[vpos : vpos + len(value)] = b" " * len(value)
blanked += 1
loaded = {}
for _ifd_key, _tag, value, _name in _ai_exif_targets(loaded):
# Blank the value bytes in place, within this EXIF block only.
vpos = out.find(value, pos, pos + _EXIF_WINDOW)
if vpos != -1:
out[vpos : vpos + len(value)] = b" " * len(value)
blanked += 1
pos = data.find(header, pos + len(header))
return bytes(out), blanked
@@ -964,8 +964,9 @@ def remove_watermark(
"""Convenience function to remove watermark from an image.
``strength=None`` lets the profile pick its vendor-adaptive default
(0.20 OpenAI / 0.30 Google / 0.30 unknown, from the C2PA SynthID proxy on the
input; same ladder for the controlnet and sdxl pipelines). Pass a value to override.
(0.10 OpenAI / 0.15 Google / 0.15 unknown, from the C2PA SynthID proxy on the
input; same ladder for the controlnet and sdxl pipelines -- the single source of
truth is ``watermark_profiles.py``). Pass a value to override.
``region=(x, y, w, h)`` restricts the regeneration to that box and preserves the
real photo elsewhere -- for AI-enhanced composites (see