From 408c4c9d5dc85786d0b5af26a5a24c4a7202ae38 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Wed, 29 Jul 2026 13:13:39 -0700 Subject: [PATCH 01/12] Separate provenance extraction and detection, release 0.21.0 --- CLAUDE.md | 2 +- docs/module-internals.md | 11 ++- docs/python-api.md | 22 ++++- pyproject.toml | 2 +- src/remove_ai_watermarks/__init__.py | 2 +- src/remove_ai_watermarks/identify.py | 135 +++++++++++++++++++++------ tests/test_identify.py | 53 +++++++++++ uv.lock | 2 +- 8 files changed, 189 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4223a78..197c726 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ Compact map. The full per-module detail (design decisions, tuned thresholds, cal - `noai/c2pa.py` — C2PA reading. `extract_c2pa_info(path)` uses the official **c2pa-python `Reader`** first (core dep, any container; `read_manifest_store_json` returns the WHOLE store JSON — active + ingredient manifests — so an AI marker on a parent manifest is seen), and falls back to the hand-rolled caBX/CBOR parser (`has_c2pa_metadata` / `extract_c2pa_chunk` / `_extract_c2pa_info_png`) for synthetic/partial blobs the validator rejects or a broken/absent wheel. The registry scan (issuer / source-type / SynthID / soft-binding) is shared by both paths via `_populate_registry_fields`, so the return-dict shape is identical. Do not reimplement chunk parsing; chunk reads are clamped to the remaining file size by design. `extract_c2pa_chunk`/`inject_c2pa_chunk` stay PNG-only (raw caBX bytes, test/extractor use). - `noai/constants.py` — the single `C2PA_AI_VENDORS` registry (+ `C2PA_SOFT_BINDINGS`) from which `C2PA_ISSUERS` / `SYNTHID_C2PA_ISSUERS` / `C2PA_IDENTITY_AI_ORGS` / `identify._ISSUER_PLATFORM` are all derived. Add a new vendor as one registry entry; never edit the derived dicts and never add inline. A vendor's `asserts_ai=True` flag means its mere presence asserts AI generation even without a `trainedAlgorithmicMedia` digital-source-type (a pure-generator brand with a distinctive issuer/generator string, e.g. **Dreamina** — ByteDance's international Jimeng brand, signed as "Bytedance Pte. Ltd." with a "Dreamina/x.y" claim generator and no source-type); NEVER set it for common-word issuers (Adobe/Google/OpenAI/Microsoft) that appear incidentally in unrelated bytes — those stay source-type-gated in `identify._attribute_platform`. - `metadata.py` — `scan_head(path)` is the shared (memoized) input for every C2PA/AIGC/IPTC byte scan; use it instead of `open().read(1MB)` for any new marker scan. Also home to `synthid_source`, `xai_signature`, `iptc_ai_system`, `aigc_label`, `huggingface_job`, `samsung_genai`, and `remove_ai_metadata` (fail-safe `strip_c2pa_boxes`). **A caller that REPORTS an outcome must use `strip_and_verify`, not `remove_ai_metadata` directly** -- the stripper is deliberately fail-safe (a file PIL cannot decode is copied through UNCHANGED rather than crashing), so its return value cannot distinguish a no-op from a real strip. `strip_and_verify` re-scans the output and, when metadata survived but `image_io` can still decode the raster, normalizes the container and scans again; that recovery preserves pixels but drops standard metadata. A truly undecodable file keeps the surviving-marker result. `metadata --remove` and `batch --mode metadata|all` use this verified path. **`remove_ai_metadata` is the SINGLE metadata stripper** (the legacy PIL-re-encoding `noai/cleaner` was deleted; the diffusion core and the public `noai.remove_ai_metadata` re-export now point here). It strips **losslessly** per container: ISOBMFF (HEIC/AVIF/MP4) blanks tokens / strips boxes in place; **JPEG uses `_strip_jpeg_metadata_lossless`** — a marker-segment walk that drops the AI-bearing APP segments (C2PA APP11; XMP APP1 carrying C2PA, a China-AIGC token, OR an IPTC `digitalSourceType` / 2025.1 AI-disclosure marker; IPTC-IIM APP13) and scrubs AI EXIF tags via piexif, copying the entropy-coded scan verbatim so **the pixels are bit-identical** (no DCT re-encode). **Detection<->removal parity across every marker placement is load-bearing** — anything a scanner flags, the strip must reach, or a re-served file still reads as AI: (a) the APP1-XMP branch of `_jpeg_app_carries_ai` checks the IPTC marker sets too, not only C2PA/AIGC (the Instagram/MidJourney/Meta "Made with AI" `digitalSourceType` lives in XMP, not the APP13 IIM record); (b) a bare `AIGC{...}` / `{"AIGC":{...}}` block in ANY JPEG APP segment — the specific C2PA(APP11)/XMP(APP1)/IPTC(APP13) checks FALL THROUGH to a generic `_is_aigc_exif_value` drop, so a bare AIGC in APP11 (the supported placement, NOT a C2PA manifest) is caught, not swallowed by the C2PA-only 0xEB branch — plus the same AIGC block in a STANDARD **PNG text chunk** value (e.g. `Description`, which `_is_ai_key` keeps) is dropped on the value; (c) the China TC260 `{"AIGC":{...}}` block in EXIF `UserComment`/`ImageDescription` is scrubbed by `_scrub_ai_exif` (Doubao producer + Tencent service-provider schemas); (d) the Samsung Galaxy AI `PhotoEditor_Re_Edit_Data` trailer past the JPEG EOI is truncated by `_strip_samsung_trailer` (and `samsung_genai` reads the file tail so a multi-MB photo's trailer past the 512 KB quick-scan window is still DETECTED). Pixels stay bit-identical throughout, so a `--strip-metadata` on a q100 removal output does NOT crush it back to q75; PNG/WebP re-saves are pixel-lossless (WebP written at cv2 lossless mode, quality 101 — quality 1-100 is lossy). **The PIL-fallback save format is chosen by the source's CONTENT, not its file extension** (`_sniff_image_format`, and the JPEG-lossless gate is content-gated too): inputs can be misnamed (a PNG served as `.jpg` is the common one), and routing on the extension re-encoded a lossless PNG/WebP into a real JPEG — a silent degradation that broke "work with originals". A **misnamed** lossless source (source-extension format != content) is preserved in its true format; a **correctly-named** source still honors a deliberate output-extension conversion (e.g. `source.png -> output.jpg`). Not yet handled: a 16-bit PNG is downconverted to 8-bit on the PIL re-save (rare; would need a byte-level PNG chunk stripper). Regression: `tests/test_metadata.py::TestHasAiMetadata::test_strip_preserves_lossless_content_with_mismatched_extension`. **`remove_ai_metadata` is fail-safe on an undecodable image:** a truncated/corrupt file (PIL raises `OSError` decoding it; some inputs) is copied through UNCHANGED rather than crashing a direct library caller (a web worker would 500 on a partial upload), mirroring `strip_c2pa_boxes` — we cannot strip what we cannot parse, but we never raise. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_remove_ai_metadata_failsafe_on_truncated_png,test_strip_and_verify_normalizes_decodable_copy_through,test_strip_and_verify_reports_markers_in_undecodable_copy_through}`. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_jpeg_metadata_strip_is_pixel_lossless, test_jpeg_strip_removes_iptc_marker_in_xmp}`, `TestSamsungGenai::{test_remove_strips_post_eoi_trailer, test_detects_trailer_past_scan_window}`, the AIGC-EXIF/bare-APP removal tests, and `tests/test_noai.py::TestISOBMFF::{test_blank_aigc_block_in_exif, test_blank_xai_signature_pair_in_exif}`. `exif_generator` matches a VALUE against `AI_GENERATOR_TOKENS` across EXIF `Software`/`Make`/`Artist`/`ImageDescription`, XMP `CreatorTool`, AND PNG `tEXt` chunks (`Software`/`Source`/`Title`/`Description` — NovelAI stamps there, not EXIF). **Detection and removal must stay in parity:** a generator that stamps an AI-shaped VALUE under a non-AI KEY (NovelAI's `Title`/`Source`) is dropped on removal by `_is_ai_value` (value-token match, mirrors `exif_generator`), NOT by `_is_ai_key` alone — else the cleaned file still reads as that generator. Add a new no-C2PA generator = one `AI_GENERATOR_TOKENS` entry (use a distinctive token, e.g. `reve.com` not bare `reve`); detection and removal then both follow. Regression: `tests/test_metadata.py::TestExifGenerator::{test_novelai_png_text_chunk_detected,test_novelai_removal_parity}`. -- `identify.py` — aggregates every locally-readable signal into one `ProvenanceReport`; `is_ai_generated` is True or None, never asserted False. `ProvenanceReport.ai_source_kind` exposes the C2PA digital-source-type split — `"generated"` (trainedAlgorithmicMedia, fully AI) vs `"enhanced"` (compositeWithTrainedAlgorithmicMedia, a real photo with an AI-composited region), else None — so a caller branches full-frame scrub vs region-targeted clean (see `noai/tiling.feather_region_composite` + `WatermarkRemover.remove_watermark(region=...)`). The sparkle provenance threshold is the SHARED `watermark_registry.GEMINI_SPARKLE_TRUST_CONF` (imported, not a private copy) so the provenance "is there a sparkle" verdict and the removal "take the sparkle" decision can never drift. `import identify` is deliberately light (lazy `noai/__init__`, fits a 512 MB host) — keep heavy imports out (the `watermark_registry` constant import stays light: engines are lazy there). Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM` only when verified against a real C2PA file; editing-app/AI-device signer tokens go to `_SIGNER_C2PA_PLATFORM`; generator/issuer platforms to `C2PA_AI_VENDORS` in `constants.py`. The IPTC `digitalSourceType` **`algorithmicMedia`** (bare) is PROCEDURAL (an algorithm not trained on sampled data), NOT AI/ML generation, so it is deliberately absent from `IPTC_AI_MARKERS` — flagging it made `identify` assert AI + `has_invisible_target` True, scrubbing clean procedural content (it is a distinct token from `trainedAlgorithmicMedia`, so real "Made with AI" labels are unaffected; regression `test_metadata.py::...test_bare_algorithmic_media_not_flagged_ai`). Integrity-clash detection is high-precision by design (only hard generator stamps feed it, source-grouped independence). `_vendor_of` normalizes ByteDance/Canva/ElevenLabs/Black Forest Labs (as well as OpenAI/Google/... ) so their C2PA claims participate in the clash check; the generic **China TC260 AIGC label names no specific vendor**, so when a TC260-applying vendor (ByteDance, `_TC260_VENDORS`) is co-attributed the label is attributed to it (a legit Doubao image carrying its own TC260 label must NOT clash), while a NON-TC260 vendor next to a TC260 label still clashes as a laundering tell. The vendor normalization must not introduce clashes on clean compatibility samples. +- `identify.py` — separates file-backed metadata extraction (`extract_provenance_evidence`) from metadata-only verdict logic (`identify_from_evidence`); the compatible `identify` wrapper adds optional pixel-backed visible and invisible checks. Detection from `ProvenanceEvidence` must never reopen the source. All paths produce one `ProvenanceReport`; `is_ai_generated` is True or None, never asserted False. `ProvenanceReport.ai_source_kind` exposes the C2PA digital-source-type split — `"generated"` (trainedAlgorithmicMedia, fully AI) vs `"enhanced"` (compositeWithTrainedAlgorithmicMedia, a real photo with an AI-composited region), else None — so a caller branches full-frame scrub vs region-targeted clean (see `noai/tiling.feather_region_composite` + `WatermarkRemover.remove_watermark(region=...)`). The sparkle provenance threshold is the SHARED `watermark_registry.GEMINI_SPARKLE_TRUST_CONF` (imported, not a private copy) so the provenance "is there a sparkle" verdict and the removal "take the sparkle" decision can never drift. `import identify` is deliberately light (lazy `noai/__init__`, fits a 512 MB host) — keep heavy imports out (the `watermark_registry` constant import stays light: engines are lazy there). Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM` only when verified against a real C2PA file; editing-app/AI-device signer tokens go to `_SIGNER_C2PA_PLATFORM`; generator/issuer platforms to `C2PA_AI_VENDORS` in `constants.py`. The IPTC `digitalSourceType` **`algorithmicMedia`** (bare) is PROCEDURAL (an algorithm not trained on sampled data), NOT AI/ML generation, so it is deliberately absent from `IPTC_AI_MARKERS` — flagging it made `identify` assert AI + `has_invisible_target` True, scrubbing clean procedural content (it is a distinct token from `trainedAlgorithmicMedia`, so real "Made with AI" labels are unaffected; regression `test_metadata.py::...test_bare_algorithmic_media_not_flagged_ai`). Integrity-clash detection is high-precision by design (only hard generator stamps feed it, source-grouped independence). `_vendor_of` normalizes ByteDance/Canva/ElevenLabs/Black Forest Labs (as well as OpenAI/Google/... ) so their C2PA claims participate in the clash check; the generic **China TC260 AIGC label names no specific vendor**, so when a TC260-applying vendor (ByteDance, `_TC260_VENDORS`) is co-attributed the label is attributed to it (a legit Doubao image carrying its own TC260 label must NOT clash), while a NON-TC260 vendor next to a TC260 label still clashes as a laundering tell. The vendor normalization must not introduce clashes on clean compatibility samples. - `watermark_registry.py` — the single catalog of known visible watermarks (gemini / doubao / jimeng / qwen / kling / samsung / runninghub / baidu / liblib / jimeng_pill). **Removal is LOCALIZE -> FILL for every mark:** each mark is localized to a binary full-frame footprint mask (a `Localization`), then ONE shared, swappable fill inpaints that mask via `fill(image, mask, backend=...)` (delegates to `region_eraser.erase`). Reverse-alpha (the old `original = (wm - a*logo)/(1-a)` inversion of a captured alpha map + thin residual inpaint) is GONE for ALL marks; why it was dropped is recorded in `docs/module-internals.md`. Backends: `cv2` (classical inpaint, no deps, the floor), `migan` (MI-GAN ONNX, light, the memory-tight pick where LaMa will not fit), `lama` (big-LaMa ONNX, best quality, heavier, auto-preferred when a learned backend is available); `auto` = LaMa > MI-GAN > cv2, best available. The captured alpha maps (`scripts/visible_alpha_solve.py`) are still used to DETECT the marks and to shape the mask, but NOT for pixel recovery. **`--mark auto` removes EVERY detected mark in one pass** via `remove_auto_marks(image, *, sensitivity="auto", provenance=frozenset(), backend="auto")` (marks coexist -- a Jimeng-basic image has the top-left pill AND the bottom-right wordmark; a single-strongest pick would leave one). **Three orthogonal axes:** `backend` (the fill), `sensitivity` (how hard to trust a borderline mark: `auto`/`strict`, see the `Sensitivity` literal), and `provenance` (vendor keys metadata confirms -- the evidence that drives `auto`). **Perception / decision / action are separated:** `_build_candidates(image)` runs every detector at BOTH trust levels (strict + relaxed) and packages raw verdicts + features into `Candidate`s (no policy); the pure arbiter `decide(candidates, Context(sensitivity, provenance)) -> [Decision]` makes every keep/drop call (per-mark `resolve_trust` + the assumed-trust floor + the pill gate) with no image/IO, so it is unit-testable in isolation; then each winner is localized -> filled. Do NOT put policy back into the engines (the one exception, the Gemini FP gate, stays in `gemini_engine` because `identify` shares that confidence). `detect_marks(..., provenance=frozenset())` stays strict (identify verdict, precision over recall); `KnownMark.remove/detect/localize(..., provenance: bool)` take the already-resolved boolean. **How `auto` decides (this is metadata-INDEPENDENT for recall):** the visual detectors are pixel-based and need no metadata; the recall gain comes from RELAXING the false-positive gate, not from metadata. `strict` never relaxes (clean images untouched); `auto` relaxes a mark only on same-product evidence -- metadata provenance for that vendor OR a confidently detected sibling mark of the SAME product (`_PRODUCT_OF`; Doubao and Jimeng are both bottom-right ByteDance but distinct products, so they do NOT cross-relax). **`resolve_trust` resolves TWO levels:** `confirmed` bypasses the engine's false-positive gate, and only `confirmed` has evidence naming THAT vendor, which is exactly what the bypass is contracted to require (`GeminiEngine.detect_watermark`'s `trust_provenance` docstring: "external metadata already proves this is a Google generation"). **Historical, kept as the reason the third level is gone:** a removed `assumed` level let `assume_ai` bypass the gate on the bare assertion an image is AI. It produced unacceptable false positives on clean camera images and was removed. A wrong relaxation only fills a small corner near-losslessly (the localize -> fill benign failure mode), which is what made a SMALL false-fire rate arguable; it was never a licence for a 60% one. Metadata provenance mapping (feeds `auto`, read by `cli._visible_provenance`): Google/Gemini C2PA issuer -> gemini; China-AIGC (TC260) label -> doubao/jimeng; `samsung_genai` -> samsung. **The `jimeng_pill` is capture-less.** It uses a synthetic silhouette and a fixed top-left footprint. `_keep_pill` requires either a confirmed sibling wordmark or Jimeng provenance with a flat footprint. Do not loosen the flatness guard. **`assume_ai` was REMOVED (2026-07-19); `--sensitivity` is now `auto`/`strict` only.** It relaxed every mark's FP gate on the bare assertion an image is AI -- which names no vendor and no location, exactly what the bypass requires -- and had no place in the model (detector finds -> remove; finds nothing -> leave alone; user SEES a mark -> act on that). It took `_ASSUMED_CONF_FLOOR` / `assumed_floor_ok` / the `assumed` trust level with it, collapsing the ladder to `strict`/`confirmed`, and `_keep_pill` lost its `sensitivity` arg. Recall/precision on the unbiased sample are unchanged, so nothing on the default path moved. **Replacement advice is per mark:** `erase --region` is sound by construction; `--mark --no-detect` is reasonable (forced mask = the real glyph blob, non-empty 13/13); **`--mark gemini --no-detect` is NOT** -- it falls back to a fixed slot that covered the true sparkle on only **31% of 97** missed sparkles, so 69% fill a clean corner AND report a removal that did not happen. `cli._no_visible_mark_exit` follows that order and no longer suggests the removed mode. Migration raises loudly (`validate_sensitivity`, called from `api.remove_visible` and `Context.__post_init__`) because a `Literal` is unenforced at runtime and would silently downgrade a 0.15 caller to `auto`. **Detection and removal-mask extraction must use the same front-end.** The continuous `tophat` path can detect a mark whose binary glyph blob is empty, so its fallback mask uses the detector's own best-match box. Keep the textured regression fixture when changing this path. **Detector thresholds and geometry are calibrated per mark; do not port them between vendors without a fresh evaluation.** **New mark assets must be synthetic.** Font-render the mark, calibrate it on local evaluation inputs, and commit only the synthetic silhouette. Never derive a committed asset from source images. - `gemini_engine.py` — visible Gemini-sparkle detector + localizer (cv2/numpy, no GPU): top-K size-weighted fusion candidate selection (`_SELECT_TOPK`), corner-promote, false-positive gate (the provenance prior relaxes the gate + lowers the trust threshold when a Google/Gemini C2PA issuer confirms the vendor). **White-core rescue:** the FP gate demotes a low-gradient match (soft edges), but a real FAINT sparkle also has soft edges -- so the gate keeps a low-grad match that is a strong (conf ≥ `_SPARKLE_KEEP_CONF` 0.52), bright (margin), near-WHITE-core sparkle (`_core_saturation` ≤ `_SPARKLE_WHITE_SAT` 0.20): a real sparkle core is white, a clean bright corner that shape-matches (sky/sun) is colored. This recovers ~14/20 metadata-stripped faint sparkles under the DEFAULT strict/auto (no flag, no metadata) at ~1.25% clean false-fire (baseline 0.55%); the ~0.51-scoring bright-bg FPs stay demoted (below 0.52). A learned classifier on the SAME features was measured WORSE than the tuned gate (2026-07 tier-1: MLP 86.7% recall vs 90.8% at equal FP), so the heuristic stays; a patch-CNN with richer features is the only lever left (roadmapped P2, low expected value -- the wall is fundamental). Detection scores the top-K size-weighted matches by full fusion (spatial+gradient+variance) and keeps the highest — NOT the raw-NCC argmax, which re-admits the tiny-patch FPs the size weight suppresses (the osachub 2026-06-12 sub-0.85 corner-sparkle regression; see `docs/module-internals.md`). Keep the 0.85 corner-promote NCC gate; a margin/chroma-gated lower promote was measured and REJECTED 2026-06-11 (~33% FP on non-Google content). Removal is localize -> fill: `footprint_mask` returns the sparkle footprint (the captured alpha thresholded LOW so the faint halo is included, then dilated by a sparkle-relative margin), and the shared `watermark_registry.fill` inpaints it. The captured alpha maps are used only to detect and to shape the mask, not for pixel recovery. `detect_sparkle_confidence` reuses a process-wide `_shared_engine()` singleton (lru_cache) — the engine holds only constant assets (captures, alpha maps, a precomputed 16..118 template ladder) and takes the image as an arg, so do NOT reconstruct `GeminiEngine()` per call: that reloaded assets + recomputed alpha maps + rebuilt the template cache on every one of ~34k `identify` calls (−24% on the sparkle path once made a singleton, output byte-identical). `detect_watermark`/`footprint_mask` guard `image.size == 0` before `to_bgr`, and return an empty (detected=False) result when no template scale fits (short side < 16 px), rather than dereferencing an empty candidate list. - `_text_mark_engine.py` — shared base for the text-mark engines (extracted 2026-06-09); the per-engine modules are config-only subclasses. Detection still matches the glyph silhouette (NCC, keys on glyph shape). The removal mask is TEMPLATE-FREE: it is the bounding box of the top-hat glyph blob (`extract_mask`), filled solid + dilated, so the shared fill inpaints the whole wordmark rectangle. This drops the fixed alpha-template placement, so a re-rendered or differently-placed mark is still masked; the captured alpha maps are now used only for the detection silhouette, not for removal. New text mark = a `TextMarkConfig` + a thin subclass + one registry row. Gemini stays a separate engine (different model). The corner anchor is `corner` = `br`/`bl`/`tl`/`bc` (tl added 2026-07-22 for runninghub, bc for liblib's centered wordmark); `detect_frontend` is `binary`/`tophat`/`gray` (`gray` = raw-grayscale NCC for the faint mid-gray runninghub mark, added 2026-07-22; contrast-DEPENDENT, so its gates never port). The detection scale ladder is per-mark (`TextMarkConfig.ladder`, default `(0.8, 1.0, 1.25)` -- added 2026-07-21 for qwen's two size modes; the shared default is unchanged for every other mark, and densifying the SHARED ladder was measured and rejected, see `docs/verification-plan.md` B2). diff --git a/docs/module-internals.md b/docs/module-internals.md index 8432062..ce02f2b 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -131,9 +131,14 @@ Regression coverage: ### Provenance report -[`identify.py`](../src/remove_ai_watermarks/identify.py) combines metadata, -registered visible marks, and optional open invisible-watermark decoders into a -`ProvenanceReport`. +[`identify.py`](../src/remove_ai_watermarks/identify.py) separates file-backed +metadata extraction from verdict logic: + +- `extract_provenance_evidence` reads the supported metadata signals into + `ProvenanceEvidence`. +- `identify_from_evidence` evaluates that evidence without reopening the source. +- `identify` preserves the path-based API and adds the optional registered + visible-mark and open invisible-watermark decoders after extraction. `is_ai_generated` is `True` or `None`; absence of evidence is not reported as a human-made verdict. `ai_source_kind` distinguishes fully generated content from diff --git a/docs/python-api.md b/docs/python-api.md index 4bc2af6..758ee27 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -88,8 +88,8 @@ print(report.platform) print(report.signals) ``` -Use `check_visible=False` and `check_invisible=False` for metadata only -inspection: +Use `check_visible=False` and `check_invisible=False` for metadata-only +inspection through the compatible path-based API: ```python report = identify( @@ -99,6 +99,24 @@ report = identify( ) ``` +Extraction and detection are also available as separate steps. This is useful +when a file-reading worker collects the metadata once and another component +evaluates the resulting evidence: + +```python +from remove_ai_watermarks.identify import ( + extract_provenance_evidence, + identify_from_evidence, +) + +evidence = extract_provenance_evidence(Path("input.png")) +report = identify_from_evidence(evidence) +``` + +`identify_from_evidence` does not reopen the source file. It evaluates metadata +only; registered visible marks and pixel-backed invisible watermarks remain in +the path-based `identify` call. + ## Strip metadata ```python diff --git a/pyproject.toml b/pyproject.toml index dd846ca..350dfcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.20.2" +version = "0.21.0" description = "AI watermark remover: strip visible and invisible AI watermarks (Gemini / Nano Banana sparkle, SynthID) and provenance metadata (C2PA, EXIF) from images" readme = "README.md" requires-python = ">=3.10.1" diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index b7cceda..0e20249 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -25,7 +25,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") _warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*") -__version__ = "0.20.2" +__version__ = "0.21.0" __all__ = ["__version__", "remove_visible", "visible_provenance"] diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index ab554a9..3c1d200 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -132,6 +132,28 @@ class Signal: confidence: str # "high" | "medium" +@dataclass(frozen=True) +class ProvenanceEvidence: + """Extracted metadata evidence used by provenance detection. + + Extraction is intentionally separate from verdict logic so a caller can + collect the file-backed evidence once and evaluate it without reopening the + source. Pixel-backed visible and invisible watermark checks remain part of + :func:`identify`. + """ + + path: Path + c2pa_info: dict[str, Any] + ai_metadata: dict[str, str] + scan: bytes + iptc_ai_system: str | None + aigc_label: dict[str, str] | None + exif_generator: str | None + xai_signature: bool + huggingface_job: str | None + samsung_genai: int | None + + @dataclass class ProvenanceReport: """Aggregated provenance verdict for one image.""" @@ -166,6 +188,22 @@ class ProvenanceReport: integrity_clashes: list[str] = field(default_factory=list[str]) +def extract_provenance_evidence(image_path: Path) -> ProvenanceEvidence: + """Read all file-backed metadata needed by provenance verdict logic once.""" + return ProvenanceEvidence( + path=image_path, + c2pa_info=extract_c2pa_info(image_path), + ai_metadata=get_ai_metadata(image_path), + scan=scan_head(image_path, _SCAN_BYTES), + iptc_ai_system=iptc_ai_system(image_path), + aigc_label=aigc_label(image_path), + exif_generator=exif_generator(image_path), + xai_signature=xai_signature(image_path), + huggingface_job=huggingface_job(image_path), + samsung_genai=samsung_genai(image_path), + ) + + def _issuers_in(data: bytes) -> list[str]: """C2PA issuer names whose signature byte appears in ``data`` (binary scan).""" return sorted({name for sig, name in C2PA_ISSUERS.items() if sig in data}) @@ -549,29 +587,25 @@ def _collect_visible_signals( return platform -def identify(image_path: Path, *, check_visible: bool = True, check_invisible: bool = True) -> ProvenanceReport: - """Identify an image's origin platform and watermark inventory. +def _identify_from_evidence( + evidence: ProvenanceEvidence, + *, + image_path: Path | None = None, + check_visible: bool = False, + check_invisible: bool = False, +) -> ProvenanceReport: + """Build a provenance verdict from extracted evidence. - Args: - image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container). - check_visible: Also run the registered visible-mark detectors through cv2. - Set False for a pure-metadata, dependency-light scan. - check_invisible: Also decode open invisible watermarks (SD/SDXL/FLUX) via - the optional imwatermark library. No-op when it is not installed. - - Returns: - A :class:`ProvenanceReport`. ``is_ai_generated`` is True when any AI - signal is found and None (unknown) when none is -- it is never asserted - False, because stripped metadata leaves no local proof of a clean origin. + ``image_path`` is supplied only by :func:`identify` for optional pixel + detectors. Metadata-only callers leave it unset and never reopen the source. """ - info = extract_c2pa_info(image_path) # PNG-structured; {} for other formats - meta = get_ai_metadata(image_path) # PNG text + EXIF + C2PA fields + synthid + if (check_visible or check_invisible) and image_path is None: + raise ValueError("Pixel-backed checks require image_path") + pixel_path = image_path - # First MB covers C2PA (PNG caBX, JPEG APP11, AVIF/HEIF/JXL uuid box) and - # IPTC markers for the non-PNG path where extract_c2pa_info returns {}. - # scan_head also seeks out late ISOBMFF provenance boxes (manifest after a - # large mdat in a streaming MP4) that a fixed first-MB read would miss. - head = scan_head(image_path, _SCAN_BYTES) + info = evidence.c2pa_info + meta = evidence.ai_metadata + head = evidence.scan signals: list[Signal] = [] watermarks: list[str] = [] @@ -688,7 +722,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # ── IPTC 2025.1 AI-disclosure fields (Iptc4xmpExt:AISystemUsed etc.) ─ iptc_ai = any(m in head for m in IPTC_AI_FIELD_MARKERS) if iptc_ai: - system = iptc_ai_system(image_path) + system = evidence.iptc_ai_system named = bool(system) and system != "fields present" signals.append( Signal("iptc_ai_system", f"IPTC AI disclosure ({system})" if named else "IPTC AI disclosure fields", "high") @@ -704,7 +738,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # URL, present in XMP and as a laundering tell even when the JSON payload is # truncated) OR the parsed label, which additionally catches the raw-JSON # PNG ``AIGC`` tEXt chunk that carries no namespaced marker at all. - aigc_data = aigc_label(image_path) + aigc_data = evidence.aigc_label aigc = aigc_data is not None or any(m in head for m in AIGC_MARKERS) if aigc: producer = (aigc_data or {}).get("ContentProducer", "") @@ -725,7 +759,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # ── EXIF Software / XMP CreatorTool / PNG-text generator (cross-format) ─ # Catches a generator tag (incl. inside AVIF/HEIF/JXL and PNG text chunks) # when there is no C2PA. - if generator_tag := exif_generator(image_path): + if generator_tag := evidence.exif_generator: signals.append(Signal("exif_generator", f"Embedded generator tag: {generator_tag}", "high")) watermarks.append(f"Embedded generator tag: {generator_tag}") if platform is None: @@ -737,7 +771,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # Grok's only provenance signal: EXIF ImageDescription "Signature: " # + a UUID Artist. Distinct from exif_generator (which matches generator # tokens); verified stable across 3 generations. See CLAUDE.md. - if xai_signature(image_path): + if evidence.xai_signature: signals.append(Signal("xai_signature", "EXIF Signature blob + UUID Artist", "high")) watermarks.append("xAI/Grok EXIF signature") if platform is None: @@ -748,7 +782,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # Marks the hosting job, not a model -- medium confidence (commonly diffusion # output). Like the visible sparkle, it lifts an otherwise-Unknown verdict to # a tentative AI, but never overrides a high-confidence metadata signal. - hf_job = huggingface_job(image_path) + hf_job = evidence.huggingface_job if hf_job: signals.append(Signal("hf_job", f"HuggingFace job {hf_job}", "medium")) watermarks.append("HuggingFace-hosted job (hf-job-id)") @@ -764,7 +798,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # verdict, but the field is undocumented, so it never overrides a high- # confidence signal. The platform is usually already "Samsung Galaxy" via the # signer-token scan; the fallback covers a future file without the cert org. - samsung_genai_type = samsung_genai(image_path) + samsung_genai_type = evidence.samsung_genai if samsung_genai_type is not None: signals.append(Signal("samsung_genai", f"Samsung genAIType={samsung_genai_type}", "medium")) watermarks.append("Samsung Galaxy AI editing marker (genAIType)") @@ -774,7 +808,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # ── Open invisible watermark (SD / SDXL / FLUX, dwtDct) ────────── # Public decoder, no key -- a definitive embedded signal on pristine files. - if check_invisible and (scheme := _invisible_watermark(image_path)) is not None: + if check_invisible and pixel_path is not None and (scheme := _invisible_watermark(pixel_path)) is not None: signals.append(Signal("invisible_watermark", scheme, "high")) watermarks.append(f"Open invisible watermark: {scheme}") caveats.append(_INVISIBLE_WM_CAVEAT) @@ -785,7 +819,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b # The watermark behind Adobe Durable Content Credentials. Decoded locally, # but it binds provenance for human-authored content too, so it enriches the # watermark inventory without by itself asserting AI origin. - if check_invisible and (tm_scheme := _trustmark(image_path)) is not None: + if check_invisible and pixel_path is not None and (tm_scheme := _trustmark(pixel_path)) is not None: signals.append(Signal("trustmark", tm_scheme, "high")) watermarks.append(f"Adobe TrustMark invisible watermark ({tm_scheme})") if platform is None: @@ -806,8 +840,8 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b or xai_sig ) - if check_visible: - platform = _collect_visible_signals(image_path, signals, watermarks, platform) + if check_visible and pixel_path is not None: + platform = _collect_visible_signals(pixel_path, signals, watermarks, platform) visible_only = any(s.name.startswith("visible_") for s in signals) and not ai_from_metadata hf_only = bool(hf_job) and not ai_from_metadata @@ -831,7 +865,7 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b caveats = list(dict.fromkeys(caveats)) return ProvenanceReport( - path=image_path, + path=evidence.path, is_ai_generated=is_ai, platform=platform, confidence=confidence, @@ -846,6 +880,45 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b ) +def identify_from_evidence(evidence: ProvenanceEvidence) -> ProvenanceReport: + """Build a metadata-only provenance verdict without reopening the source.""" + return _identify_from_evidence(evidence) + + +def identify( + image_path: Path, + *, + check_visible: bool = True, + check_invisible: bool = True, +) -> ProvenanceReport: + """Identify an image's origin platform and watermark inventory. + + Args: + image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container). + check_visible: Also run the registered visible-mark detectors through cv2. + Set False for a metadata-only, dependency-light scan. + check_invisible: Also decode open invisible watermarks (SD/SDXL/FLUX) via + the optional imwatermark library. No-op when it is not installed. + + File-backed metadata extraction runs first. The extracted evidence is then + evaluated independently, followed by the optional pixel-backed visible and + invisible watermark checks. + + Returns: + A :class:`ProvenanceReport`. ``is_ai_generated`` is True when any AI + signal is found and None (unknown) when none is found. It is never + asserted False because stripped metadata leaves no local proof of a + clean origin. + """ + evidence = extract_provenance_evidence(image_path) + return _identify_from_evidence( + evidence, + image_path=image_path, + check_visible=check_visible, + check_invisible=check_invisible, + ) + + def has_invisible_target(image_path: Path) -> bool: """True when a locally-detectable invisible/metadata AI signal is present. diff --git a/tests/test_identify.py b/tests/test_identify.py index 833d30b..caaaca8 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -16,14 +16,17 @@ from unittest.mock import patch import pytest from remove_ai_watermarks.identify import ( + ProvenanceEvidence, ProvenanceReport, _ai_tools_in, _attribute_platform, _integrity_clashes, _issuers_in, _vendor_of, + extract_provenance_evidence, has_invisible_target, identify, + identify_from_evidence, ) from remove_ai_watermarks.watermark_registry import GEMINI_SPARKLE_TRUST_CONF @@ -33,6 +36,56 @@ _SPARKLE_TARGET = "remove_ai_watermarks.gemini_engine.detect_sparkle_confidence" SAMPLES_DIR = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance" +class TestProvenanceEvidence: + @pytest.mark.parametrize( + "filename", + [ + "chatgpt-1.png", + "chatgpt-2.png", + "doubao-1.png", + "firefly-1.png", + "flux-1.jpg", + "flux-1.png", + "grok-1.jpg", + "mj-1.png", + ], + ) + def test_metadata_only_identify_matches_extracted_evidence(self, filename: str): + path = SAMPLES_DIR / filename + + direct = identify(path, check_visible=False, check_invisible=False) + evidence = extract_provenance_evidence(path) + extracted = identify_from_evidence(evidence) + + assert isinstance(evidence, ProvenanceEvidence) + assert extracted == direct + + def test_identify_from_evidence_does_not_read_the_source(self, monkeypatch, tmp_path: Path): + path = tmp_path / "generated.jpg" + path.write_bytes(b"\xff\xd8\xff\xe1jumbc2paOpenAI DALL-E trainedAlgorithmicMedia\xff\xd9") + evidence = extract_provenance_evidence(path) + + def fail_if_called(*args, **kwargs): + raise AssertionError("identify_from_evidence must not read the source file") + + monkeypatch.setattr("remove_ai_watermarks.identify.extract_c2pa_info", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.get_ai_metadata", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.scan_head", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.iptc_ai_system", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.aigc_label", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.exif_generator", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.xai_signature", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.huggingface_job", fail_if_called) + monkeypatch.setattr("remove_ai_watermarks.identify.samsung_genai", fail_if_called) + monkeypatch.setattr("builtins.open", fail_if_called) + monkeypatch.setattr(Path, "open", fail_if_called) + + report = identify_from_evidence(evidence) + + assert report.is_ai_generated is True + assert any(signal.name == "c2pa" for signal in report.signals) + + # ── Pure attribution logic (no file IO) ───────────────────────────── diff --git a/uv.lock b/uv.lock index 62924fd..edf9f42 100644 --- a/uv.lock +++ b/uv.lock @@ -3187,7 +3187,7 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.20.2" +version = "0.21.0" source = { editable = "." } dependencies = [ { name = "c2pa-python" }, From c013704ded46457d4d5c7fdd9880f59ae147f1e8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:17:17 +0000 Subject: [PATCH 02/12] Sync conda recipe with v0.21.0 --- packaging/conda/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index a684e01..5d098d8 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -1,7 +1,7 @@ schema_version: 1 context: - version: "0.20.2" + version: "0.21.0" python_min: "3.10" package: @@ -10,7 +10,7 @@ package: source: url: https://pypi.org/packages/source/r/remove-ai-watermarks/remove_ai_watermarks-${{ version }}.tar.gz - sha256: 99a3991d30c80e1719122464a26befd2f372cfa8e3b1b3909f2869fce35e0f24 + sha256: 69eaa8cbb25eec113916358bad4baf4d067cb41854f92d389d2b5c0f22f94f94 build: noarch: python From 4ef180541f93de84fd708e2660349073600779d6 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Wed, 29 Jul 2026 14:32:25 -0700 Subject: [PATCH 03/12] Add external metadata evidence ingestion --- CLAUDE.md | 2 +- docs/module-internals.md | 2 + docs/python-api.md | 22 +++ docs/watermarking-landscape.md | 2 +- pyproject.toml | 2 +- src/remove_ai_watermarks/__init__.py | 2 +- src/remove_ai_watermarks/identify.py | 193 +++++++++++++++++++++++++- src/remove_ai_watermarks/metadata.py | 180 ++++++++++++------------ src/remove_ai_watermarks/noai/c2pa.py | 50 +++++-- tests/test_identify.py | 24 ++++ uv.lock | 148 ++++++++++---------- 11 files changed, 446 insertions(+), 181 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 197c726..84c177a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ Compact map. The full per-module detail (design decisions, tuned thresholds, cal - `noai/c2pa.py` — C2PA reading. `extract_c2pa_info(path)` uses the official **c2pa-python `Reader`** first (core dep, any container; `read_manifest_store_json` returns the WHOLE store JSON — active + ingredient manifests — so an AI marker on a parent manifest is seen), and falls back to the hand-rolled caBX/CBOR parser (`has_c2pa_metadata` / `extract_c2pa_chunk` / `_extract_c2pa_info_png`) for synthetic/partial blobs the validator rejects or a broken/absent wheel. The registry scan (issuer / source-type / SynthID / soft-binding) is shared by both paths via `_populate_registry_fields`, so the return-dict shape is identical. Do not reimplement chunk parsing; chunk reads are clamped to the remaining file size by design. `extract_c2pa_chunk`/`inject_c2pa_chunk` stay PNG-only (raw caBX bytes, test/extractor use). - `noai/constants.py` — the single `C2PA_AI_VENDORS` registry (+ `C2PA_SOFT_BINDINGS`) from which `C2PA_ISSUERS` / `SYNTHID_C2PA_ISSUERS` / `C2PA_IDENTITY_AI_ORGS` / `identify._ISSUER_PLATFORM` are all derived. Add a new vendor as one registry entry; never edit the derived dicts and never add inline. A vendor's `asserts_ai=True` flag means its mere presence asserts AI generation even without a `trainedAlgorithmicMedia` digital-source-type (a pure-generator brand with a distinctive issuer/generator string, e.g. **Dreamina** — ByteDance's international Jimeng brand, signed as "Bytedance Pte. Ltd." with a "Dreamina/x.y" claim generator and no source-type); NEVER set it for common-word issuers (Adobe/Google/OpenAI/Microsoft) that appear incidentally in unrelated bytes — those stay source-type-gated in `identify._attribute_platform`. - `metadata.py` — `scan_head(path)` is the shared (memoized) input for every C2PA/AIGC/IPTC byte scan; use it instead of `open().read(1MB)` for any new marker scan. Also home to `synthid_source`, `xai_signature`, `iptc_ai_system`, `aigc_label`, `huggingface_job`, `samsung_genai`, and `remove_ai_metadata` (fail-safe `strip_c2pa_boxes`). **A caller that REPORTS an outcome must use `strip_and_verify`, not `remove_ai_metadata` directly** -- the stripper is deliberately fail-safe (a file PIL cannot decode is copied through UNCHANGED rather than crashing), so its return value cannot distinguish a no-op from a real strip. `strip_and_verify` re-scans the output and, when metadata survived but `image_io` can still decode the raster, normalizes the container and scans again; that recovery preserves pixels but drops standard metadata. A truly undecodable file keeps the surviving-marker result. `metadata --remove` and `batch --mode metadata|all` use this verified path. **`remove_ai_metadata` is the SINGLE metadata stripper** (the legacy PIL-re-encoding `noai/cleaner` was deleted; the diffusion core and the public `noai.remove_ai_metadata` re-export now point here). It strips **losslessly** per container: ISOBMFF (HEIC/AVIF/MP4) blanks tokens / strips boxes in place; **JPEG uses `_strip_jpeg_metadata_lossless`** — a marker-segment walk that drops the AI-bearing APP segments (C2PA APP11; XMP APP1 carrying C2PA, a China-AIGC token, OR an IPTC `digitalSourceType` / 2025.1 AI-disclosure marker; IPTC-IIM APP13) and scrubs AI EXIF tags via piexif, copying the entropy-coded scan verbatim so **the pixels are bit-identical** (no DCT re-encode). **Detection<->removal parity across every marker placement is load-bearing** — anything a scanner flags, the strip must reach, or a re-served file still reads as AI: (a) the APP1-XMP branch of `_jpeg_app_carries_ai` checks the IPTC marker sets too, not only C2PA/AIGC (the Instagram/MidJourney/Meta "Made with AI" `digitalSourceType` lives in XMP, not the APP13 IIM record); (b) a bare `AIGC{...}` / `{"AIGC":{...}}` block in ANY JPEG APP segment — the specific C2PA(APP11)/XMP(APP1)/IPTC(APP13) checks FALL THROUGH to a generic `_is_aigc_exif_value` drop, so a bare AIGC in APP11 (the supported placement, NOT a C2PA manifest) is caught, not swallowed by the C2PA-only 0xEB branch — plus the same AIGC block in a STANDARD **PNG text chunk** value (e.g. `Description`, which `_is_ai_key` keeps) is dropped on the value; (c) the China TC260 `{"AIGC":{...}}` block in EXIF `UserComment`/`ImageDescription` is scrubbed by `_scrub_ai_exif` (Doubao producer + Tencent service-provider schemas); (d) the Samsung Galaxy AI `PhotoEditor_Re_Edit_Data` trailer past the JPEG EOI is truncated by `_strip_samsung_trailer` (and `samsung_genai` reads the file tail so a multi-MB photo's trailer past the 512 KB quick-scan window is still DETECTED). Pixels stay bit-identical throughout, so a `--strip-metadata` on a q100 removal output does NOT crush it back to q75; PNG/WebP re-saves are pixel-lossless (WebP written at cv2 lossless mode, quality 101 — quality 1-100 is lossy). **The PIL-fallback save format is chosen by the source's CONTENT, not its file extension** (`_sniff_image_format`, and the JPEG-lossless gate is content-gated too): inputs can be misnamed (a PNG served as `.jpg` is the common one), and routing on the extension re-encoded a lossless PNG/WebP into a real JPEG — a silent degradation that broke "work with originals". A **misnamed** lossless source (source-extension format != content) is preserved in its true format; a **correctly-named** source still honors a deliberate output-extension conversion (e.g. `source.png -> output.jpg`). Not yet handled: a 16-bit PNG is downconverted to 8-bit on the PIL re-save (rare; would need a byte-level PNG chunk stripper). Regression: `tests/test_metadata.py::TestHasAiMetadata::test_strip_preserves_lossless_content_with_mismatched_extension`. **`remove_ai_metadata` is fail-safe on an undecodable image:** a truncated/corrupt file (PIL raises `OSError` decoding it; some inputs) is copied through UNCHANGED rather than crashing a direct library caller (a web worker would 500 on a partial upload), mirroring `strip_c2pa_boxes` — we cannot strip what we cannot parse, but we never raise. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_remove_ai_metadata_failsafe_on_truncated_png,test_strip_and_verify_normalizes_decodable_copy_through,test_strip_and_verify_reports_markers_in_undecodable_copy_through}`. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_jpeg_metadata_strip_is_pixel_lossless, test_jpeg_strip_removes_iptc_marker_in_xmp}`, `TestSamsungGenai::{test_remove_strips_post_eoi_trailer, test_detects_trailer_past_scan_window}`, the AIGC-EXIF/bare-APP removal tests, and `tests/test_noai.py::TestISOBMFF::{test_blank_aigc_block_in_exif, test_blank_xai_signature_pair_in_exif}`. `exif_generator` matches a VALUE against `AI_GENERATOR_TOKENS` across EXIF `Software`/`Make`/`Artist`/`ImageDescription`, XMP `CreatorTool`, AND PNG `tEXt` chunks (`Software`/`Source`/`Title`/`Description` — NovelAI stamps there, not EXIF). **Detection and removal must stay in parity:** a generator that stamps an AI-shaped VALUE under a non-AI KEY (NovelAI's `Title`/`Source`) is dropped on removal by `_is_ai_value` (value-token match, mirrors `exif_generator`), NOT by `_is_ai_key` alone — else the cleaned file still reads as that generator. Add a new no-C2PA generator = one `AI_GENERATOR_TOKENS` entry (use a distinctive token, e.g. `reve.com` not bare `reve`); detection and removal then both follow. Regression: `tests/test_metadata.py::TestExifGenerator::{test_novelai_png_text_chunk_detected,test_novelai_removal_parity}`. -- `identify.py` — separates file-backed metadata extraction (`extract_provenance_evidence`) from metadata-only verdict logic (`identify_from_evidence`); the compatible `identify` wrapper adds optional pixel-backed visible and invisible checks. Detection from `ProvenanceEvidence` must never reopen the source. All paths produce one `ProvenanceReport`; `is_ai_generated` is True or None, never asserted False. `ProvenanceReport.ai_source_kind` exposes the C2PA digital-source-type split — `"generated"` (trainedAlgorithmicMedia, fully AI) vs `"enhanced"` (compositeWithTrainedAlgorithmicMedia, a real photo with an AI-composited region), else None — so a caller branches full-frame scrub vs region-targeted clean (see `noai/tiling.feather_region_composite` + `WatermarkRemover.remove_watermark(region=...)`). The sparkle provenance threshold is the SHARED `watermark_registry.GEMINI_SPARKLE_TRUST_CONF` (imported, not a private copy) so the provenance "is there a sparkle" verdict and the removal "take the sparkle" decision can never drift. `import identify` is deliberately light (lazy `noai/__init__`, fits a 512 MB host) — keep heavy imports out (the `watermark_registry` constant import stays light: engines are lazy there). Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM` only when verified against a real C2PA file; editing-app/AI-device signer tokens go to `_SIGNER_C2PA_PLATFORM`; generator/issuer platforms to `C2PA_AI_VENDORS` in `constants.py`. The IPTC `digitalSourceType` **`algorithmicMedia`** (bare) is PROCEDURAL (an algorithm not trained on sampled data), NOT AI/ML generation, so it is deliberately absent from `IPTC_AI_MARKERS` — flagging it made `identify` assert AI + `has_invisible_target` True, scrubbing clean procedural content (it is a distinct token from `trainedAlgorithmicMedia`, so real "Made with AI" labels are unaffected; regression `test_metadata.py::...test_bare_algorithmic_media_not_flagged_ai`). Integrity-clash detection is high-precision by design (only hard generator stamps feed it, source-grouped independence). `_vendor_of` normalizes ByteDance/Canva/ElevenLabs/Black Forest Labs (as well as OpenAI/Google/... ) so their C2PA claims participate in the clash check; the generic **China TC260 AIGC label names no specific vendor**, so when a TC260-applying vendor (ByteDance, `_TC260_VENDORS`) is co-attributed the label is attributed to it (a legit Doubao image carrying its own TC260 label must NOT clash), while a NON-TC260 vendor next to a TC260 label still clashes as a laundering tell. The vendor normalization must not introduce clashes on clean compatibility samples. +- `identify.py` — separates file-backed metadata extraction (`extract_provenance_evidence`) from metadata-only verdict logic (`identify_from_evidence`); the compatible `identify` wrapper adds optional pixel-backed visible and invisible checks. External metadata ingestion (`evidence_from_metadata_record`) must reuse the same pure parser primitives in `metadata.py` as file-backed extraction; never duplicate signal regexes or field registries. Detection from `ProvenanceEvidence` must never reopen the source. All paths produce one `ProvenanceReport`; `is_ai_generated` is True or None, never asserted False. `ProvenanceReport.ai_source_kind` exposes the C2PA digital-source-type split — `"generated"` (trainedAlgorithmicMedia, fully AI) vs `"enhanced"` (compositeWithTrainedAlgorithmicMedia, a real photo with an AI-composited region), else None — so a caller branches full-frame scrub vs region-targeted clean (see `noai/tiling.feather_region_composite` + `WatermarkRemover.remove_watermark(region=...)`). The sparkle provenance threshold is the SHARED `watermark_registry.GEMINI_SPARKLE_TRUST_CONF` (imported, not a private copy) so the provenance "is there a sparkle" verdict and the removal "take the sparkle" decision can never drift. `import identify` is deliberately light (lazy `noai/__init__`, fits a 512 MB host) — keep heavy imports out (the `watermark_registry` constant import stays light: engines are lazy there). Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM` only when verified against a real C2PA file; editing-app/AI-device signer tokens go to `_SIGNER_C2PA_PLATFORM`; generator/issuer platforms to `C2PA_AI_VENDORS` in `constants.py`. The IPTC `digitalSourceType` **`algorithmicMedia`** (bare) is PROCEDURAL (an algorithm not trained on sampled data), NOT AI/ML generation, so it is deliberately absent from `IPTC_AI_MARKERS` — flagging it made `identify` assert AI + `has_invisible_target` True, scrubbing clean procedural content (it is a distinct token from `trainedAlgorithmicMedia`, so real "Made with AI" labels are unaffected; regression `test_metadata.py::...test_bare_algorithmic_media_not_flagged_ai`). Integrity-clash detection is high-precision by design (only hard generator stamps feed it, source-grouped independence). `_vendor_of` normalizes ByteDance/Canva/ElevenLabs/Black Forest Labs (as well as OpenAI/Google/... ) so their C2PA claims participate in the clash check; the generic **China TC260 AIGC label names no specific vendor**, so when a TC260-applying vendor (ByteDance, `_TC260_VENDORS`) is co-attributed the label is attributed to it (a legit Doubao image carrying its own TC260 label must NOT clash), while a NON-TC260 vendor next to a TC260 label still clashes as a laundering tell. The vendor normalization must not introduce clashes on clean compatibility samples. - `watermark_registry.py` — the single catalog of known visible watermarks (gemini / doubao / jimeng / qwen / kling / samsung / runninghub / baidu / liblib / jimeng_pill). **Removal is LOCALIZE -> FILL for every mark:** each mark is localized to a binary full-frame footprint mask (a `Localization`), then ONE shared, swappable fill inpaints that mask via `fill(image, mask, backend=...)` (delegates to `region_eraser.erase`). Reverse-alpha (the old `original = (wm - a*logo)/(1-a)` inversion of a captured alpha map + thin residual inpaint) is GONE for ALL marks; why it was dropped is recorded in `docs/module-internals.md`. Backends: `cv2` (classical inpaint, no deps, the floor), `migan` (MI-GAN ONNX, light, the memory-tight pick where LaMa will not fit), `lama` (big-LaMa ONNX, best quality, heavier, auto-preferred when a learned backend is available); `auto` = LaMa > MI-GAN > cv2, best available. The captured alpha maps (`scripts/visible_alpha_solve.py`) are still used to DETECT the marks and to shape the mask, but NOT for pixel recovery. **`--mark auto` removes EVERY detected mark in one pass** via `remove_auto_marks(image, *, sensitivity="auto", provenance=frozenset(), backend="auto")` (marks coexist -- a Jimeng-basic image has the top-left pill AND the bottom-right wordmark; a single-strongest pick would leave one). **Three orthogonal axes:** `backend` (the fill), `sensitivity` (how hard to trust a borderline mark: `auto`/`strict`, see the `Sensitivity` literal), and `provenance` (vendor keys metadata confirms -- the evidence that drives `auto`). **Perception / decision / action are separated:** `_build_candidates(image)` runs every detector at BOTH trust levels (strict + relaxed) and packages raw verdicts + features into `Candidate`s (no policy); the pure arbiter `decide(candidates, Context(sensitivity, provenance)) -> [Decision]` makes every keep/drop call (per-mark `resolve_trust` + the assumed-trust floor + the pill gate) with no image/IO, so it is unit-testable in isolation; then each winner is localized -> filled. Do NOT put policy back into the engines (the one exception, the Gemini FP gate, stays in `gemini_engine` because `identify` shares that confidence). `detect_marks(..., provenance=frozenset())` stays strict (identify verdict, precision over recall); `KnownMark.remove/detect/localize(..., provenance: bool)` take the already-resolved boolean. **How `auto` decides (this is metadata-INDEPENDENT for recall):** the visual detectors are pixel-based and need no metadata; the recall gain comes from RELAXING the false-positive gate, not from metadata. `strict` never relaxes (clean images untouched); `auto` relaxes a mark only on same-product evidence -- metadata provenance for that vendor OR a confidently detected sibling mark of the SAME product (`_PRODUCT_OF`; Doubao and Jimeng are both bottom-right ByteDance but distinct products, so they do NOT cross-relax). **`resolve_trust` resolves TWO levels:** `confirmed` bypasses the engine's false-positive gate, and only `confirmed` has evidence naming THAT vendor, which is exactly what the bypass is contracted to require (`GeminiEngine.detect_watermark`'s `trust_provenance` docstring: "external metadata already proves this is a Google generation"). **Historical, kept as the reason the third level is gone:** a removed `assumed` level let `assume_ai` bypass the gate on the bare assertion an image is AI. It produced unacceptable false positives on clean camera images and was removed. A wrong relaxation only fills a small corner near-losslessly (the localize -> fill benign failure mode), which is what made a SMALL false-fire rate arguable; it was never a licence for a 60% one. Metadata provenance mapping (feeds `auto`, read by `cli._visible_provenance`): Google/Gemini C2PA issuer -> gemini; China-AIGC (TC260) label -> doubao/jimeng; `samsung_genai` -> samsung. **The `jimeng_pill` is capture-less.** It uses a synthetic silhouette and a fixed top-left footprint. `_keep_pill` requires either a confirmed sibling wordmark or Jimeng provenance with a flat footprint. Do not loosen the flatness guard. **`assume_ai` was REMOVED (2026-07-19); `--sensitivity` is now `auto`/`strict` only.** It relaxed every mark's FP gate on the bare assertion an image is AI -- which names no vendor and no location, exactly what the bypass requires -- and had no place in the model (detector finds -> remove; finds nothing -> leave alone; user SEES a mark -> act on that). It took `_ASSUMED_CONF_FLOOR` / `assumed_floor_ok` / the `assumed` trust level with it, collapsing the ladder to `strict`/`confirmed`, and `_keep_pill` lost its `sensitivity` arg. Recall/precision on the unbiased sample are unchanged, so nothing on the default path moved. **Replacement advice is per mark:** `erase --region` is sound by construction; `--mark --no-detect` is reasonable (forced mask = the real glyph blob, non-empty 13/13); **`--mark gemini --no-detect` is NOT** -- it falls back to a fixed slot that covered the true sparkle on only **31% of 97** missed sparkles, so 69% fill a clean corner AND report a removal that did not happen. `cli._no_visible_mark_exit` follows that order and no longer suggests the removed mode. Migration raises loudly (`validate_sensitivity`, called from `api.remove_visible` and `Context.__post_init__`) because a `Literal` is unenforced at runtime and would silently downgrade a 0.15 caller to `auto`. **Detection and removal-mask extraction must use the same front-end.** The continuous `tophat` path can detect a mark whose binary glyph blob is empty, so its fallback mask uses the detector's own best-match box. Keep the textured regression fixture when changing this path. **Detector thresholds and geometry are calibrated per mark; do not port them between vendors without a fresh evaluation.** **New mark assets must be synthetic.** Font-render the mark, calibrate it on local evaluation inputs, and commit only the synthetic silhouette. Never derive a committed asset from source images. - `gemini_engine.py` — visible Gemini-sparkle detector + localizer (cv2/numpy, no GPU): top-K size-weighted fusion candidate selection (`_SELECT_TOPK`), corner-promote, false-positive gate (the provenance prior relaxes the gate + lowers the trust threshold when a Google/Gemini C2PA issuer confirms the vendor). **White-core rescue:** the FP gate demotes a low-gradient match (soft edges), but a real FAINT sparkle also has soft edges -- so the gate keeps a low-grad match that is a strong (conf ≥ `_SPARKLE_KEEP_CONF` 0.52), bright (margin), near-WHITE-core sparkle (`_core_saturation` ≤ `_SPARKLE_WHITE_SAT` 0.20): a real sparkle core is white, a clean bright corner that shape-matches (sky/sun) is colored. This recovers ~14/20 metadata-stripped faint sparkles under the DEFAULT strict/auto (no flag, no metadata) at ~1.25% clean false-fire (baseline 0.55%); the ~0.51-scoring bright-bg FPs stay demoted (below 0.52). A learned classifier on the SAME features was measured WORSE than the tuned gate (2026-07 tier-1: MLP 86.7% recall vs 90.8% at equal FP), so the heuristic stays; a patch-CNN with richer features is the only lever left (roadmapped P2, low expected value -- the wall is fundamental). Detection scores the top-K size-weighted matches by full fusion (spatial+gradient+variance) and keeps the highest — NOT the raw-NCC argmax, which re-admits the tiny-patch FPs the size weight suppresses (the osachub 2026-06-12 sub-0.85 corner-sparkle regression; see `docs/module-internals.md`). Keep the 0.85 corner-promote NCC gate; a margin/chroma-gated lower promote was measured and REJECTED 2026-06-11 (~33% FP on non-Google content). Removal is localize -> fill: `footprint_mask` returns the sparkle footprint (the captured alpha thresholded LOW so the faint halo is included, then dilated by a sparkle-relative margin), and the shared `watermark_registry.fill` inpaints it. The captured alpha maps are used only to detect and to shape the mask, not for pixel recovery. `detect_sparkle_confidence` reuses a process-wide `_shared_engine()` singleton (lru_cache) — the engine holds only constant assets (captures, alpha maps, a precomputed 16..118 template ladder) and takes the image as an arg, so do NOT reconstruct `GeminiEngine()` per call: that reloaded assets + recomputed alpha maps + rebuilt the template cache on every one of ~34k `identify` calls (−24% on the sparkle path once made a singleton, output byte-identical). `detect_watermark`/`footprint_mask` guard `image.size == 0` before `to_bgr`, and return an empty (detected=False) result when no template scale fits (short side < 16 px), rather than dereferencing an empty candidate list. - `_text_mark_engine.py` — shared base for the text-mark engines (extracted 2026-06-09); the per-engine modules are config-only subclasses. Detection still matches the glyph silhouette (NCC, keys on glyph shape). The removal mask is TEMPLATE-FREE: it is the bounding box of the top-hat glyph blob (`extract_mask`), filled solid + dilated, so the shared fill inpaints the whole wordmark rectangle. This drops the fixed alpha-template placement, so a re-rendered or differently-placed mark is still masked; the captured alpha maps are now used only for the detection silhouette, not for removal. New text mark = a `TextMarkConfig` + a thin subclass + one registry row. Gemini stays a separate engine (different model). The corner anchor is `corner` = `br`/`bl`/`tl`/`bc` (tl added 2026-07-22 for runninghub, bc for liblib's centered wordmark); `detect_frontend` is `binary`/`tophat`/`gray` (`gray` = raw-grayscale NCC for the faint mid-gray runninghub mark, added 2026-07-22; contrast-DEPENDENT, so its gates never port). The detection scale ladder is per-mark (`TextMarkConfig.ladder`, default `(0.8, 1.0, 1.25)` -- added 2026-07-21 for qwen's two size modes; the shared default is unchanged for every other mark, and densifying the SHARED ladder was measured and rejected, see `docs/verification-plan.md` B2). diff --git a/docs/module-internals.md b/docs/module-internals.md index ce02f2b..550c834 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -136,6 +136,8 @@ metadata extraction from verdict logic: - `extract_provenance_evidence` reads the supported metadata signals into `ProvenanceEvidence`. +- `evidence_from_metadata_record` normalizes an externally collected nested + metadata record into the same evidence type without file access. - `identify_from_evidence` evaluates that evidence without reopening the source. - `identify` preserves the path-based API and adds the optional registered visible-mark and open invisible-watermark decoders after extraction. diff --git a/docs/python-api.md b/docs/python-api.md index 758ee27..24ac64e 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -113,6 +113,28 @@ evidence = extract_provenance_evidence(Path("input.png")) report = identify_from_evidence(evidence) ``` +If metadata was collected by another component, normalize its nested record +without reopening the original file: + +```python +from remove_ai_watermarks.identify import ( + evidence_from_metadata_record, + identify_from_evidence, +) + +record = { + "pil": {"info:parameters": "Steps: 20, Sampler: Euler"}, + "exif": {"0th": {"Software": "Stable Diffusion"}}, +} +evidence = evidence_from_metadata_record(record, path=Path("input.png")) +report = identify_from_evidence(evidence) +``` + +The normalizer recursively preserves text and byte values. It also decodes +strings prefixed with `hex:` and fields named `base64` or ending in +`_base64`. Pass a C2PA manifest-store dictionary in `record["c2pa_store"]`, or +through the explicit `c2pa_manifest_store` argument. + `identify_from_evidence` does not reopen the source file. It evaluates metadata only; registered visible marks and pixel-backed invisible watermarks remain in the path-based `identify` call. diff --git a/docs/watermarking-landscape.md b/docs/watermarking-landscape.md index 49f7729..a7afa30 100644 --- a/docs/watermarking-landscape.md +++ b/docs/watermarking-landscape.md @@ -33,7 +33,7 @@ Grok JPEG downloads (Aurora model) carry **no C2PA, no XMP, no SynthID, no IPTC* **Stripped on removal too:** `remove_ai_metadata` calls `_scrub_ai_exif` on JPEG EXIF, which deletes the xAI Signature and UUID Artist pair plus supported AI generator values while retaining unrelated camera and editor EXIF. The -shared `_is_xai_signature_pair` helper is the single source of truth for the +shared `xai_signature_pair` helper is the single source of truth for the pair. On the ISOBMFF path, `blank_ai_exif_tokens` provides the corresponding in-place scrub for supported EXIF values, TC260 AIGC blocks, and the xAI pair. - **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). diff --git a/pyproject.toml b/pyproject.toml index 350dfcd..1103cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.21.0" +version = "0.21.1" description = "AI watermark remover: strip visible and invisible AI watermarks (Gemini / Nano Banana sparkle, SynthID) and provenance metadata (C2PA, EXIF) from images" readme = "README.md" requires-python = ">=3.10.1" diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index 0e20249..a22d32b 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -25,7 +25,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") _warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*") -__version__ = "0.21.0" +__version__ = "0.21.1" __all__ = ["__version__", "remove_visible", "visible_provenance"] diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index 3c1d200..1b83fcc 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -19,10 +19,12 @@ never as "clean". See CLAUDE.md "SynthID detection is metadata-only". from __future__ import annotations +import base64 +import contextlib import itertools import logging from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from remove_ai_watermarks.metadata import ( AI_METADATA_KEYS, @@ -30,17 +32,27 @@ from remove_ai_watermarks.metadata import ( IPTC_AI_FIELD_MARKERS, IPTC_AI_MARKERS, aigc_label, + aigc_label_from_metadata, c2pa_cloud_manifest_in, c2pa_marker_in, exif_generator, + generator_from_metadata, get_ai_metadata, huggingface_job, iptc_ai_system, + iptc_ai_system_in, samsung_genai, + samsung_genai_in, scan_head, xai_signature, + xai_signature_pair, +) +from remove_ai_watermarks.noai.c2pa import ( + c2pa_info_from_manifest_store, + cbor_text_after, + extract_c2pa_info, + soft_binding_vendors_in, ) -from remove_ai_watermarks.noai.c2pa import cbor_text_after, extract_c2pa_info, soft_binding_vendors_in from remove_ai_watermarks.noai.constants import ( C2PA_AI_TOOLS, C2PA_AI_VENDORS, @@ -51,7 +63,6 @@ from remove_ai_watermarks.watermark_registry import GEMINI_SPARKLE_TRUST_CONF if TYPE_CHECKING: from pathlib import Path - from typing import Any from numpy.typing import NDArray @@ -154,6 +165,182 @@ class ProvenanceEvidence: samsung_genai: int | None +def _external_metadata(value: Any) -> tuple[list[tuple[str, Any]], bytes]: + """Index nested metadata and recover common encoded binary values in one pass.""" + pairs: list[tuple[str, Any]] = [] + parts: list[bytes] = [] + + def visit(item: Any) -> None: + if isinstance(item, dict): + mapping = cast("dict[object, Any]", item) + for key, nested in mapping.items(): + key_text = str(key) + pairs.append((key_text, nested)) + parts.append(key_text.encode("utf-8", "replace")) + if isinstance(nested, str) and (key_text == "base64" or key_text.endswith("_base64")): + encoded = nested.split("...TRUNCATED", 1)[0] + with contextlib.suppress(ValueError, TypeError): + parts.append(base64.b64decode(encoded, validate=True)) + visit(nested) + elif isinstance(item, (list, tuple)): + sequence = cast("list[Any] | tuple[Any, ...]", item) + for nested in sequence: + visit(nested) + elif isinstance(item, bytes): + parts.append(item) + elif isinstance(item, str): + parts.append(item.encode("utf-8", "replace")) + if item.startswith("hex:"): + with contextlib.suppress(ValueError): + parts.append(bytes.fromhex(item[4:])) + elif item is not None: + parts.append(str(item).encode("utf-8", "replace")) + + visit(value) + return pairs, b"\n".join(parts) + + +def _external_text(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("latin-1", "replace").strip() + if not isinstance(value, str): + return str(value).strip() + if value.startswith("hex:"): + try: + return bytes.fromhex(value[4:]).decode("latin-1", "replace").strip() + except ValueError: + pass + return value.strip() + + +def _external_exif_generator(pairs: list[tuple[str, Any]], scan: bytes) -> str | None: + candidate_keys = { + "software", + "make", + "artist", + "imagedescription", + "source", + "title", + "description", + "creatortool", + } + candidates = [ + str(value) + for key, value in pairs + if key.lower().removeprefix("info:") in candidate_keys and isinstance(value, (str, bytes)) + ] + return generator_from_metadata(candidates, scan) + + +def evidence_from_metadata_record( + record: dict[str, Any], *, path: Path, c2pa_manifest_store: str | dict[str, Any] | None = None +) -> ProvenanceEvidence: + """Normalize an externally collected metadata record into provenance evidence. + + The record may contain arbitrary nested dictionaries and lists. Text, bytes, + hexadecimal values prefixed with ``hex:``, and fields named ``base64`` or + ending in ``_base64`` are included in the shared byte scan. No source file is + opened. + """ + pairs, scan = _external_metadata(record) + store = c2pa_manifest_store + if store is None: + candidate = record.get("c2pa_store") + store = ( + cast("dict[str, Any]", candidate) + if isinstance(candidate, dict) + else candidate + if isinstance(candidate, str) + else None + ) + c2pa_info = c2pa_info_from_manifest_store(store) if store is not None else {} + + ai_metadata: dict[str, str] = {} + pil_info = record.get("pil") + pil_pairs = cast("dict[str, Any]", pil_info).items() if isinstance(pil_info, dict) else () + for key, value in pil_pairs: + normalized_key = key.lower().removeprefix("info:") + if normalized_key not in AI_METADATA_KEYS or isinstance(value, (dict, list, tuple)): + continue + text = value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value) + ai_metadata.setdefault(normalized_key, text[:200] + ("…" if len(text) > 200 else "")) + for key, value in pairs: + if key != "text" or not isinstance(value, str) or "\x00" not in value: + continue + metadata_key, metadata_value = value.split("\x00", 1) + normalized_key = metadata_key.lower() + if normalized_key in AI_METADATA_KEYS: + ai_metadata.setdefault( + normalized_key, + metadata_value[:200] + ("…" if len(metadata_value) > 200 else ""), + ) + for key in ( + "c2pa_manifest", + "claim_generator", + "c2pa_spec", + "issuer", + "source_type", + "actions", + "synthid_watermark", + "soft_binding", + ): + if key in c2pa_info: + ai_metadata.setdefault(key, str(c2pa_info[key])) + + iptc_system = iptc_ai_system_in(scan) + + values_by_key: dict[str, str] = {} + for key, value in pairs: + if isinstance(value, (bytes, str)): + values_by_key.setdefault(key.lower(), _external_text(value)) + description = values_by_key.get("imagedescription", "") + artist = values_by_key.get("artist", "") + xai = xai_signature_pair(description, artist) + + hf_job = next( + ( + str(value).strip() + for key, value in pairs + if key.lower().removeprefix("info:") == "hf-job-id" and str(value).strip() + ), + None, + ) + samsung = samsung_genai_in(scan) + + aigc_candidates = tuple( + value for key, value in pairs if key.lower().removeprefix("info:") == "aigc" and isinstance(value, str) + ) + aigc = aigc_label_from_metadata(scan, aigc_candidates) + exif_gen = _external_exif_generator(pairs, scan) + if aigc is not None: + producer = aigc.get("ContentProducer", "") + ai_metadata.setdefault( + "aigc_label", + f"China AIGC label (TC260){f'; producer {producer}' if producer else ''}", + ) + if xai: + ai_metadata.setdefault("xai_signature", "xAI/Grok EXIF signature (Artist UUID + Signature blob)") + if iptc_system: + ai_metadata.setdefault("ai_system", f"IPTC 2025.1 AI disclosure ({iptc_system})") + if hf_job: + ai_metadata.setdefault("huggingface_job", f"HuggingFace-hosted job ({hf_job})") + if samsung is not None: + ai_metadata.setdefault("samsung_genai", f"Samsung Galaxy AI editing marker (genAIType={samsung})") + + return ProvenanceEvidence( + path=path, + c2pa_info=c2pa_info, + ai_metadata=ai_metadata, + scan=scan, + iptc_ai_system=iptc_system, + aigc_label=aigc, + exif_generator=exif_gen, + xai_signature=xai, + huggingface_job=hf_job, + samsung_genai=samsung, + ) + + @dataclass class ProvenanceReport: """Aggregated provenance verdict for one image.""" diff --git a/src/remove_ai_watermarks/metadata.py b/src/remove_ai_watermarks/metadata.py index 9de9459..33bd24b 100644 --- a/src/remove_ai_watermarks/metadata.py +++ b/src/remove_ai_watermarks/metadata.py @@ -356,6 +356,54 @@ def has_ai_metadata(image_path: Path) -> bool: return xai_signature(image_path) +def aigc_label_from_metadata(data: bytes, candidates: tuple[str, ...] = ()) -> dict[str, str] | None: + """Parse a China TC260 AI-labeling block from already collected metadata.""" + import html + import json + from typing import cast + + def _parse(text: str, *, require_tc260_field: bool) -> dict[str, str] | None: + try: + parsed = json.loads(text) + except ValueError: + return None + if not isinstance(parsed, dict): + return None + fields = {str(k): str(v) for k, v in cast("dict[object, object]", parsed).items()} + if require_tc260_field and not (_TC260_FIELDS & fields.keys()): + return None + return fields + + for candidate in candidates: + if result := _parse(candidate, require_tc260_field=True): + return result + + match = re.search( + rb'(.*?)|TC260:AIGC\s*=\s*"(.*?)"', + data, + re.DOTALL, + ) + if match: + body = match.group(1) if match.group(1) is not None else match.group(2) + return _parse(html.unescape(body.decode("utf-8", "replace")), require_tc260_field=False) + + text = data.decode("latin-1") + for needle in ('"AIGC"', "AIGC{"): + start = text.find(needle) + if start == -1: + continue + brace = text.find("{", start) + if brace == -1: + continue + try: + _, end = json.JSONDecoder().raw_decode(text, brace) + except ValueError: + continue + if result := _parse(text[brace:end], require_tc260_field=True): + return result + return None + + def aigc_label(image_path: Path) -> dict[str, str] | None: """Parse a China TC260 AI-labeling block, if present. @@ -378,24 +426,6 @@ def aigc_label(image_path: Path) -> dict[str, str] | None: if they carry at least one known TC260 field (``_TC260_FIELDS``); the namespaced XMP element is unambiguous, so any JSON object is accepted. """ - import html - import json - from typing import cast - - def _parse(text: str, *, require_tc260_field: bool) -> dict[str, str] | None: - try: - parsed = json.loads(text) - except ValueError: - return None - if not isinstance(parsed, dict): - return None - fields = {str(k): str(v) for k, v in cast("dict[object, object]", parsed).items()} - if require_tc260_field and not (_TC260_FIELDS & fields.keys()): - return None - return fields - - # PNG tEXt chunk keyed "AIGC" with raw JSON (Doubao and other China gens). - # The key is generic, so require a TC260 field to avoid a false positive. try: from PIL import Image @@ -404,50 +434,9 @@ def aigc_label(image_path: Path) -> dict[str, str] | None: except Exception as exc: logger.debug("PIL could not open %s for AIGC chunk scan: %s", image_path, exc) value = None - if isinstance(value, str) and (result := _parse(value, require_tc260_field=True)): - return result - - # XMP TC260:AIGC, namespaced (unambiguous) in either serialization RDF allows: - # an element {...} or an attribute TC260:AIGC="{...}" - # (the attribute form is what PicWish writes). Both are HTML-entity encoded. data = scan_head(image_path) - match = re.search( - rb'(.*?)|TC260:AIGC\s*=\s*"(.*?)"', - data, - re.DOTALL, - ) - if match: - body = match.group(1) if match.group(1) is not None else match.group(2) - return _parse(html.unescape(body.decode("utf-8", "replace")), require_tc260_field=False) - - # Generic raw-JSON forms the PNG-chunk and XMP paths above both miss, each - # gated on a TC260 field: the ``"AIGC":{...}`` key wrapper (as written into - # JPEG EXIF UserComment) and the bare ``AIGC{...}`` blob (the label glued - # straight to its JSON, no key wrapper, in a JPEG APP segment near the JFIF - # header). `raw_decode` brace-matches the inner object (respecting nested - # braces / quoted strings); `_parse` applies the same dict coercion + TC260 - # gate as the PNG-chunk path. A non-matching hit (no TC260 field, or an - # undecodable brace) must FALL THROUGH to the next form, never short-circuit: - # a quoted ``"AIGC"`` can appear later in an XMP packet while the real label - # is a bare ``AIGC{...}`` blob earlier in the file, so an unconditional return - # on the quoted form would shadow the bare form. - text = data.decode("latin-1") - for needle in ('"AIGC"', "AIGC{"): - start = text.find(needle) - if start == -1: - continue - # First brace at/after the needle: the object brace for ``"AIGC":{`` and - # the glued brace (at start+4) for the bare ``AIGC{`` -- one search covers both. - brace = text.find("{", start) - if brace == -1: - continue - try: - _, end = json.JSONDecoder().raw_decode(text, brace) - except ValueError: - continue - if result := _parse(text[brace:end], require_tc260_field=True): - return result - return None + candidates = (value,) if isinstance(value, str) else () + return aigc_label_from_metadata(data, candidates) # C2PA "Durable Content Credentials" manifest repositories (C2PA 2.4). When the @@ -541,6 +530,16 @@ def _read_file_tail(image_path: Path, size: int) -> bytes: return b"" +def samsung_genai_in(data: bytes) -> int | None: + """Return Samsung's non-zero ``genAIType`` from collected metadata bytes.""" + if _SAMSUNG_EDITOR_MARKER not in data: + return None + match = _SAMSUNG_GENAI_RE.search(data) + if match is None: + return None + return int(match.group(1)) or None + + def samsung_genai(image_path: Path) -> int | None: """Return Samsung's non-zero ``genAIType`` value if the image carries the Galaxy AI editing marker, else None. @@ -565,12 +564,17 @@ def samsung_genai(image_path: Path) -> int | None: oversize = False if oversize: data = _read_file_tail(image_path, _QUICK_SCAN_BYTES) - if _SAMSUNG_EDITOR_MARKER not in data: + return samsung_genai_in(data) + + +def iptc_ai_system_in(data: bytes) -> str | None: + """Return an IPTC 2025.1 AI-disclosure note from collected metadata bytes.""" + if not any(marker in data for marker in IPTC_AI_FIELD_MARKERS): return None - m = _SAMSUNG_GENAI_RE.search(data) - if m is None: - return None - return int(m.group(1)) or None + match = re.search(rb"AISystemUsed[=:\s]*[\"'>]\s*([^<\"']{1,120})", data) + if match and (value := match.group(1).decode("utf-8", "replace").strip()): + return value + return "fields present" def iptc_ai_system(image_path: Path) -> str | None: @@ -583,13 +587,7 @@ def iptc_ai_system(image_path: Path) -> str | None: extractable, otherwise the literal ``"fields present"``. Container-agnostic raw-byte scan; handles both XMP element and attribute serializations. """ - data = scan_head(image_path) - if not any(marker in data for marker in IPTC_AI_FIELD_MARKERS): - return None - match = re.search(rb"AISystemUsed[=:\s]*[\"'>]\s*([^<\"']{1,120})", data) - if match and (value := match.group(1).decode("utf-8", "replace").strip()): - return value - return "fields present" + return iptc_ai_system_in(scan_head(image_path)) def synthid_source(image_path: Path) -> str | None: @@ -632,6 +630,20 @@ def synthid_source(image_path: Path) -> str | None: return ", ".join(matched) if matched else None +def generator_from_metadata(candidates: list[str], scan: bytes = b"") -> str | None: + """Return a known AI generator from collected EXIF, PNG, or XMP values.""" + from remove_ai_watermarks.noai.constants import AI_GENERATOR_TOKENS + + candidates.extend( + match.group(1).decode("latin1", "replace") + for match in re.finditer(rb"CreatorTool[>\"'=\s]{1,4}([^<\"']{1,80})", scan) + ) + for value in candidates: + if any(token in value.lower() for token in AI_GENERATOR_TOKENS): + return value.strip() + return None + + def exif_generator(image_path: Path) -> str | None: """Return an AI-generator name from the EXIF ``Software`` / XMP ``CreatorTool`` field (or a PNG text chunk), if it matches a known generator (see @@ -644,10 +656,6 @@ def exif_generator(image_path: Path) -> str | None: chunks rather than EXIF. Only AI tokens match, so ordinary editors (plain "Adobe Photoshop", "GIMP") are not flagged. """ - import re - - from remove_ai_watermarks.noai.constants import AI_GENERATOR_TOKENS - candidates: list[str] = [] # EXIF Software / Artist / ImageDescription (0th IFD) via PIL exif bytes, @@ -682,18 +690,12 @@ def exif_generator(image_path: Path) -> str | None: except Exception as exc: # unopenable format / malformed EXIF logger.debug("EXIF generator read failed for %s: %s", image_path, exc) - # XMP CreatorTool: text, container-agnostic (covers HEIF/JXL via raw scan). try: head = scan_head(image_path) - for match in re.finditer(rb"CreatorTool[>\"'=\s]{1,4}([^<\"']{1,80})", head): - candidates.append(match.group(1).decode("latin1", "replace")) except Exception as exc: logger.debug("XMP CreatorTool scan failed for %s: %s", image_path, exc) - - for value in candidates: - if any(token in value.lower() for token in AI_GENERATOR_TOKENS): - return value.strip() - return None + head = b"" + return generator_from_metadata(candidates, head) # xAI / Grok EXIF signature scheme. A 64+ char base64 blob after "Signature:" @@ -703,7 +705,7 @@ _XAI_SIGNATURE_RE = re.compile(r"Signature:\s*[A-Za-z0-9+/=]{64,}") _UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE) -def _is_xai_signature_pair(description: str, artist: str) -> bool: +def xai_signature_pair(description: str, artist: str) -> bool: """True if an EXIF (ImageDescription, Artist) pair is xAI/Grok's scheme.""" return _XAI_SIGNATURE_RE.match(description) is not None and _UUID_RE.fullmatch(artist) is not None @@ -739,7 +741,7 @@ def xai_signature(image_path: Path) -> bool: logger.debug("xAI-signature EXIF read failed for %s: %s", image_path, exc) return False - return _is_xai_signature_pair( + return xai_signature_pair( _exif_text(tags, piexif.ImageIFD.ImageDescription), _exif_text(tags, piexif.ImageIFD.Artist) ) @@ -794,9 +796,7 @@ def _ai_exif_targets(loaded: dict[str, Any]) -> list[tuple[str, int, bytes, str] targets.append((ifd_key, tag, value, name)) # (a) xAI / Grok: the Signature blob and the UUID Artist go together. - if _is_xai_signature_pair( - _exif_text(ifd0, piexif.ImageIFD.ImageDescription), _exif_text(ifd0, piexif.ImageIFD.Artist) - ): + if xai_signature_pair(_exif_text(ifd0, piexif.ImageIFD.ImageDescription), _exif_text(ifd0, piexif.ImageIFD.Artist)): add("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription") add("0th", ifd0, piexif.ImageIFD.Artist, "Artist") # (b) known AI generator token in a 0th text tag. diff --git a/src/remove_ai_watermarks/noai/c2pa.py b/src/remove_ai_watermarks/noai/c2pa.py index 310318c..6005476 100644 --- a/src/remove_ai_watermarks/noai/c2pa.py +++ b/src/remove_ai_watermarks/noai/c2pa.py @@ -189,9 +189,8 @@ def _active_manifest(store: dict[str, Any]) -> dict[str, Any]: return cast("dict[str, Any]", active) if isinstance(active, dict) else {} -def _info_from_store_json(store_json: str) -> dict[str, Any]: - """Build the C2PA info dict from a c2pa-python manifest-store JSON string.""" - store_bytes = store_json.encode("utf-8") +def _info_from_store(store: dict[str, Any], store_bytes: bytes) -> dict[str, Any]: + """Build normalized C2PA info from one parsed manifest store.""" c2pa_info: dict[str, Any] = { "has_c2pa": True, "type": "C2PA (Coalition for Content Provenance and Authenticity)", @@ -202,13 +201,6 @@ def _info_from_store_json(store_json: str) -> dict[str, Any]: # registry scan that runs on the raw caBX chunk applies unchanged here. _populate_registry_fields(store_bytes, c2pa_info) - try: - parsed: Any = json.loads(store_json) - except (ValueError, TypeError): - return c2pa_info - if not isinstance(parsed, dict): - return c2pa_info - store = cast("dict[str, Any]", parsed) if generator := _claim_generator_from_store(store): c2pa_info["claim_generator"] = generator sig: Any = _active_manifest(store).get("signature_info") @@ -217,6 +209,44 @@ def _info_from_store_json(store_json: str) -> dict[str, Any]: return c2pa_info +def _info_from_store_json(store_json: str) -> dict[str, Any]: + """Build the C2PA info dict from a c2pa-python manifest-store JSON string.""" + store_bytes = store_json.encode("utf-8") + try: + parsed: Any = json.loads(store_json) + except (ValueError, TypeError): + parsed = {} + store = cast("dict[str, Any]", parsed) if isinstance(parsed, dict) else {} + return _info_from_store(store, store_bytes) + + +def c2pa_info_from_manifest_store(store: str | dict[str, Any]) -> dict[str, Any]: + """Build normalized C2PA evidence from an externally collected manifest store. + + ``store`` may be the JSON string returned by ``c2pa.Reader.json()`` or its + decoded dictionary form. This is the non-file-backed counterpart to + :func:`extract_c2pa_info`. + """ + if isinstance(store, dict): + parsed = store + try: + store_json = json.dumps(store, ensure_ascii=False) + except (TypeError, ValueError): + return {} + else: + store_json = store + try: + decoded: Any = json.loads(store_json) + except (TypeError, ValueError): + return {} + if not isinstance(decoded, dict): + return {} + parsed = cast("dict[str, Any]", decoded) + if not store_json or not parsed or parsed.get("error"): + return {} + return _info_from_store(parsed, store_json.encode("utf-8")) + + def extract_c2pa_info(image_path: Path) -> dict[str, Any]: """ Extract C2PA metadata information from an image. diff --git a/tests/test_identify.py b/tests/test_identify.py index caaaca8..9e7f3d8 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -23,6 +23,7 @@ from remove_ai_watermarks.identify import ( _integrity_clashes, _issuers_in, _vendor_of, + evidence_from_metadata_record, extract_provenance_evidence, has_invisible_target, identify, @@ -37,6 +38,29 @@ SAMPLES_DIR = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "pr class TestProvenanceEvidence: + def test_external_metadata_record_builds_equivalent_evidence(self, tmp_path: Path): + path = tmp_path / "external.jpg" + signature = "A" * 64 + artist = "c8045292-06d2-4c7d-b4f0-4f93b94e4801" + record = { + "pil": {"info:parameters": "Steps: 20, Sampler: Euler"}, + "exif": { + "0th": { + "ImageDescription": f"Signature: {signature}", + "Artist": artist, + } + }, + } + + evidence = evidence_from_metadata_record(record, path=path) + report = identify_from_evidence(evidence) + + assert evidence.path == path + assert evidence.ai_metadata["parameters"] == "Steps: 20, Sampler: Euler" + assert evidence.xai_signature is True + assert report.is_ai_generated is True + assert {signal.name for signal in report.signals} >= {"gen_params", "xai_signature"} + @pytest.mark.parametrize( "filename", [ diff --git a/uv.lock b/uv.lock index edf9f42..9d600cf 100644 --- a/uv.lock +++ b/uv.lock @@ -660,7 +660,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -695,43 +695,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufft = [ - { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufile = [ - { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] curand = [ - { name = "nvidia-curand", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] [[package]] @@ -842,8 +842,8 @@ name = "email-validator" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dnspython", marker = "python_full_version >= '3.12'" }, - { name = "idna", marker = "python_full_version >= '3.12'" }, + { name = "dnspython" }, + { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } wheels = [ @@ -855,7 +855,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1192,8 +1192,8 @@ name = "inflect" version = "7.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "python_full_version >= '3.12'" }, - { name = "typeguard", marker = "python_full_version >= '3.12'" }, + { name = "more-itertools" }, + { name = "typeguard" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" } wheels = [ @@ -1674,7 +1674,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cuda-nvrtc" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -1713,7 +1713,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -1725,7 +1725,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -1755,9 +1755,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -1769,7 +1769,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -1844,11 +1844,11 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "flatbuffers", marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "protobuf", marker = "python_full_version < '3.11'" }, - { name = "sympy", marker = "python_full_version < '3.11'" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/15/41/3253db975a90c3ce1d475e2a230773a21cd7998537f0657947df6fb79861/onnxruntime-1.24.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3e6456801c66b095c5cd68e690ca25db970ea5202bd0c5b84a2c3ef7731c5a3c", size = 17332766, upload-time = "2026-03-05T17:18:59.714Z" }, @@ -1899,10 +1899,10 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "flatbuffers", marker = "python_full_version >= '3.11'" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "protobuf", marker = "python_full_version >= '3.11'" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d4/e4/5353d7e09ced4a8f473f843223fc75d726b2b5519dcefc12f22a6c92852d/onnxruntime-1.27.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8ba14a38c570087f3cdb8cfba33f7a38a1e826c1e5b29e17c28ceda0cc910016", size = 18416484, upload-time = "2026-06-15T22:43:43.894Z" }, @@ -2070,10 +2070,10 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "pytz", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "tzdata", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2143,9 +2143,9 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32')" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } wheels = [ @@ -2624,10 +2624,10 @@ name = "pydantic" version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-types", marker = "python_full_version >= '3.12'" }, - { name = "pydantic-core", marker = "python_full_version >= '3.12'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, - { name = "typing-inspection", marker = "python_full_version >= '3.12'" }, + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ @@ -2636,7 +2636,7 @@ wheels = [ [package.optional-dependencies] email = [ - { name = "email-validator", marker = "python_full_version >= '3.12'" }, + { name = "email-validator" }, ] [[package]] @@ -2644,7 +2644,7 @@ name = "pydantic-core" version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ @@ -2881,7 +2881,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/45/bfaaab38545a33a9f06c61211fc3bea2e23e8a8e00fedeb8e57feda722ff/pywavelets-1.8.0.tar.gz", hash = "sha256:f3800245754840adc143cbc29534a1b8fc4b8cff6e9d403326bd52b7bb5c35aa", size = 3935274, upload-time = "2024-12-04T19:54:20.593Z" } wheels = [ @@ -2946,7 +2946,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/75/50581633d199812205ea8cdd0f6d52f12a624886b74bf1486335b67f01ff/pywavelets-1.9.0.tar.gz", hash = "sha256:148d12203377772bea452a59211d98649c8ee4a05eff019a9021853a36babdc8", size = 3938340, upload-time = "2025-08-04T16:20:04.978Z" } wheels = [ @@ -3187,7 +3187,7 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.21.0" +version = "0.21.1" source = { editable = "." } dependencies = [ { name = "c2pa-python" }, @@ -3490,7 +3490,7 @@ name = "stamina" version = "26.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tenacity", marker = "python_full_version >= '3.12'" }, + { name = "tenacity" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/bd/b2f71ae14368a066f103d182f25bbc6c3bf4aa695889f3ed3cba026d6f36/stamina-26.1.0.tar.gz", hash = "sha256:0214d05fdf5102c518194a4aac7520ce53cf660550ae3b940701aad88cf50c17", size = 568171, upload-time = "2026-04-13T17:44:31.012Z" } wheels = [ @@ -3790,7 +3790,7 @@ name = "typeguard" version = "4.5.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/1c/dfba5c4633cafc4c701f237d2ba63b416805047fd6d96aab4cfc40969f98/typeguard-4.5.2.tar.gz", hash = "sha256:5a16dcac23502039299c97c8941651bc33d7ea8cc4b2f7d6bbb1b528f6eea423", size = 80240, upload-time = "2026-05-14T12:59:40.857Z" } wheels = [ @@ -3826,7 +3826,7 @@ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ @@ -3856,10 +3856,10 @@ name = "uv-outdated" version = "1.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "rich", marker = "python_full_version >= '3.12'" }, - { name = "typer", marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "rich" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/84/78736b81c0e6ebefd3810b04a3bc6cb82bf7ea63474821b02d5cd9040439/uv_outdated-1.0.4.tar.gz", hash = "sha256:126745028823d8d452a82faaf53ea1d4ab5cdea7bba3159fc2ce7e5d0443146c", size = 19176, upload-time = "2025-12-25T10:54:22.77Z" } wheels = [ @@ -3871,18 +3871,18 @@ name = "uv-secure" version = "0.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "python_full_version >= '3.12'" }, - { name = "cvss", marker = "python_full_version >= '3.12'" }, - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "humanize", marker = "python_full_version >= '3.12'" }, - { name = "inflect", marker = "python_full_version >= '3.12'" }, - { name = "orjson", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", extra = ["email"], marker = "python_full_version >= '3.12'" }, - { name = "rich", marker = "python_full_version >= '3.12'" }, - { name = "stamina", marker = "python_full_version >= '3.12'" }, - { name = "tomlkit", marker = "python_full_version >= '3.12'" }, - { name = "typer", marker = "python_full_version >= '3.12'" }, + { name = "anyio" }, + { name = "cvss" }, + { name = "httpx" }, + { name = "humanize" }, + { name = "inflect" }, + { name = "orjson" }, + { name = "packaging" }, + { name = "pydantic", extra = ["email"] }, + { name = "rich" }, + { name = "stamina" }, + { name = "tomlkit" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/99/29318cedfc5583cf2d503f0eedb9c4e96829541c356ce5d2aacfe09ef67f/uv_secure-0.17.2.tar.gz", hash = "sha256:e394939e0872df392d8f650d15ac1571b9267fc2f3671a183aa73c0977f0f402", size = 47240, upload-time = "2026-04-18T08:45:38.185Z" } wheels = [ From 37f3b3b1c94d68eae71e4a018cc2d954055082ef Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:34:11 +0000 Subject: [PATCH 04/12] Sync conda recipe with v0.21.1 --- packaging/conda/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index 5d098d8..9349754 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -1,7 +1,7 @@ schema_version: 1 context: - version: "0.21.0" + version: "0.21.1" python_min: "3.10" package: @@ -10,7 +10,7 @@ package: source: url: https://pypi.org/packages/source/r/remove-ai-watermarks/remove_ai_watermarks-${{ version }}.tar.gz - sha256: 69eaa8cbb25eec113916358bad4baf4d067cb41854f92d389d2b5c0f22f94f94 + sha256: a233074030ef189f4339228601281badbd8ef2f52d5bb36dc78c17c4691b7d2d build: noarch: python From 7c9bec0ebaa15eb62e10d1591e7511d183693d25 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Wed, 29 Jul 2026 16:46:42 -0700 Subject: [PATCH 05/12] Ignore collector diagnostics in metadata evidence --- docs/module-internals.md | 4 ++- docs/python-api.md | 6 ++-- pyproject.toml | 2 +- src/remove_ai_watermarks/__init__.py | 2 +- src/remove_ai_watermarks/identify.py | 3 ++ tests/test_identify.py | 43 ++++++++++++++++++++++++++++ uv.lock | 2 +- 7 files changed, 56 insertions(+), 6 deletions(-) diff --git a/docs/module-internals.md b/docs/module-internals.md index 550c834..9750e29 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -137,7 +137,9 @@ metadata extraction from verdict logic: - `extract_provenance_evidence` reads the supported metadata signals into `ProvenanceEvidence`. - `evidence_from_metadata_record` normalizes an externally collected nested - metadata record into the same evidence type without file access. + metadata record into the same evidence type without file access. Diagnostic + values under `error` and `kind` are excluded from evidence while nested raw + bytes remain available through encoded binary fields. - `identify_from_evidence` evaluates that evidence without reopening the source. - `identify` preserves the path-based API and adds the optional registered visible-mark and open invisible-watermark decoders after extraction. diff --git a/docs/python-api.md b/docs/python-api.md index 24ac64e..5cf04fc 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -132,8 +132,10 @@ report = identify_from_evidence(evidence) The normalizer recursively preserves text and byte values. It also decodes strings prefixed with `hex:` and fields named `base64` or ending in -`_base64`. Pass a C2PA manifest-store dictionary in `record["c2pa_store"]`, or -through the explicit `c2pa_manifest_store` argument. +`_base64`. Diagnostic values under `error` and `kind` are ignored because they +describe the collector rather than the source file. Pass a C2PA manifest-store +dictionary in `record["c2pa_store"]`, or through the explicit +`c2pa_manifest_store` argument. `identify_from_evidence` does not reopen the source file. It evaluates metadata only; registered visible marks and pixel-backed invisible watermarks remain in diff --git a/pyproject.toml b/pyproject.toml index 1103cfd..8242281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.21.1" +version = "0.21.2" description = "AI watermark remover: strip visible and invisible AI watermarks (Gemini / Nano Banana sparkle, SynthID) and provenance metadata (C2PA, EXIF) from images" readme = "README.md" requires-python = ">=3.10.1" diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index a22d32b..a96a921 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -25,7 +25,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") _warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*") -__version__ = "0.21.1" +__version__ = "0.21.2" __all__ = ["__version__", "remove_visible", "visible_provenance"] diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index 1b83fcc..cfb4fdb 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -169,6 +169,7 @@ def _external_metadata(value: Any) -> tuple[list[tuple[str, Any]], bytes]: """Index nested metadata and recover common encoded binary values in one pass.""" pairs: list[tuple[str, Any]] = [] parts: list[bytes] = [] + diagnostic_keys = {"error", "kind"} def visit(item: Any) -> None: if isinstance(item, dict): @@ -177,6 +178,8 @@ def _external_metadata(value: Any) -> tuple[list[tuple[str, Any]], bytes]: key_text = str(key) pairs.append((key_text, nested)) parts.append(key_text.encode("utf-8", "replace")) + if key_text.lower() in diagnostic_keys: + continue if isinstance(nested, str) and (key_text == "base64" or key_text.endswith("_base64")): encoded = nested.split("...TRUNCATED", 1)[0] with contextlib.suppress(ValueError, TypeError): diff --git a/tests/test_identify.py b/tests/test_identify.py index 9e7f3d8..c683d21 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -6,6 +6,7 @@ against the real committed C2PA / IPTC fixtures in data/fixtures/provenance/. from __future__ import annotations +import base64 import json import subprocess import sys @@ -61,6 +62,48 @@ class TestProvenanceEvidence: assert report.is_ai_generated is True assert {signal.name for signal in report.signals} >= {"gen_params", "xai_signature"} + def test_external_scanner_diagnostics_do_not_create_c2pa_evidence(self, tmp_path: Path): + path = tmp_path / "plain.jpg" + record = { + "c2pa_store": {"error": "ManifestNotFound: no JUMBF data found"}, + "jpeg": { + "segments": [ + { + "marker": "APP11", + "kind": "c2pa_or_jumbf", + "base64": "AAA=", + } + ] + }, + } + + report = identify_from_evidence(evidence_from_metadata_record(record, path=path)) + + assert report.is_ai_generated is None + assert report.signals == [] + assert report.watermarks == [] + + def test_external_scanner_raw_bytes_still_create_c2pa_evidence(self, tmp_path: Path): + path = tmp_path / "signed.jpg" + manifest = b"jumb c2pa OpenAI trainedAlgorithmicMedia" + record = { + "jpeg": { + "segments": [ + { + "marker": "APP11", + "kind": "c2pa_or_jumbf", + "base64": base64.b64encode(manifest).decode(), + } + ] + } + } + + report = identify_from_evidence(evidence_from_metadata_record(record, path=path)) + + assert report.is_ai_generated is True + assert report.platform == "OpenAI (ChatGPT / gpt-image / DALL-E / Sora)" + assert [signal.name for signal in report.signals] == ["c2pa"] + @pytest.mark.parametrize( "filename", [ diff --git a/uv.lock b/uv.lock index 9d600cf..8366acd 100644 --- a/uv.lock +++ b/uv.lock @@ -3187,7 +3187,7 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.21.1" +version = "0.21.2" source = { editable = "." } dependencies = [ { name = "c2pa-python" }, From a67b209d669a60d253c758773c598fc04cb9ad90 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:48:36 +0000 Subject: [PATCH 06/12] Sync conda recipe with v0.21.2 --- packaging/conda/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index 9349754..f68c5f9 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -1,7 +1,7 @@ schema_version: 1 context: - version: "0.21.1" + version: "0.21.2" python_min: "3.10" package: @@ -10,7 +10,7 @@ package: source: url: https://pypi.org/packages/source/r/remove-ai-watermarks/remove_ai_watermarks-${{ version }}.tar.gz - sha256: a233074030ef189f4339228601281badbd8ef2f52d5bb36dc78c17c4691b7d2d + sha256: a013af322d40c137302c500638806c8864d5945699bdc2cdcbdcda3e2f37b3ac build: noarch: python From 3d941069e501d8795601a5970e1a16b41f81a6d8 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Thu, 30 Jul 2026 16:21:50 -0700 Subject: [PATCH 07/12] Keep Dependabot updates within compatible dependency lines --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 402d72c..73bf8b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,21 @@ updates: update-types: - "minor" - "patch" + # The `all` extra must remain jointly resolvable. These candidates violate + # upstream constraints: TrustMark still requires NumPy 1.x, OpenCV 5 requires + # NumPy 2 on newer Python versions, and this project keeps tokenizers below + # 0.23 for the stable Transformers path. Keep Dependabot on the compatible + # lines until those constraints move. + ignore: + - dependency-name: "numpy" + versions: + - ">=2" + - dependency-name: "opencv-python-headless" + versions: + - ">=5" + - dependency-name: "tokenizers" + versions: + - ">=0.23" - package-ecosystem: "github-actions" directory: "/" From 0c4b26bb77c850518e85dea67774236693f8816a Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Thu, 30 Jul 2026 16:29:29 -0700 Subject: [PATCH 08/12] Fix OpenCV Dependabot compatibility bound --- .github/dependabot.yml | 10 +++++----- tests/test_dependabot.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 tests/test_dependabot.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73bf8b8..49ce021 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,17 +15,17 @@ updates: - "minor" - "patch" # The `all` extra must remain jointly resolvable. These candidates violate - # upstream constraints: TrustMark still requires NumPy 1.x, OpenCV 5 requires - # NumPy 2 on newer Python versions, and this project keeps tokenizers below - # 0.23 for the stable Transformers path. Keep Dependabot on the compatible - # lines until those constraints move. + # upstream constraints: TrustMark still requires NumPy 1.x, OpenCV 4.13+ + # requires NumPy 2 on newer Python versions, and this project keeps tokenizers + # below 0.23 for the stable Transformers path. Keep Dependabot on the + # compatible lines until those constraints move. ignore: - dependency-name: "numpy" versions: - ">=2" - dependency-name: "opencv-python-headless" versions: - - ">=5" + - ">=4.13" - dependency-name: "tokenizers" versions: - ">=0.23" diff --git a/tests/test_dependabot.py b/tests/test_dependabot.py new file mode 100644 index 0000000..3524f03 --- /dev/null +++ b/tests/test_dependabot.py @@ -0,0 +1,11 @@ +"""Regression tests for Dependabot compatibility constraints.""" + +from pathlib import Path + + +def test_dependabot_blocks_opencv_releases_that_require_numpy_2() -> None: + config = Path(".github/dependabot.yml").read_text() + + opencv_ignore = config.split('dependency-name: "opencv-python-headless"', maxsplit=1)[1] + assert '"<4.13"' not in opencv_ignore + assert '- ">=4.13"' in opencv_ignore From fc177bacde107de172d6ef320bf3c48dc0a1761a Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Thu, 30 Jul 2026 18:18:49 -0700 Subject: [PATCH 09/12] Correct OpenCV Dependabot compatibility cutoff --- .github/dependabot.yml | 4 ++-- tests/test_dependabot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 49ce021..3eac6d1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,7 +15,7 @@ updates: - "minor" - "patch" # The `all` extra must remain jointly resolvable. These candidates violate - # upstream constraints: TrustMark still requires NumPy 1.x, OpenCV 4.13+ + # upstream constraints: TrustMark still requires NumPy 1.x, OpenCV 4.12+ # requires NumPy 2 on newer Python versions, and this project keeps tokenizers # below 0.23 for the stable Transformers path. Keep Dependabot on the # compatible lines until those constraints move. @@ -25,7 +25,7 @@ updates: - ">=2" - dependency-name: "opencv-python-headless" versions: - - ">=4.13" + - ">=4.12" - dependency-name: "tokenizers" versions: - ">=0.23" diff --git a/tests/test_dependabot.py b/tests/test_dependabot.py index 3524f03..041da9a 100644 --- a/tests/test_dependabot.py +++ b/tests/test_dependabot.py @@ -7,5 +7,5 @@ def test_dependabot_blocks_opencv_releases_that_require_numpy_2() -> None: config = Path(".github/dependabot.yml").read_text() opencv_ignore = config.split('dependency-name: "opencv-python-headless"', maxsplit=1)[1] - assert '"<4.13"' not in opencv_ignore - assert '- ">=4.13"' in opencv_ignore + assert '"<4.12"' not in opencv_ignore + assert '- ">=4.12"' in opencv_ignore From 9c9e81c75604193d3e63811755d1cdb92664e8b4 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Thu, 30 Jul 2026 18:32:25 -0700 Subject: [PATCH 10/12] Compact CLAUDE.md and route development guidance --- .claude/rules/development.md | 31 ++++++++++ CLAUDE.md | 114 ++++++++++------------------------- docs/development.md | 35 +++++++++++ 3 files changed, 98 insertions(+), 82 deletions(-) create mode 100644 .claude/rules/development.md create mode 100644 docs/development.md diff --git a/.claude/rules/development.md b/.claude/rules/development.md new file mode 100644 index 0000000..ef957ee --- /dev/null +++ b/.claude/rules/development.md @@ -0,0 +1,31 @@ +--- +globs: ["src/**/*.py", "tests/**/*.py", "scripts/**/*.py", "pyproject.toml", "uv.lock", "maintain.sh", ".github/workflows/*.yml"] +description: Command contracts, project gate, typing boundaries, and model-adjacent test invariants. +--- + +# Development invariants + +## Command contracts + +Every single-image command declares `source` with `dir_okay=False`; `batch` declares its directory with `file_okay=False`. Keep `tests/test_cli_robustness.py::TestDirectoryInputIsRejected` as the regression guard. + +Exit-code and no-signal behavior is a public contract. Read the command-line section of [`../../docs/module-internals.md`](../../docs/module-internals.md) before changing it. + +## Local gate + +Run `bash maintain.sh` from the repository root. The authoritative type gate is scoped to `src/`; full-project Pyright can exhaust Node memory on the ML dependency graph. + +Boundary modules for cv2, Torch, and Diffusers may carry narrow per-file relaxations for unknown third-party types. Keep pure-logic files strict, preserve the local piexif stub, and fix real errors before widening a pragma. + +## Model-adjacent tests + +Do not classify an entire module as untestable because its main path downloads a model. Keep pure behavior covered without downloads, including: + +- target-size selection in `test_invisible_engine.py`; +- unsharp and adaptive-polish helpers in `test_humanizer.py`; +- mocked device fallback in `test_img2img_runner.py`; +- tiling geometry and blending in `test_tiling.py`. + +Use availability checks only for paths that actually load large models. + +Environment setup, dependency recovery, CI behavior, and fixture policy: [`../../docs/development.md`](../../docs/development.md). diff --git a/CLAUDE.md b/CLAUDE.md index 84c177a..b76f024 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,104 +1,54 @@ -# Remove-AI-Watermarks +# Remove AI Watermarks -You are a **principal Python engineer** maintaining a CLI tool and library for removing visible and invisible AI watermarks from images. +You are a **principal Python engineer** maintaining a CLI tool and library for removing visible and invisible AI provenance watermarks. ## Scope and non-goals -The mission is removing **AI-provenance watermarks** that a platform stamps onto content the user generated themselves — SynthID, the Gemini / Nano Banana sparkle, the Doubao / Jimeng / Qwen / Kling / Tencent Yuanbao / Samsung visible AI labels, the Chinese TC260 "由…AI生成" label, and C2PA / IPTC / EXIF "Made with AI" metadata. The point is user autonomy over their own generated output. +The project gives users control over provenance marks on content they generated or edited themselves. It does not automatically remove stock-agency, marketplace, classifieds, tiled-preview, or other marks that protect a third party's paid or copyrighted asset. -It deliberately does **not** remove watermarks that protect someone else's paid or copyrighted content — stock-agency overlays (Shutterstock, Getty, iStock, Adobe Stock), classifieds-site marks, or any tiled / diagonal "preview" watermark whose job is to gate a purchase. Stripping those makes a paid resource free off someone else's work; out of scope **by principle, not by technical difficulty**. The line: a visible mark is in scope when it labels the user's **own** AI generation, and out of scope when it protects a **third party's paid asset**. +- Add visible templates only for AI-generation labels. +- Do not add stock, agency, or classifieds marks to `watermark_registry.py`. +- Keep `erase --region` generic and user-directed; do not build an automatic stock-watermark remover on it. -Consequences for contributors (do not drift back into the stock niche just because it is technically feasible): -- Do not add stock / agency / classifieds watermark removal to `watermark_registry.py` or the eraser, and do not build tiled-overlay or multi-image watermark-estimation features aimed at them. -- `erase --region` stays a generic **user-driven** tool (the user points at their own object); do not ship an *automatic* stock-watermark detector/remover on top of it. -- New visible-mark templates are for **AI-generation labels only**. - -(Established 2026-06-13 by user instruction: "Я пытаюсь сделать платные ресурсы бесплатными — это не то, против чего мы боремся.") +Full boundary and legal context: [`docs/legal-and-safety.md`](docs/legal-and-safety.md). ## How to run -Per-command exit-code semantics (the no-signal / GPU-missing skip branches), test traps, and regression-guard paths live in `docs/module-internals.md` (section "CLI commands (`cli.py`)") — read it before changing any command's skip/exit behavior. Every single-image command's `source` argument declares `dir_okay=False`: without it `click.Path(exists=True)` accepts a directory, which then reached `open()` and raised `IsADirectoryError` (Tier E, 2026-07-20; `batch`'s `directory` already declared `file_okay=False`). Regression: `tests/test_cli_robustness.py::TestDirectoryInputIsRejected`. +```bash +uv run remove-ai-watermarks --help +bash maintain.sh +``` -- `uv run remove-ai-watermarks all -o ` — full pipeline (visible + invisible + metadata). Same diffusion knobs as `invisible`, plus the visible-pass `--backend auto|cv2|migan|lama` (default `auto`) and `--sensitivity auto|strict` (default `auto`) for the localize -> fill visible removal (see the `visible` bullet). Skips step 2 (invisible/SynthID) when the `[gpu]` extra is absent or no invisible signal is detectable; see the module doc for the distinct exit codes. -- `uv run remove-ai-watermarks invisible -o ` — diffusion SynthID removal. **Full knob set** (kept identical across `invisible`/`all`/`batch`): `--strength` (vendor-adaptive default except resolution-adaptive `qwen-zimage`), `--steps` (**interacts with `--strength`** on the diffusers profiles; `watermark_profiles.viable_steps` prevents zero effective steps. `qwen-zimage` instead fixes its Lightning stage at 4 steps), `--guidance-scale`, `--pipeline sdxl|controlnet|qwen|qwen-zimage` (default `controlnet`; `qwen` and `qwen-zimage` are manual opt-ins), `--controlnet-scale`, `--model`, `--device`, `--seed`, `--hf-token`, `--max-resolution`/`--min-resolution`, `--upscaler lanczos|esrgan`, `--humanize`, `--unsharp`, `--adaptive-polish/--no-adaptive-polish`, `--tile/--no-tile` + `--tile-size`/`--tile-overlap`, `--cpu-offload/--no-cpu-offload`, `--force/--no-force`. `--cpu-offload` trades speed for lower CUDA VRAM use by moving Diffusers model components between CPU and GPU; on `qwen-zimage` it forces the face stack to offload instead of using automatic residency. It has no effect on CPU/MPS. ControlNet is the compatibility and cost default, not the highest-fidelity mode. Recommend the CUDA-only `qwen-zimage` profile when output quality, especially face identity, matters more than runtime and cost; it needs the separate extra, uses a fixed Qwen-Image-2512 + Z-Image stack, rejects `--model`, defaults to the oracle-candidate seed 0, and supports tiling only for its global Qwen pass. The full-frame face stage runs once after tile blending. Tiled outputs still need separate oracle certification. `--auto` is deprecated and a no-op that only warns. Skips the diffusion when no invisible signal is detectable; see the module doc. -- `uv run remove-ai-watermarks visible -o ` — known-visible-mark removal by **localize -> fill**: each detected mark is localized to a binary full-frame footprint mask, then one shared, swappable fill inpaints that mask. `--backend auto|cv2|migan|lama` (default `auto`) picks the fill: `cv2` (classical inpaint, no deps, the floor), `migan` (MI-GAN ONNX, light, the memory-tight pick where LaMa will not fit), `lama` (big-LaMa ONNX, best quality, heavier, auto-preferred when a learned backend is available); `auto` = LaMa > MI-GAN > cv2, best available. `--mark auto` (default) removes EVERY detected mark in one pass (a Jimeng-basic image carries the top-left "AI生成" pill AND the bottom-right "★ 即梦AI" wordmark) from: Gemini sparkle, Doubao "豆包AI生成", Jimeng "★ 即梦AI", Qwen "千问AI生成", Kling "可灵AI 3.0", Tencent Yuanbao "元宝 / AI生成", Samsung Galaxy AI "✦ Contenuti generati dall'AI", Baidu "百度 AI生成", LibLibAI wordmark (bottom-center), RunningHub "RunningHub AI生成" (top-left), and the capture-less Jimeng "AI生成" pill (top-left, metadata-gated); `--mark gemini|doubao|jimeng|qwen|kling|yuanbao|samsung|baidu|liblib|runninghub|jimeng_pill` forces one. `--sensitivity auto|strict` (default `auto`) sets how hard a borderline mark is trusted: `auto` relaxes a mark's gate only on same-product evidence (metadata provenance for that vendor, or a confidently detected sibling mark of the same product — clean images stay untouched); `strict` never relaxes. Metadata provenance is read automatically and feeds `auto`. (`assume-ai` was REMOVED in 0.16 — see the registry bullet; a user who can SEE a missed mark should point at it with `erase --region`, or name it with `--mark --no-detect`.) For arbitrary logos/objects use `erase`. When no known mark is detected the command writes no output and exits with the no-visible-mark code instead of re-serving the input; `--no-detect` forces the gemini fallback and proceeds. See the module doc for the routing/exit detail. `--backend` and `--sensitivity` are shared across `visible`/`all`/`batch`. -- `uv run remove-ai-watermarks erase --region x,y,w,h -o ` — universal region eraser (any logo/object, any position). `--backend cv2` (default, no deps), `--backend migan` (MI-GAN via onnxruntime, extra `migan`; ~28 MB, ~1 GB RAM, near-LaMa), or `--backend lama` (big-LaMa, extra `lama`; best quality but ~4.7 GB RAM); `--region` is repeatable. -- `uv run remove-ai-watermarks identify ` — provenance verdict (platform + watermark inventory + confidence); `--json` for machine output, `--no-visible` to skip both registered visible detectors and the optional open invisible-watermark decoder -- `uv run remove-ai-watermarks metadata --check` — inspect AI metadata (C2PA, EXIF, PNG chunks) -- `uv run remove-ai-watermarks metadata --remove -o ` — strip all AI metadata -- `uv run remove-ai-watermarks batch ` — process every supported image in a directory (output defaults to `_clean/`, set with `-o`). `--mode visible|invisible|metadata|all` (default `visible`); the invisible/all path reuses the full `invisible` knob set above, plus `--backend` and `--sensitivity` for the visible localize -> fill pass. Applies the same no-signal skip per image; see the module doc. **Exit code:** non-zero when any image errored OR (mirroring single `all`) a `--mode invisible`/`all` image carried an invisible signal but the GPU extra was absent, so its SynthID scrub was skipped — it emits a loud warning and copies the input through (invisible mode) so the output dir stays complete; a wrapping service can then detect the incomplete run instead of trusting a silent exit 0. - -## Test and lint - -- **CI** (`.github/workflows/test.yml`): runs on push to `main` + every PR. A `lint` job (ubuntu: `ruff check` + `ruff format --check`) plus a `test` matrix (ubuntu/macos/windows x py3.10/3.12) that does `uv sync --frozen --extra dev` then `pytest`. The matrix installs only core + dev (no `gpu` extra), so the GPU/model-running tests skip there and it exercises the metadata/identify/visible/cv2-eraser surface on all three OSes. Keep `uv.lock` valid (don't break `--frozen`) when editing `pyproject.toml`. -- Dependency PR checks run the GitHub merge ref against current `main`, not the contributor branch in isolation. If `main` moves after the dependency branch was opened, merge current `main` locally and rerun the full gate; a new linter can expose stale directives in code that landed later. -- **Release flow + distribution channels** (PyPI publish via `publish.yml`/`uv publish`, the automated Homebrew-tap + HF-Space bumps in `distribute.yml`, conda-forge, ComfyUI Registry, the sdist `data/` exclusion, hatchling pin history): see `docs/release-and-distribution.md` before cutting a release. -- `bash maintain.sh` — uv-outdated, uv-secure, ruff check/fix, ruff format, pyright (scoped `src/`, see the OOM note below), pytest -n auto. The helper tools live in the `dev` extra (`pytest-xdist`, plus `uv-outdated`/`uv-secure` marker-gated to py3.12+ so the py3.10 resolution stays solvable) — a bare env without `--extra dev` does not have them. -- **Strict pyright is clean across `src/` (0 errors).** The cv2/torch/diffusers boundary files (`gemini_engine`, `region_eraser`, `doubao_engine`, `humanizer`, `invisible_engine`, `noai/watermark_remover`) carry a documented per-file `# pyright:` relax pragma that turns off only the unknown-type / untyped-third-party rules — those libs ship no usable types, so strict typing there fights the ecosystem. Pure-logic files stay fully strict; `typings/piexif/__init__.pyi` is a local stub so `metadata.py`/`extractor.py` resolve piexif. Public ndarray-returning signatures on the relaxed engines are still annotated `NDArray[Any]` so strict consumers (`cli.py`) stay clean. When touching a relaxed file, prefer fixing real issues over widening the pragma; keep the pragma scoped to genuinely-untyped boundaries. The `uv-secure` CVE-resolution history (idna/aiohttp bumps, retired basicsr, the dismissed torch `GHSA-rrmf-rvhw-rf47`) lives in `docs/release-and-distribution.md` — read it before re-triaging a dependency alert. -- **Full-project `uv run pyright` (no path) OOMs/crashes node on this ML-heavy repo** (emits a `libnode` stack frame, no summary) — a known environment limit, not a code error. Gate with `uv run --extra dev --extra gpu pyright src/` (completes, authoritative) or scope to changed files; also run `uv run ruff check` and `uv run pytest` directly. -- Run `uv run` from the repo root — from another cwd it falls back to a bare env without numpy/cv2/torch. -- **Stale `trustmark` remnant in site-packages after an extras change:** the `trustmark` package downloads model weights INTO its own package dir, so when a narrower `uv sync` prunes the package, a `trustmark/models/` directory survives as an empty namespace package. Symptom: pyright `"TrustMark" is unknown import symbol` on `trustmark_detector.py` and `find_spec("trustmark")` returning a loader-less spec (so `is_available()` lies True). Fix: `rm -rf .venv/lib/python3.12/site-packages/trustmark` (regenerable weights cache). -- To add a dev tool (pytest/ruff/pyright) into the env, use `uv sync --frozen --extra dev --extra gpu`, **never `uv pip install`** — `uv pip install` re-resolves and rewrites `uv.lock`, which silently bumped `transformers` to a build incompatible with the pinned `diffusers` (`cannot import name 'Qwen3VLForConditionalGeneration'`) and broke every `identify`/metadata import. Recovery: `git checkout uv.lock && uv sync --frozen --extra gpu --extra dev`. The `gpu` extra holds `diffusers`/`transformers`/`torch`, so a bare `uv sync` (no extras) removes them; `noai/__init__` is now **lazy** (PEP 562 `__getattr__`, so importing `identify`/`metadata` no longer pulls `watermark_remover`/torch), so a bare env breaks only when the removal pipeline is actually invoked, not on import. `maintain.sh`'s `uv sync --all-extras` also pulls the heavy `trustmark`/`lama` wheels (pytorch-lightning, onnxruntime) — fine on a good connection, but on flaky DNS sync only `--extra gpu --extra dev` and run the lint/test steps by hand. -- Metadata/C2PA tests assert against real committed fixtures in `data/fixtures/provenance/` (`chatgpt-*.png` = OpenAI C2PA, `firefly-1.png` = Adobe, `mj-1.png` = Midjourney IPTC, `doubao-1.png` = ByteDance Doubao with the China TC260 `` XMP label **and** a visible "豆包AI生成" text mark bottom-right; `grok-1.jpg` = xAI Grok with its EXIF-only `Signature:` blob + UUID `Artist` and no C2PA/SynthID/IPTC; `flux-1.png` / `flux-1.jpg` = real Black Forest Labs FLUX.2 Playground output, signed C2PA (issuer "Black Forest Labs" + `trainedAlgorithmicMedia`) -- `flux-1.jpg` is the first committed **JPEG-with-C2PA** fixture, exercising the c2pa-python non-PNG reader path end to end; whether BFL hosted output also embeds the open DWT-DCT pixel watermark is UNRESOLVED -- our detector returns None on these fox samples, but they are high-texture carriers where even a known-embedded watermark fails the round-trip, see the content-fragility caveat in `docs/watermarking-landscape.md`); synthetic byte blobs cover the remaining JPEG/ISOBMFF format paths. The `clean_photo` conftest fixture generates a deterministic metadata-free PNG; no real negative photo is committed for tests. -- Repository data follows `data/README.md`: executable provenance fixtures live in `data/fixtures/`, minimal detector rebuild inputs in `data/calibration/`, canonical provider-oracle originals in `data/synthid/`, and evaluation-only ground truth in `data/evaluations/`. Store each binary once and point every consumer at the canonical path. -- SynthID oracle fixtures: `scripts/synthid_corpus.py` ingests labeled originals into `data/synthid/originals/`. The tracked `manifest.csv` is kept in sync with the files on disk, one row per image. `full-pipeline-quality.csv` is the reusable full-pipeline test set: read its single canonical copy, preserve `source_filename` in outputs, and keep provider groups separate for their respective oracles. Generated or cleaned outputs stay outside the repository; record their reproducible command, hash, and oracle verdict instead. +Run `uv` from the repository root. Command selection, options, defaults, and examples live in [`docs/cli.md`](docs/cli.md). Before changing command routing, no-signal behavior, or exit codes, read the command-line section of [`docs/module-internals.md`](docs/module-internals.md). ## Configuration -- GPU/ML modules (invisible_engine, watermark_remover) are optional — guard imports with `is_available()` checks -- Optional detection extras: `detect` (imwatermark — open SD/SDXL/FLUX watermark) and `trustmark` (Adobe TrustMark decoder; pulls torch + downloads weights). Both are guarded by `is_available()` and skipped by `identify` when absent. -- Optional `esrgan` extra (spandrel only): Real-ESRGAN pre-diffusion super-resolution for small inputs (`upscaler.py`, CLI `--upscaler esrgan` on `invisible`/`all`/`batch`). Guarded by `upscaler.is_available()`; the default upscaler stays Lanczos (cv2, no deps) and the engine falls back to Lanczos when the extra is absent or the model errors. spandrel is MIT and pulls NO basicsr (only torch/torchvision/safetensors/numpy/einops); Real-ESRGAN weights are BSD-3-Clause and download on first use via `torch.hub` (never bundled). Kept OUT of `all` (heavy + model download). -- Tests for the *model-running* paths are limited to availability checks (multi-GB downloads). But the **pure helpers inside ML-adjacent modules are unit-tested without any download** and must stay that way: `_target_size` (native-vs-downscale-cap-vs-upscale-floor, `test_invisible_engine.py`), `humanizer.unsharp_mask`/`adaptive_polish` (`test_humanizer.py`), and the MPS->CPU fallback control flow via mocked pipelines (`test_img2img_runner.py`, 100% cover). Don't skip these as "ML, needs a model" — only `remove_watermark`/the diffusion bodies do. +GPU and ML modules are optional. Guard their imports with `is_available()`. -## Key modules +Optional features and installation groups are documented in [`docs/installation.md`](docs/installation.md). Model-running paths may use availability tests, while pure helpers in ML-adjacent modules must remain unit-tested without downloads. -Compact map. The full per-module detail (design decisions, tuned thresholds, calibration history, incident records, and the regression-guard map) lives in `docs/module-internals.md` — **read the relevant section there before changing any module below.** +## Test and lint -- `noai/c2pa.py` — C2PA reading. `extract_c2pa_info(path)` uses the official **c2pa-python `Reader`** first (core dep, any container; `read_manifest_store_json` returns the WHOLE store JSON — active + ingredient manifests — so an AI marker on a parent manifest is seen), and falls back to the hand-rolled caBX/CBOR parser (`has_c2pa_metadata` / `extract_c2pa_chunk` / `_extract_c2pa_info_png`) for synthetic/partial blobs the validator rejects or a broken/absent wheel. The registry scan (issuer / source-type / SynthID / soft-binding) is shared by both paths via `_populate_registry_fields`, so the return-dict shape is identical. Do not reimplement chunk parsing; chunk reads are clamped to the remaining file size by design. `extract_c2pa_chunk`/`inject_c2pa_chunk` stay PNG-only (raw caBX bytes, test/extractor use). -- `noai/constants.py` — the single `C2PA_AI_VENDORS` registry (+ `C2PA_SOFT_BINDINGS`) from which `C2PA_ISSUERS` / `SYNTHID_C2PA_ISSUERS` / `C2PA_IDENTITY_AI_ORGS` / `identify._ISSUER_PLATFORM` are all derived. Add a new vendor as one registry entry; never edit the derived dicts and never add inline. A vendor's `asserts_ai=True` flag means its mere presence asserts AI generation even without a `trainedAlgorithmicMedia` digital-source-type (a pure-generator brand with a distinctive issuer/generator string, e.g. **Dreamina** — ByteDance's international Jimeng brand, signed as "Bytedance Pte. Ltd." with a "Dreamina/x.y" claim generator and no source-type); NEVER set it for common-word issuers (Adobe/Google/OpenAI/Microsoft) that appear incidentally in unrelated bytes — those stay source-type-gated in `identify._attribute_platform`. -- `metadata.py` — `scan_head(path)` is the shared (memoized) input for every C2PA/AIGC/IPTC byte scan; use it instead of `open().read(1MB)` for any new marker scan. Also home to `synthid_source`, `xai_signature`, `iptc_ai_system`, `aigc_label`, `huggingface_job`, `samsung_genai`, and `remove_ai_metadata` (fail-safe `strip_c2pa_boxes`). **A caller that REPORTS an outcome must use `strip_and_verify`, not `remove_ai_metadata` directly** -- the stripper is deliberately fail-safe (a file PIL cannot decode is copied through UNCHANGED rather than crashing), so its return value cannot distinguish a no-op from a real strip. `strip_and_verify` re-scans the output and, when metadata survived but `image_io` can still decode the raster, normalizes the container and scans again; that recovery preserves pixels but drops standard metadata. A truly undecodable file keeps the surviving-marker result. `metadata --remove` and `batch --mode metadata|all` use this verified path. **`remove_ai_metadata` is the SINGLE metadata stripper** (the legacy PIL-re-encoding `noai/cleaner` was deleted; the diffusion core and the public `noai.remove_ai_metadata` re-export now point here). It strips **losslessly** per container: ISOBMFF (HEIC/AVIF/MP4) blanks tokens / strips boxes in place; **JPEG uses `_strip_jpeg_metadata_lossless`** — a marker-segment walk that drops the AI-bearing APP segments (C2PA APP11; XMP APP1 carrying C2PA, a China-AIGC token, OR an IPTC `digitalSourceType` / 2025.1 AI-disclosure marker; IPTC-IIM APP13) and scrubs AI EXIF tags via piexif, copying the entropy-coded scan verbatim so **the pixels are bit-identical** (no DCT re-encode). **Detection<->removal parity across every marker placement is load-bearing** — anything a scanner flags, the strip must reach, or a re-served file still reads as AI: (a) the APP1-XMP branch of `_jpeg_app_carries_ai` checks the IPTC marker sets too, not only C2PA/AIGC (the Instagram/MidJourney/Meta "Made with AI" `digitalSourceType` lives in XMP, not the APP13 IIM record); (b) a bare `AIGC{...}` / `{"AIGC":{...}}` block in ANY JPEG APP segment — the specific C2PA(APP11)/XMP(APP1)/IPTC(APP13) checks FALL THROUGH to a generic `_is_aigc_exif_value` drop, so a bare AIGC in APP11 (the supported placement, NOT a C2PA manifest) is caught, not swallowed by the C2PA-only 0xEB branch — plus the same AIGC block in a STANDARD **PNG text chunk** value (e.g. `Description`, which `_is_ai_key` keeps) is dropped on the value; (c) the China TC260 `{"AIGC":{...}}` block in EXIF `UserComment`/`ImageDescription` is scrubbed by `_scrub_ai_exif` (Doubao producer + Tencent service-provider schemas); (d) the Samsung Galaxy AI `PhotoEditor_Re_Edit_Data` trailer past the JPEG EOI is truncated by `_strip_samsung_trailer` (and `samsung_genai` reads the file tail so a multi-MB photo's trailer past the 512 KB quick-scan window is still DETECTED). Pixels stay bit-identical throughout, so a `--strip-metadata` on a q100 removal output does NOT crush it back to q75; PNG/WebP re-saves are pixel-lossless (WebP written at cv2 lossless mode, quality 101 — quality 1-100 is lossy). **The PIL-fallback save format is chosen by the source's CONTENT, not its file extension** (`_sniff_image_format`, and the JPEG-lossless gate is content-gated too): inputs can be misnamed (a PNG served as `.jpg` is the common one), and routing on the extension re-encoded a lossless PNG/WebP into a real JPEG — a silent degradation that broke "work with originals". A **misnamed** lossless source (source-extension format != content) is preserved in its true format; a **correctly-named** source still honors a deliberate output-extension conversion (e.g. `source.png -> output.jpg`). Not yet handled: a 16-bit PNG is downconverted to 8-bit on the PIL re-save (rare; would need a byte-level PNG chunk stripper). Regression: `tests/test_metadata.py::TestHasAiMetadata::test_strip_preserves_lossless_content_with_mismatched_extension`. **`remove_ai_metadata` is fail-safe on an undecodable image:** a truncated/corrupt file (PIL raises `OSError` decoding it; some inputs) is copied through UNCHANGED rather than crashing a direct library caller (a web worker would 500 on a partial upload), mirroring `strip_c2pa_boxes` — we cannot strip what we cannot parse, but we never raise. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_remove_ai_metadata_failsafe_on_truncated_png,test_strip_and_verify_normalizes_decodable_copy_through,test_strip_and_verify_reports_markers_in_undecodable_copy_through}`. Regression: `tests/test_metadata.py::TestHasAiMetadata::{test_jpeg_metadata_strip_is_pixel_lossless, test_jpeg_strip_removes_iptc_marker_in_xmp}`, `TestSamsungGenai::{test_remove_strips_post_eoi_trailer, test_detects_trailer_past_scan_window}`, the AIGC-EXIF/bare-APP removal tests, and `tests/test_noai.py::TestISOBMFF::{test_blank_aigc_block_in_exif, test_blank_xai_signature_pair_in_exif}`. `exif_generator` matches a VALUE against `AI_GENERATOR_TOKENS` across EXIF `Software`/`Make`/`Artist`/`ImageDescription`, XMP `CreatorTool`, AND PNG `tEXt` chunks (`Software`/`Source`/`Title`/`Description` — NovelAI stamps there, not EXIF). **Detection and removal must stay in parity:** a generator that stamps an AI-shaped VALUE under a non-AI KEY (NovelAI's `Title`/`Source`) is dropped on removal by `_is_ai_value` (value-token match, mirrors `exif_generator`), NOT by `_is_ai_key` alone — else the cleaned file still reads as that generator. Add a new no-C2PA generator = one `AI_GENERATOR_TOKENS` entry (use a distinctive token, e.g. `reve.com` not bare `reve`); detection and removal then both follow. Regression: `tests/test_metadata.py::TestExifGenerator::{test_novelai_png_text_chunk_detected,test_novelai_removal_parity}`. -- `identify.py` — separates file-backed metadata extraction (`extract_provenance_evidence`) from metadata-only verdict logic (`identify_from_evidence`); the compatible `identify` wrapper adds optional pixel-backed visible and invisible checks. External metadata ingestion (`evidence_from_metadata_record`) must reuse the same pure parser primitives in `metadata.py` as file-backed extraction; never duplicate signal regexes or field registries. Detection from `ProvenanceEvidence` must never reopen the source. All paths produce one `ProvenanceReport`; `is_ai_generated` is True or None, never asserted False. `ProvenanceReport.ai_source_kind` exposes the C2PA digital-source-type split — `"generated"` (trainedAlgorithmicMedia, fully AI) vs `"enhanced"` (compositeWithTrainedAlgorithmicMedia, a real photo with an AI-composited region), else None — so a caller branches full-frame scrub vs region-targeted clean (see `noai/tiling.feather_region_composite` + `WatermarkRemover.remove_watermark(region=...)`). The sparkle provenance threshold is the SHARED `watermark_registry.GEMINI_SPARKLE_TRUST_CONF` (imported, not a private copy) so the provenance "is there a sparkle" verdict and the removal "take the sparkle" decision can never drift. `import identify` is deliberately light (lazy `noai/__init__`, fits a 512 MB host) — keep heavy imports out (the `watermark_registry` constant import stays light: engines are lazy there). Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM` only when verified against a real C2PA file; editing-app/AI-device signer tokens go to `_SIGNER_C2PA_PLATFORM`; generator/issuer platforms to `C2PA_AI_VENDORS` in `constants.py`. The IPTC `digitalSourceType` **`algorithmicMedia`** (bare) is PROCEDURAL (an algorithm not trained on sampled data), NOT AI/ML generation, so it is deliberately absent from `IPTC_AI_MARKERS` — flagging it made `identify` assert AI + `has_invisible_target` True, scrubbing clean procedural content (it is a distinct token from `trainedAlgorithmicMedia`, so real "Made with AI" labels are unaffected; regression `test_metadata.py::...test_bare_algorithmic_media_not_flagged_ai`). Integrity-clash detection is high-precision by design (only hard generator stamps feed it, source-grouped independence). `_vendor_of` normalizes ByteDance/Canva/ElevenLabs/Black Forest Labs (as well as OpenAI/Google/... ) so their C2PA claims participate in the clash check; the generic **China TC260 AIGC label names no specific vendor**, so when a TC260-applying vendor (ByteDance, `_TC260_VENDORS`) is co-attributed the label is attributed to it (a legit Doubao image carrying its own TC260 label must NOT clash), while a NON-TC260 vendor next to a TC260 label still clashes as a laundering tell. The vendor normalization must not introduce clashes on clean compatibility samples. -- `watermark_registry.py` — the single catalog of known visible watermarks (gemini / doubao / jimeng / qwen / kling / samsung / runninghub / baidu / liblib / jimeng_pill). **Removal is LOCALIZE -> FILL for every mark:** each mark is localized to a binary full-frame footprint mask (a `Localization`), then ONE shared, swappable fill inpaints that mask via `fill(image, mask, backend=...)` (delegates to `region_eraser.erase`). Reverse-alpha (the old `original = (wm - a*logo)/(1-a)` inversion of a captured alpha map + thin residual inpaint) is GONE for ALL marks; why it was dropped is recorded in `docs/module-internals.md`. Backends: `cv2` (classical inpaint, no deps, the floor), `migan` (MI-GAN ONNX, light, the memory-tight pick where LaMa will not fit), `lama` (big-LaMa ONNX, best quality, heavier, auto-preferred when a learned backend is available); `auto` = LaMa > MI-GAN > cv2, best available. The captured alpha maps (`scripts/visible_alpha_solve.py`) are still used to DETECT the marks and to shape the mask, but NOT for pixel recovery. **`--mark auto` removes EVERY detected mark in one pass** via `remove_auto_marks(image, *, sensitivity="auto", provenance=frozenset(), backend="auto")` (marks coexist -- a Jimeng-basic image has the top-left pill AND the bottom-right wordmark; a single-strongest pick would leave one). **Three orthogonal axes:** `backend` (the fill), `sensitivity` (how hard to trust a borderline mark: `auto`/`strict`, see the `Sensitivity` literal), and `provenance` (vendor keys metadata confirms -- the evidence that drives `auto`). **Perception / decision / action are separated:** `_build_candidates(image)` runs every detector at BOTH trust levels (strict + relaxed) and packages raw verdicts + features into `Candidate`s (no policy); the pure arbiter `decide(candidates, Context(sensitivity, provenance)) -> [Decision]` makes every keep/drop call (per-mark `resolve_trust` + the assumed-trust floor + the pill gate) with no image/IO, so it is unit-testable in isolation; then each winner is localized -> filled. Do NOT put policy back into the engines (the one exception, the Gemini FP gate, stays in `gemini_engine` because `identify` shares that confidence). `detect_marks(..., provenance=frozenset())` stays strict (identify verdict, precision over recall); `KnownMark.remove/detect/localize(..., provenance: bool)` take the already-resolved boolean. **How `auto` decides (this is metadata-INDEPENDENT for recall):** the visual detectors are pixel-based and need no metadata; the recall gain comes from RELAXING the false-positive gate, not from metadata. `strict` never relaxes (clean images untouched); `auto` relaxes a mark only on same-product evidence -- metadata provenance for that vendor OR a confidently detected sibling mark of the SAME product (`_PRODUCT_OF`; Doubao and Jimeng are both bottom-right ByteDance but distinct products, so they do NOT cross-relax). **`resolve_trust` resolves TWO levels:** `confirmed` bypasses the engine's false-positive gate, and only `confirmed` has evidence naming THAT vendor, which is exactly what the bypass is contracted to require (`GeminiEngine.detect_watermark`'s `trust_provenance` docstring: "external metadata already proves this is a Google generation"). **Historical, kept as the reason the third level is gone:** a removed `assumed` level let `assume_ai` bypass the gate on the bare assertion an image is AI. It produced unacceptable false positives on clean camera images and was removed. A wrong relaxation only fills a small corner near-losslessly (the localize -> fill benign failure mode), which is what made a SMALL false-fire rate arguable; it was never a licence for a 60% one. Metadata provenance mapping (feeds `auto`, read by `cli._visible_provenance`): Google/Gemini C2PA issuer -> gemini; China-AIGC (TC260) label -> doubao/jimeng; `samsung_genai` -> samsung. **The `jimeng_pill` is capture-less.** It uses a synthetic silhouette and a fixed top-left footprint. `_keep_pill` requires either a confirmed sibling wordmark or Jimeng provenance with a flat footprint. Do not loosen the flatness guard. **`assume_ai` was REMOVED (2026-07-19); `--sensitivity` is now `auto`/`strict` only.** It relaxed every mark's FP gate on the bare assertion an image is AI -- which names no vendor and no location, exactly what the bypass requires -- and had no place in the model (detector finds -> remove; finds nothing -> leave alone; user SEES a mark -> act on that). It took `_ASSUMED_CONF_FLOOR` / `assumed_floor_ok` / the `assumed` trust level with it, collapsing the ladder to `strict`/`confirmed`, and `_keep_pill` lost its `sensitivity` arg. Recall/precision on the unbiased sample are unchanged, so nothing on the default path moved. **Replacement advice is per mark:** `erase --region` is sound by construction; `--mark --no-detect` is reasonable (forced mask = the real glyph blob, non-empty 13/13); **`--mark gemini --no-detect` is NOT** -- it falls back to a fixed slot that covered the true sparkle on only **31% of 97** missed sparkles, so 69% fill a clean corner AND report a removal that did not happen. `cli._no_visible_mark_exit` follows that order and no longer suggests the removed mode. Migration raises loudly (`validate_sensitivity`, called from `api.remove_visible` and `Context.__post_init__`) because a `Literal` is unenforced at runtime and would silently downgrade a 0.15 caller to `auto`. **Detection and removal-mask extraction must use the same front-end.** The continuous `tophat` path can detect a mark whose binary glyph blob is empty, so its fallback mask uses the detector's own best-match box. Keep the textured regression fixture when changing this path. **Detector thresholds and geometry are calibrated per mark; do not port them between vendors without a fresh evaluation.** **New mark assets must be synthetic.** Font-render the mark, calibrate it on local evaluation inputs, and commit only the synthetic silhouette. Never derive a committed asset from source images. -- `gemini_engine.py` — visible Gemini-sparkle detector + localizer (cv2/numpy, no GPU): top-K size-weighted fusion candidate selection (`_SELECT_TOPK`), corner-promote, false-positive gate (the provenance prior relaxes the gate + lowers the trust threshold when a Google/Gemini C2PA issuer confirms the vendor). **White-core rescue:** the FP gate demotes a low-gradient match (soft edges), but a real FAINT sparkle also has soft edges -- so the gate keeps a low-grad match that is a strong (conf ≥ `_SPARKLE_KEEP_CONF` 0.52), bright (margin), near-WHITE-core sparkle (`_core_saturation` ≤ `_SPARKLE_WHITE_SAT` 0.20): a real sparkle core is white, a clean bright corner that shape-matches (sky/sun) is colored. This recovers ~14/20 metadata-stripped faint sparkles under the DEFAULT strict/auto (no flag, no metadata) at ~1.25% clean false-fire (baseline 0.55%); the ~0.51-scoring bright-bg FPs stay demoted (below 0.52). A learned classifier on the SAME features was measured WORSE than the tuned gate (2026-07 tier-1: MLP 86.7% recall vs 90.8% at equal FP), so the heuristic stays; a patch-CNN with richer features is the only lever left (roadmapped P2, low expected value -- the wall is fundamental). Detection scores the top-K size-weighted matches by full fusion (spatial+gradient+variance) and keeps the highest — NOT the raw-NCC argmax, which re-admits the tiny-patch FPs the size weight suppresses (the osachub 2026-06-12 sub-0.85 corner-sparkle regression; see `docs/module-internals.md`). Keep the 0.85 corner-promote NCC gate; a margin/chroma-gated lower promote was measured and REJECTED 2026-06-11 (~33% FP on non-Google content). Removal is localize -> fill: `footprint_mask` returns the sparkle footprint (the captured alpha thresholded LOW so the faint halo is included, then dilated by a sparkle-relative margin), and the shared `watermark_registry.fill` inpaints it. The captured alpha maps are used only to detect and to shape the mask, not for pixel recovery. `detect_sparkle_confidence` reuses a process-wide `_shared_engine()` singleton (lru_cache) — the engine holds only constant assets (captures, alpha maps, a precomputed 16..118 template ladder) and takes the image as an arg, so do NOT reconstruct `GeminiEngine()` per call: that reloaded assets + recomputed alpha maps + rebuilt the template cache on every one of ~34k `identify` calls (−24% on the sparkle path once made a singleton, output byte-identical). `detect_watermark`/`footprint_mask` guard `image.size == 0` before `to_bgr`, and return an empty (detected=False) result when no template scale fits (short side < 16 px), rather than dereferencing an empty candidate list. -- `_text_mark_engine.py` — shared base for the text-mark engines (extracted 2026-06-09); the per-engine modules are config-only subclasses. Detection still matches the glyph silhouette (NCC, keys on glyph shape). The removal mask is TEMPLATE-FREE: it is the bounding box of the top-hat glyph blob (`extract_mask`), filled solid + dilated, so the shared fill inpaints the whole wordmark rectangle. This drops the fixed alpha-template placement, so a re-rendered or differently-placed mark is still masked; the captured alpha maps are now used only for the detection silhouette, not for removal. New text mark = a `TextMarkConfig` + a thin subclass + one registry row. Gemini stays a separate engine (different model). The corner anchor is `corner` = `br`/`bl`/`tl`/`bc` (tl added 2026-07-22 for runninghub, bc for liblib's centered wordmark); `detect_frontend` is `binary`/`tophat`/`gray` (`gray` = raw-grayscale NCC for the faint mid-gray runninghub mark, added 2026-07-22; contrast-DEPENDENT, so its gates never port). The detection scale ladder is per-mark (`TextMarkConfig.ladder`, default `(0.8, 1.0, 1.25)` -- added 2026-07-21 for qwen's two size modes; the shared default is unchanged for every other mark, and densifying the SHARED ladder was measured and rejected, see `docs/verification-plan.md` B2). -- `pill_engine.py` — detects the capture-less Jimeng "AI生成" pill with a synthetic silhouette and removes it through the shared fill path. Its weak detector is registry-gated: a sibling wordmark may confirm it, while metadata-only removal also requires a flat footprint. Do not loosen those gates. -- `doubao_engine.py` / `jimeng_engine.py` / `samsung_engine.py` — thin `TextMarkEngine` subclasses: Doubao "豆包AI生成" (bottom-right), Jimeng "★ 即梦AI" (bottom-right), Samsung Galaxy AI "✦ Contenuti generati dall'AI" (bottom-LEFT, locale-specific — Italian variant calibrated). Detection matches the glyph silhouette (NCC); removal localizes the glyph blob to a solid dilated box (`extract_mask`) and hands it to the shared fill. Calibration confirms that doubao and jimeng localize and remove cleanly, while clean compatibility images remain unchanged. **Samsung detection is calibrated only for the Italian "Contenuti generati dall'AI" string** (a pre-existing limit, unchanged by the localize -> fill refactor but now surfaced because detection gates removal): non-Italian Samsung locales are not detected, and thus not removed, even though the fill mask itself is locale-independent; other locales need their own detection silhouette (the locale string font-rendered + calibrated on real positives), NOT an app capture. -- `qwen_engine.py` — detects the Qwen "千问AI生成" bottom-right text mark with a synthetic silhouette and per-mark geometry. Keep its calibration independent from similar CJK marks. -- `kling_engine.py` — detects the Kling "可灵AI 3.0" bottom-right text mark. It uses short-side geometry and a synthetic silhouette. -- `yuanbao_engine.py` — detects the standard two-line Tencent Yuanbao "元宝 / AI生成" bottom-right mark through polarity-independent local contrast. The one-line overlay variant remains unsupported. -- `runninghub_engine.py` — detects the faint top-left RunningHub mark through grayscale silhouette matching. The anchor gate and detector-owned match box are part of its false-positive control. -- `baidu_engine.py` — detects the Baidu "百度 AI生成" bottom-right mark. Rival margins separate it from similar CJK marks, and its custom footprint includes the adjacent tag. -- `liblib_engine.py` — detects the bottom-center LibLibAI wordmark with a synthetic silhouette, contrast gating, and detector-owned footprint. -- `region_eraser.py` — universal region eraser (`erase` CLI) and the shared fill backend behind `watermark_registry.fill` for the visible localize -> fill removal. Three backends: `cv2` (default for the user-directed `erase` command, no deps, the floor), `migan` (MI-GAN ONNX, extra `migan`, MIT, ~28 MB / ~0.19 s, the memory-tight learned tier), `lama` (big-LaMa ONNX, extra `lama`, ~200 MB / ~4.7 GB peak, best quality but too heavy for a minimal worker). The visible registry's `auto` resolution is **LaMa > MI-GAN > cv2**; select MI-GAN or OpenCV explicitly when memory matters. Both `migan` and `lama` **crop a padded region around the mask** before inference and paste only masked pixels back, so peak RAM is bounded by the MARK size, not the image (`migan` ~0.6-0.9 GB regardless of upload size — feeding the whole frame scaled it to ~2.4 GB at 25 MP; `migan` feeds the crop at native resolution, `lama` resizes to its fixed 512²). **Measured end to end 2026-07-20** (`scripts/resource_ceilings.py`, fresh process per cell, 1 MP → 25 MP): `migan` 603 → 775 MB and `lama` 4679 → 4779 MB, both **flat in input size** — the crop-around-the-mask design holds and both documented figures reproduce. **`cv2` is the only backend that GROWS with the input** (74 → 440 MB, 5.9x) because it inpaints the full frame rather than a crop; still the cheapest tier, but size it for the largest upload accepted. Cold wall time 0.02-0.12 s (cv2) / ~0.6 s (migan) / ~3.8 s (lama), model load included. (The harness's own no-op check originally allocated a full-frame temp before reading peak RSS and inflated these by up to 17% at 25 MP; it now compares only the mask box. The conclusion survived re-measurement, the digits moved.) **MI-GAN mask polarity is INVERTED** (0=hole/255=known) vs this package's 255-erase convention; `erase_migan` inverts before feeding the model (feeding 255=hole regenerates the whole frame into stripes — verified). Both ONNX models download on first use, never bundled. The `erase` command keeps its own `--backend`/`--inpaint-method` (unchanged). -- `invisible_watermark.py` — decodes the OPEN DWT-DCT watermarks (SD / SDXL / FLUX) via `imwatermark` (extra `detect`, pulls torch). Fragile two ways: (1) does not survive JPEG re-encode/resize; (2) **carrier-fragile on a broad class of pristine images** -- a clean encode->decode round-trip recovers 48/48 on chatgpt/firefly/random but FAILS (28-39/48, below the `_MATCH_48`=44 gate) on the FLUX fox, doubao, a flat FLUX generation, AND a clean synthetic flat fill with no watermark. The failure does NOT track texture; it goes with a degenerate **all-ones decode that is a CARRIER ARTIFACT, not a watermark** (synthetic clean image reproduces it). So `detect_invisible_watermark` is **positive-only**: trust a hit; a `None` is inconclusive unless a same-carrier positive-control embed first recovers >=44. Verified 2026-06-19; full caveat in `docs/watermarking-landscape.md`. -- `trustmark_detector.py` — Adobe TrustMark open decoder (extra `trustmark`). Do NOT remove the JPEG re-encode false-positive gate — a lone TrustMark hit without it is almost always content noise. -- `noai/watermark_remover.py` — `WatermarkRemover` with four diffusion pipelines selected by the explicit `pipeline` ctor arg, never inferred from `model_id`: `sdxl` (plain SDXL img2img), `controlnet` (SDXL + canny ControlNet, **the compatibility and cost DEFAULT since 2026-06-09**), `qwen` (Qwen-Image 20B img2img), and `qwen-zimage` (delegates to the fixed two-stage runtime below). Removal comes from img2img strength. Both SDXL loaders pass `add_watermarker=False`; diffusers otherwise re-stamps an open SDXL DWT-DCT watermark. Qwen's certified floors and fidelity results remain as documented below. The base `qwen` profile stays the manual text lane; `qwen-zimage` is the recommended high-quality manual mode, especially for face identity, while remaining experimental rather than an auto-router. -- `noai/qwen_zimage_pipeline.py` — CUDA-only Qwen-Image-2512 Lightning + DiffSynth Canny full-frame regeneration, followed by YuNet face boxes, SAM masks, and Z-Image Turbo regeneration from original face crops. Ports both adaptive denoise formulas from Synthid-Bypass v2, then scales the face result by 0.5 because this runtime lacks the reference latent noise-mask feather and uses a different sampler/compositing path; paired face evaluations and both provider oracles certified the scaled value while the global stage stayed unchanged. The active upstream face path is YOLO + SAM; this port keeps its center-point and box prompts, proposal selection, detector-box intersection, crop factor, and paste feather while replacing YOLO with YuNet to avoid an AGPL runtime. The port is architectural, not bit-identical: it uses full safetensors instead of GGUF, DiffSynth samplers instead of the Comfy sampler pairs, and no latent detailer feather. DiffSynth input pixels, Canny control, and explicit dimensions must share the same /16 grid. SAM pixels follow the model dtype, geometric prompts stay float32, and bfloat16 outputs convert through float32 before NumPy. The YuNet download verifies its SHA-256. Separate `qwen-zimage` extra; fixed four-step global and eight-step face schedules; no custom `--model`; `--tile` applies only to the global stage, followed by one full-frame face stage; CLI adaptive polish defaults off. `InvisibleEngine.preload(global_only=True)` warms Qwen and YuNet while leaving Z-Image and SAM lazy until a face is detected; the default `preload()` remains a full preload. GPUs with at least 64 GiB VRAM keep the face stack resident, while smaller devices retain CPU offload. Fixed prompt embeddings are cached only when they do not depend on an edit image. The exact seed-0 release candidate passed the corresponding provider-oracle checks; broader seeded text, face, and tiled-output certification remains open. -- `noai/tiling.py` — sliding-window tiled diffusion for large inputs (CLI `--tile`). The SDXL, ControlNet, and base Qwen paths branch to `run_tiled` when `tile` is set AND the long side exceeds `tile_size`, refactoring the single-pass `_generate` into a per-tile `_generate_one` (the ControlNet edge map is rebuilt per tile inside it). `qwen-zimage` instead calls `run_tiled` only around its global Qwen stage, blends the tiles, then runs one full-frame face stage. Pure helpers `plan_tiles` (uniform-size tiles, last one flush to the edge) and `feather_weights` (strictly-positive separable taper -> partition-of-unity blend) are unit-tested without the model. Also home to `feather_region_composite(base, regenerated, box, *, feather)` — the pure region-targeted compositor for **AI-enhanced composites** (`ai_source_kind == "enhanced"`): blends the regenerated AI box back over the original with a feathered seam, leaving the real photo OUTSIDE the box pixel-exact. It backs `WatermarkRemover.remove_watermark(region=...)` (regenerate ONLY the AI region, not the whole frame); the no-model lossless region path stays `region_eraser.erase`. New tile/region-blend tuning goes in these pure helpers; do not inline blend math into the runner. -- `auto_config.py` + the content-detection layer were REMOVED 2026-06-09; `--auto` is a deprecated no-op (controlnet is the default pipeline and adaptive polish is ON by default for the original profiles, while `qwen-zimage` leaves it off to preserve the upstream two-stage output). -- `upscaler.py` — optional Real-ESRGAN pre-diffusion super-resolution for small inputs (extra `esrgan`, spandrel only). Manual opt-in; the default `--upscaler` stays `lanczos` and the engine always falls back to Lanczos on absence/error. ESRGAN can degrade faces and thin text. -- `image_io.py` — centralizes Unicode-safe image IO, alpha preservation, content-based format sniffing, and HEIC/AVIF fallbacks. Callers must check `imwrite` success. No-op visible removal preserves original bytes when the output format is unchanged. -- `api.py` — the high-level convenience API, re-exported lazily at the package top level via `__init__.__getattr__` (PEP 562, so `import remove_ai_watermarks` stays cheap): `remove_visible(source, output=None, *, sensitivity="auto", backend="auto", strip_metadata=True, write_noop=True) -> (result_bgr, [labels])` (source = path OR BGR ndarray; a PATH auto-reads metadata provenance and preserves alpha, an ARRAY does neither; `write_noop=True` writes a clean passthrough copy when nothing is removed, `False` leaves `output` untouched so a "no mark = produce nothing" caller like the CLI `visible` command does not clobber a pre-existing file there) and `visible_provenance(path) -> frozenset[str]` (the single metadata→vendor-keys mapper; `cli._visible_provenance` is a thin None-guarded wrapper over it). **`remove_visible` is the ONE path the CLI and library share** — `cli.cmd_visible`'s `--mark auto` branch delegates entirely to it (read → provenance → `remove_auto_marks` → write → `strip_metadata`), so there is no CLI-vs-library drift; `strip_metadata` defaults True to match `visible --strip-metadata`. This is where a library caller should start — NOT the engines directly (`GeminiEngine`/`TextMarkEngine` have no `remove_watermark` any more; removal is registry `remove_auto_marks`/`KnownMark.remove`; the old single-strongest `best_auto_mark` is gone — removal takes EVERY mark). `identify` is NOT top-level re-exported (it collides with the `identify` submodule); use `from remove_ai_watermarks.identify import identify`. +`maintain.sh` runs dependency freshness and security checks, Ruff, Pyright scoped to `src/`, and the parallel test suite. Full-project Pyright is not the project gate because the ML dependency graph can exhaust Node memory. -For the Doubao alpha-distillation history (why content-image reverse-alpha distillation fails by physics and controlled captures were required), see `docs/research-doubao-distillation.md`. +Command, gate, typing, and model-test invariants auto-load from [`.claude/rules/development.md`](.claude/rules/development.md). Environment recovery, CI behavior, and fixture policy live in [`docs/development.md`](docs/development.md). -## Watermarking landscape +Before a release, read [`docs/release-and-distribution.md`](docs/release-and-distribution.md). Keep the source-distribution exclusion for `data/`. -Who embeds what (C2PA / IPTC / EXIF / TC260 AIGC / xAI signature / open and proprietary invisible watermarks), whether each is locally detectable, the C2PA 2.4 durable-credentials implications, and the regulatory driver table live in `docs/watermarking-landscape.md` (research 2026-05-24, updated through 2026-06-10). Read it before adding a new `identify` signal, vendor token, or metadata marker. See `identify.py` for what we read today. +## Module architecture -## Known limitations +[`docs/module-internals.md`](docs/module-internals.md) is the canonical per-module map, including design decisions, thresholds, calibration history, incident records, and regression guards. Read the relevant section before changing a subsystem. -Compact list. Full measurements, incident history, and oracle-validation runs live in `docs/known-limitations.md` — **read the relevant section there before changing the diffusion pipelines, strength defaults, resolution handling, or metadata coverage.** +Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). -- **Visible-mark fill quality is background/backend-dependent.** The fill only touches the mark footprint (no outside-box damage) and whether the mark is removed is fill-independent — cv2/MI-GAN/LaMa all strip the shape; only the recovered region's *quality* differs. Flat backgrounds: all clean (cv2 often crispest). Textured/regular-structured (fabric, grid): cv2 smears, MI-GAN can ghost/hallucinate, LaMa best. The old reverse-alpha recovered true pixels so it was sometimes cleaner on structure, but localize -> fill trades that for robustness (moved/re-rendered marks, no per-mark capture); `auto` = LaMa > MI-GAN > cv2 with a one-time cv2-fallback warning. Head-to-head vs v0.12.1 on the full visible set: doubao/jimeng identical (100%/100%), gemini strict coverage a few points lower (the metadata-stripped faint ones now mostly recovered by the default white-core rescue in the gemini FP gate), clearance ~98% both. Detail in `docs/known-limitations.md`. -- `invisible` processes at native resolution for inputs >= 1024px long side and auto-upscales smaller inputs to a 1024px floor (`--min-resolution 0` disables; `--max-resolution N` is an opt-in cap to bound GPU/MPS memory). MPS OOM is memory-tier dependent, not a hard limit: ~24 GB unified memory falls back to CPU (slow but weight-identical output), 32 GB runs native on MPS. The native-vs-cap-vs-floor decision lives in the pure helper `invisible_engine._target_size` — keep the logic there, unit-tested without the model. For large inputs that OOM, `--tile` is the **lossless** alternative to `--max-resolution`: sliding-window diffusion at native resolution, each tile near SDXL's 1024 training size, feather-blended over the overlap (`noai/tiling.py`). It only engages when the long side exceeds `--tile-size`; the geometry (`plan_tiles`) and the blend window (`feather_weights`) are pure and unit-tested (`tests/test_tiling.py`). Caveat: each tile is an independent low-strength regeneration, so at the certified removal strengths (0.20-0.30) tile drift is minimal but not zero; tiling is a memory workaround, not a quality upgrade over a single native pass. -- fp16 VAE black-output (issues #29/#41): the fp16-fixed SDXL VAE (`madebyollin/sdxl-vae-fp16-fix`) is swapped in for the default SDXL checkpoint on cuda/xpu fp16, plus a model-agnostic backstop that detects a degenerate (all-black) fp16 output and re-runs once in fp32. cpu/mps run fp32 and never reproduce the bug. -- Pyright first run is slow (2-3 min) due to ML deps (torch/diffusers/transformers stubs); full-project `uv run pyright` can stall for many minutes — scope it to changed files. -- A third-party PIL plugin autoload (e.g. an HEIF/AVIF plugin) can raise a non-OSError (`ModuleNotFoundError`), not `UnidentifiedImageError`, when opening a file. Code that opens user-supplied or unknown-format files should `except Exception`, not just `OSError`/`UnidentifiedImageError`. -- rich was dropped: the CLI + analysis scripts print plain text (`click.echo` / the `scripts/_plain_console.py` shim). `rich` is NOT a dependency — importing it breaks the core+dev CI sync; new scripts must use the shim. No Unicode glyphs / colors / progress bars in CLI output by design. -- HEIC/AVIF are decodable on BOTH paths now: the pixel/removal path via the `image_io.imread` Pillow fallback (+ core `pillow-heif`), and metadata detection via a plugin-free binary scan. C2PA removal in those containers (and MP4/MOV/M4V) is `noai/isobmff.py`; JPEG-XL stays metadata/strip-only (Pillow can't decode it without `pillow-jxl`, not a dep). Non-ISOBMFF audio/video (WebM/MP3/WAV/FLAC/OGG) strips losslessly via ffmpeg on PATH. On the ISOBMFF path `remove_ai_metadata` routes to the container branch and never runs the JPEG `_scrub_ai_exif`, so `isobmff.blank_ai_exif_tokens` is the ONLY EXIF scrubber there and must stay in PARITY with it: it blanks **in place** (same-length space overwrite, piexif-validated so a coincidental II/MM run in pixels is ignored — no `iinf`/`iloc` surgery, mirrors `blank_ai_xmp_packets`) an AI-generator token in `Software`/`Make`/`Artist`/`ImageDescription`, the China TC260 `{"AIGC":{...}}` block in `ImageDescription`/`UserComment` (via `_is_aigc_exif_value`), AND the xAI/Grok `Signature:` + UUID-`Artist` pair — leaving camera/editor EXIF intact. Still NOT built: Resemble PerTh audio detection (no presence/confidence flag exists). -- **SynthID technical reference: `docs/synthid.md`** — primary-source-cited doc covering mechanism (post-hoc encoder/decoder pair, 136-bit payload at 512x512, pixel-space, model weights NOT modified), robustness numbers (arXiv:2510.09263: ~99.98% TPR@0.1%FPR across 30 transforms including JPEG/crop/resize/color/noise), removal attacks and forensic detectability (arXiv:2605.09203: all 6 attacks detectable at >98% TPR@1%FPR), detectability limits (no public decoder, metadata-proxy only), oracle scope, and adoption landscape. Read that doc first before adding notes here. -- **SynthID detection is metadata-only.** No local pixel detector is possible by design (Google's decoder is proprietary, trusted-testers only); we read the C2PA companion proxy, which goes quiet once metadata is stripped — a quiet proxy is not proof the pixel watermark is gone. Each vendor has its OWN oracle and it detects only that vendor's content: the Gemini app "Verify with SynthID" for Google, `openai.com/verify` for OpenAI. **Validate the OpenAI arm FIRST** — `openai.com/verify` is more accessible (fewer per-check restrictions) and the strongest automation candidate (Playwright / Chrome MCP); the Gemini flow is more manual. Ordering/throughput choice, not a substitution (see `docs/synthid.md`). SynthID survives JPEG re-encode, so GitHub issue attachments remain valid pixel-watermark test subjects. Every spectral/phase detection approach evaluated (reverse-SynthID, our own probes) works only on controlled solid fills, never on real content. -- **External AI-vs-real classifier models are out of scope** (decided 2026-05-24): per-generator, degrade off-distribution, and our own light SDXL pass would likely defeat them. Detection stays local + signal-based. -- **Default strength is VENDOR-ADAPTIVE, one ladder for BOTH pipelines** (since 2026-06-09): `resolve_strength(strength, vendor)` picks OpenAI **0.10** / Gemini **0.15** / unknown **0.15** when `--strength` is unset (the 2026-06-14 lowering from the 2026-06-04 cert floors of 0.20/0.30 — the single source of truth is `watermark_profiles.py`, and the full cert/lowering history is in `docs/known-limitations.md`); explicit `--strength` always wins. Removal at low strength is content x pipeline dependent, and near-threshold removal is SEED-NON-DETERMINISTIC — pick a strength with margin and oracle-revalidate per content type. -- **`controlnet` is the default pipeline**; `--pipeline sdxl` is the lighter opt-down. Neither pipeline clears all content at low strength (photoreal survives controlnet, flat graphics survive sdxl — the lever is higher strength). A removal-priority caller MUST oracle-validate strength across content types; prod recipe: controlnet + per-vendor floor + FIXED seed. Forensic-stealth caveat (arXiv:2605.09203): defeating the SynthID verifier is NOT forensic invisibility — removal-processed images are flaggable at >98% TPR@1%FPR. +## Data safety + +Follow [`data/README.md`](data/README.md) for public fixture, calibration, oracle, and evaluation layout. Store each tracked binary once and keep generated evaluation outputs outside the repository. + +## Rules and conventions + +Topic-specific rules live in `.claude/rules/*.md` and are auto-loaded when matching files are touched. + +| File | Covers | +|---|---| +| `development.md` | Command contracts, project gate, typing boundaries, and model-adjacent tests | diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..49a6591 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,35 @@ +# Development + +Read this reference for environment setup, dependency recovery, CI behavior, and fixture policy. The always-loaded invariants remain in [`.claude/rules/development.md`](../.claude/rules/development.md). + +## Local environment + +- Use `uv sync --frozen --extra dev` and add only the feature extras needed for the task. +- Do not use `uv pip install` for development tools. It can re-resolve `uv.lock` outside the compatible ML dependency set. +- A core-only sync removes GPU packages by design. Package imports remain light through lazy exports; only removal paths should require the heavy stack. +- On an unreliable connection, sync the needed `dev` and `gpu` extras and run the lint, type, and test commands directly instead of downloading every optional learned backend. +- Run `uv` from the repository root or it may create a bare environment without the project dependencies. + +The optional TrustMark decoder downloads weights into its installed package directory. After pruning that extra, a leftover weights directory can make availability checks see an empty namespace package. If Pyright reports an unknown `TrustMark` import and `find_spec("trustmark")` returns a loader-less spec, remove that regenerable remnant from the active virtual environment and resync. + +## CI + +`.github/workflows/test.yml` runs Ruff and a cross-platform supported-Python test matrix with core plus development dependencies. GPU and model-running tests skip in that matrix; metadata, identification, visible removal, and the OpenCV eraser remain covered across operating systems. + +Keep `uv.lock` compatible with `uv sync --frozen`. Dependency pull-request checks use GitHub's merge result against current `main`; if `main` moves, merge it locally and rerun the full gate because a newer linter can expose stale directives in later code. + +Release and distribution behavior is canonical in [`release-and-distribution.md`](release-and-distribution.md). + +## Fixture and data policy + +[`../data/README.md`](../data/README.md) is the source of truth: + +- executable provenance fixtures live under `data/fixtures/`; +- minimal controlled detector inputs live under `data/calibration/`; +- canonical provider-oracle originals and their manifests live under `data/synthid/`; +- evaluation-only ground truth lives under `data/evaluations/`; +- runtime detector assets live in the package; unregistered research candidates remain outside the shipped wheel. + +Store each binary once. Point tests and manifests at its canonical path. Keep generated and cleaned outputs outside the repository and retain only reproducible public records allowed by the data policy. + +Use synthetic byte blobs for unsupported format paths and deterministic generated negatives where a real negative fixture is unnecessary. Detection and removal tests must preserve their format-specific invariants. From 08dc078d91e4352da7f497f4f7fe3fd41d82e01f Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Fri, 31 Jul 2026 10:39:13 -0700 Subject: [PATCH 11/12] Release 0.22.0 with composable feature extras --- README.md | 34 +++++- docs/cli.md | 38 ++++++- docs/development.md | 6 +- docs/installation.md | 104 ++++++++++++++---- docs/known-limitations.md | 11 +- docs/module-internals.md | 8 +- docs/python-api.md | 15 +++ docs/release-and-distribution.md | 5 +- docs/supported-signals.md | 5 +- docs/synthid-robust-identity-research.md | 2 +- packaging/conda/recipe.yaml | 13 +-- pyproject.toml | 56 +++++----- src/remove_ai_watermarks/__init__.py | 2 +- src/remove_ai_watermarks/cli.py | 17 +-- src/remove_ai_watermarks/dwt_dct.py | 95 ++++++++++++++++ src/remove_ai_watermarks/identify.py | 11 +- src/remove_ai_watermarks/invisible_engine.py | 4 +- .../invisible_watermark.py | 59 +++++----- .../licenses/invisible-watermark-MIT.txt | 21 ++++ src/remove_ai_watermarks/noai/__init__.py | 2 +- src/remove_ai_watermarks/noai/c2pa.py | 2 +- src/remove_ai_watermarks/noai/constants.py | 2 +- .../noai/watermark_remover.py | 5 +- src/remove_ai_watermarks/upscaler.py | 2 +- tests/test_cli.py | 10 +- tests/test_identify.py | 11 +- tests/test_invisible_engine.py | 6 +- tests/test_invisible_watermark.py | 23 +++- tests/test_noai.py | 4 +- tests/test_packaging.py | 67 +++++++++++ tests/test_platform.py | 4 +- uv.lock | 95 +++++++++++----- 32 files changed, 567 insertions(+), 172 deletions(-) create mode 100644 src/remove_ai_watermarks/dwt_dct.py create mode 100644 src/remove_ai_watermarks/licenses/invisible-watermark-MIT.txt create mode 100644 tests/test_packaging.py diff --git a/README.md b/README.md index 99ffea4..1579c86 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,24 @@ Remove AI provenance marks from images you generated yourself: | Run visible, invisible, and metadata removal | `all` | Recommended | | Process a directory | `batch` | Depends on mode | +## Installation modes + +| Need | Install | +| --- | --- | +| Metadata inspection and stripping | `remove-ai-watermarks` | +| Visible detection and removal | `remove-ai-watermarks[visible]` | +| Torch-free DWT-DCT detection | `remove-ai-watermarks[detect]` | +| Diffusion removal | `remove-ai-watermarks[diffusion]` | +| Every production feature | `remove-ai-watermarks[all]` | + +Lower-level and specialized extras include `pixels`, `heif`, `trustmark`, +`migan`, `lama`, `esrgan`, and `qwen-zimage`. The +[installation guide](docs/installation.md#feature-extras) documents their exact +dependency composition and model requirements. + ## Quick start -Install the core CLI: +Install the metadata-focused default CLI: ```bash uv tool install remove-ai-watermarks @@ -46,7 +61,13 @@ Inspect an image: remove-ai-watermarks identify image.png ``` -Remove a known visible mark and AI metadata: +For visible watermark removal, install the pixel dependencies: + +```bash +uv tool install --force "remove-ai-watermarks[visible]" +``` + +Then remove a known visible mark and AI metadata: ```bash remove-ai-watermarks visible image.png -o clean.png @@ -61,7 +82,7 @@ remove-ai-watermarks metadata image.png --remove -o clean.png For invisible watermark removal, install the diffusion dependencies: ```bash -uv tool install --force "remove-ai-watermarks[gpu]" +uv tool install --force "remove-ai-watermarks[diffusion]" remove-ai-watermarks invisible image.png -o clean.png ``` @@ -129,8 +150,9 @@ remove-ai-watermarks erase image.png \ ### Use a learned fill backend -The core install uses OpenCV inpainting when no learned backend is installed. -For more difficult backgrounds: +The `visible` extra uses OpenCV inpainting when no learned backend is installed. +For more difficult backgrounds, the learned-backend extras include the same +pixel dependencies automatically: ```bash uv tool install --force "remove-ai-watermarks[migan]" @@ -197,6 +219,8 @@ See [supported signals](docs/supported-signals.md) and ## Python API +The visible-removal API requires `remove-ai-watermarks[visible]`. + ```python import remove_ai_watermarks as raiw diff --git a/docs/cli.md b/docs/cli.md index da76b8e..a264b31 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -9,15 +9,35 @@ remove-ai-watermarks [OPTIONS] COMMAND [ARGS] Run `remove-ai-watermarks COMMAND --help` for the complete option list and defaults. This page focuses on choosing the right command. +## Command dependency map + +| Command or signal | Required installation | +| --- | --- | +| `metadata` and metadata-only `identify` | Default package | +| Visible signals in `identify` | `remove-ai-watermarks[visible]` (`pixels` is the minimal runtime) | +| Open DWT-DCT signals in `identify` | `remove-ai-watermarks[detect]` | +| Adobe TrustMark signals in `identify` | `remove-ai-watermarks[trustmark]` | +| `visible` and `erase` with OpenCV | `remove-ai-watermarks[visible]` (`pixels` is the minimal runtime) | +| `visible` or `erase` with MI-GAN | `remove-ai-watermarks[migan]` | +| `visible` or `erase` with big-LaMa | `remove-ai-watermarks[lama]` | +| `invisible` | `remove-ai-watermarks[diffusion]` | +| `invisible --pipeline qwen-zimage` | `remove-ai-watermarks[qwen-zimage]` | +| HEIC/HEIF/AVIF pixel input | Add `remove-ai-watermarks[heif]` | +| Every production command and backend | `remove-ai-watermarks[all]` | + +`batch` requires the same extra as its selected mode. Extras can be combined in +one installation, for example `remove-ai-watermarks[visible,detect,heif]`. + ## Inspect an image ```bash remove-ai-watermarks identify image.png ``` -`identify` combines supported metadata and pixel signals into one provenance -report. When no signal is found, it reports the origin as unknown. It does not -claim the image is clean. +`identify` always inspects supported metadata. When pixel extras are installed, +it also evaluates supported visible and invisible pixel signals. When no signal +is found, it reports the origin as unknown. It does not claim the image is +clean. Machine readable output: @@ -36,6 +56,8 @@ invisible pixel detectors. Metadata inspection still runs. ## Remove known visible marks +Install `remove-ai-watermarks[visible]` before using `visible` or `erase`. + ```bash remove-ai-watermarks visible image.png -o clean.png ``` @@ -130,7 +152,7 @@ non-ISOBMFF audio and video path. Install the diffusion dependencies first: ```bash -uv tool install --force "remove-ai-watermarks[gpu]" +uv tool install --force "remove-ai-watermarks[diffusion]" ``` Then run: @@ -194,6 +216,12 @@ It is a memory strategy, not a guarantee of better quality. ## Run the full pipeline +The `all` command and the `all` installation extra are separate concepts. The +command runs every applicable stage. Installing `remove-ai-watermarks[all]` +makes every production backend available; a smaller installation such as +`remove-ai-watermarks[visible,diffusion]` can also run the command with fewer +optional backends. + ```bash remove-ai-watermarks all image.png -o clean.png ``` @@ -206,7 +234,7 @@ The command runs: The visible options and diffusion options are also available on `all`. -If diffusion is required but the `gpu` extra is unavailable, `all` still +If diffusion is required but the `diffusion` extra is unavailable, `all` still writes the result of the visible and metadata stages, prints a prominent warning, and exits with code 1. This prevents a partial result from being reported as complete. diff --git a/docs/development.md b/docs/development.md index 49a6591..e951ca3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,15 +6,15 @@ Read this reference for environment setup, dependency recovery, CI behavior, and - Use `uv sync --frozen --extra dev` and add only the feature extras needed for the task. - Do not use `uv pip install` for development tools. It can re-resolve `uv.lock` outside the compatible ML dependency set. -- A core-only sync removes GPU packages by design. Package imports remain light through lazy exports; only removal paths should require the heavy stack. -- On an unreliable connection, sync the needed `dev` and `gpu` extras and run the lint, type, and test commands directly instead of downloading every optional learned backend. +- A default-only sync removes every pixel and model package by design. Package imports remain light through lazy exports. +- On an unreliable connection, sync `dev` plus only the required feature extras, such as `diffusion`, and run the checks directly instead of downloading every optional learned backend. - Run `uv` from the repository root or it may create a bare environment without the project dependencies. The optional TrustMark decoder downloads weights into its installed package directory. After pruning that extra, a leftover weights directory can make availability checks see an empty namespace package. If Pyright reports an unknown `TrustMark` import and `find_spec("trustmark")` returns a loader-less spec, remove that regenerable remnant from the active virtual environment and resync. ## CI -`.github/workflows/test.yml` runs Ruff and a cross-platform supported-Python test matrix with core plus development dependencies. GPU and model-running tests skip in that matrix; metadata, identification, visible removal, and the OpenCV eraser remain covered across operating systems. +`.github/workflows/test.yml` runs Ruff and a cross-platform supported-Python test matrix with default plus development dependencies. Diffusion and model-running tests skip in that matrix; metadata, identification, visible removal, the DWT-DCT decoder, and the OpenCV eraser remain covered across operating systems. Keep `uv.lock` compatible with `uv sync --frozen`. Dependency pull-request checks use GitHub's merge result against current `main`; if `main` moves, merge it locally and rerun the full gate because a newer linter can expose stale directives in later code. diff --git a/docs/installation.md b/docs/installation.md index 4f7d2b2..1a21ba9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,15 +2,17 @@ Python 3.10.1 or newer is required. -## Core install +## Default metadata mode -The core package provides: +The default package provides: - provenance inspection; -- visible watermark removal with OpenCV; -- manual region erasing with OpenCV; - AI metadata inspection and removal. +It installs Pillow, piexif, and c2pa-python for reading metadata directly from +files. It does not install NumPy, OpenCV, pillow-heif, Torch, diffusion models, +or invisible-watermark decoders. + Install it as an isolated command with uv: ```bash @@ -29,12 +31,27 @@ You can also install the Homebrew package on macOS or Linux: brew install wiltodelta/tap/remove-ai-watermarks ``` -## Invisible watermark removal +## Visible watermark removal -Diffusion based removal needs the `gpu` extra: +Visible mark detection, OpenCV inpainting, and manual region erasing need the +`visible` extra: ```bash -uv tool install --force "remove-ai-watermarks[gpu]" +uv tool install --force "remove-ai-watermarks[visible]" +``` + +Add `heif` only when the pixel path must decode HEIC, HEIF, or AVIF: + +```bash +uv tool install --force "remove-ai-watermarks[visible,heif]" +``` + +## Invisible watermark removal + +Diffusion based removal needs the `diffusion` extra: + +```bash +uv tool install --force "remove-ai-watermarks[diffusion]" ``` The code supports CUDA, XPU, MPS, and CPU devices. A GPU is recommended because @@ -46,28 +63,73 @@ For the CUDA only Qwen Image plus Z-Image profile: uv tool install --force "remove-ai-watermarks[qwen-zimage]" ``` -The `qwen-zimage` extra includes the normal `gpu` dependencies. +The `qwen-zimage` extra includes the normal `diffusion` dependencies. -## Optional features +## Feature extras -Install only what you need: +Extras are composable. Install only the capabilities and file formats the +application actually uses: -| Extra | Adds | -| --- | --- | -| `migan` | MI-GAN ONNX fill backend | -| `lama` | big-LaMa ONNX fill backend | -| `detect` | Open DWT-DCT watermark decoder used by `identify` | -| `trustmark` | Adobe TrustMark decoder | -| `esrgan` | Real-ESRGAN upscaling before diffusion | -| `qwen-zimage` | CUDA only Qwen Image plus Z-Image pipeline | +| Extra | Capability | Automatically includes | Torch or model download | +| --- | --- | --- | --- | +| `pixels` | Shared BGR array and image-processing runtime | NumPy, headless OpenCV | No | +| `heif` | HEIC, HEIF, and AVIF pixel decoding | pillow-heif | No | +| `visible` | Visible mark detection, OpenCV inpainting, and manual erasing | `pixels` | No | +| `detect` | Open DWT-DCT detection for Stable Diffusion, SDXL, and FLUX | `pixels`, PyWavelets | No | +| `trustmark` | Adobe TrustMark detection | trustmark | Yes | +| `diffusion` | Diffusion-based invisible watermark removal | `pixels`, Torch, Diffusers | Yes | +| `migan` | MI-GAN ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch | +| `lama` | big-LaMa ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch | +| `esrgan` | Real-ESRGAN upscaling before diffusion | `pixels`, spandrel | Yes | +| `qwen-zimage` | CUDA-only Qwen Image plus Z-Image pipeline | `diffusion`, DiffSynth | Yes | +| `all` | Every production feature | All rows above | Yes | +| `dev` | Tests, linting, typing, and upstream parity checks | `visible`, `detect`, upstream invisible-watermark | Yes, for parity tests | -Example: +Dependency composition: + +```mermaid +flowchart LR + visible --> pixels + detect --> pixels + diffusion --> pixels + migan --> visible + lama --> visible + esrgan --> pixels + qwen["qwen-zimage"] --> diffusion + heif + trustmark +``` + +`heif` and `trustmark` are independent branches. Combine them explicitly with +another feature when required. The `all` bundle contains every production +branch but never includes `dev`. + +Examples: ```bash +# Metadata plus torch-free DWT-DCT detection +uv tool install --force "remove-ai-watermarks[detect]" + +# Visible removal with HEIC/AVIF support and MI-GAN +uv tool install --force "remove-ai-watermarks[migan,heif]" + +# DWT-DCT and TrustMark detection without diffusion removal +uv tool install --force "remove-ai-watermarks[detect,trustmark]" + +# Every production capability +uv tool install --force "remove-ai-watermarks[all]" + +# An arbitrary minimal combination uv tool install --force "remove-ai-watermarks[migan,detect]" ``` -Some optional models download their weights on first use. +`heif` stays independent so applications that only process PNG, JPEG, or WebP +do not install libheif. `detect` uses the in-tree torch-free decoder and does +not install the upstream `invisible-watermark` package. Optional models download +their weights on first use. + +The old `gpu` and `remove` aliases are intentionally not provided. Use +`diffusion` and `visible` respectively. ## Install from the repository @@ -81,7 +143,7 @@ Add the feature groups required for your work: ```bash uv sync --frozen --extra dev -uv sync --frozen --extra dev --extra gpu +uv sync --frozen --extra dev --extra diffusion ``` Run commands from the repository root: diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 72c9e7d..59f26fa 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -11,7 +11,8 @@ superseded experiments live in the research archive listed in Visible removal changes only the selected mask, but the hidden pixels still have to be reconstructed. -- OpenCV is fast and dependency free. It works well on flat backgrounds but +- OpenCV is fast and requires no model download. It works well on flat + backgrounds but can smear texture or repeated structure. - MI-GAN is a lighter learned backend. It can improve natural texture but may ghost or invent structure. @@ -162,11 +163,13 @@ The metadata path recognizes JPEG XL containers, but the visible and diffusion image paths do not list `.jxl` as a supported pixel format because the package does not include a JPEG XL pixel decoder. -### HEIC, HEIF, and AVIF use a Pillow fallback +### HEIC, HEIF, and AVIF pixel decoding uses an optional Pillow fallback OpenCV does not decode these formats in the project. `image_io.imread` falls -back to Pillow with `pillow-heif`. A corrupt or truncated file may still fail to -decode. +back to Pillow with `pillow-heif` when the `heif` extra is installed alongside +a pixel feature. The +default metadata path scans these containers without that plugin. A corrupt or +truncated file may still fail to decode. ### Some metadata removal requires ffmpeg diff --git a/docs/module-internals.md b/docs/module-internals.md index 9750e29..5d371e5 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -144,6 +144,11 @@ metadata extraction from verdict logic: - `identify` preserves the path-based API and adds the optional registered visible-mark and open invisible-watermark decoders after extraction. +The `detect` extra composes the shared `pixels` runtime with PyWavelets. Its +in-tree [`dwt_dct.py`](../src/remove_ai_watermarks/dwt_dct.py) decoder preserves +the upstream matrix algorithm without installing Torch or non-headless OpenCV. +The upstream MIT notice ships inside the wheel under `licenses/`. + `is_ai_generated` is `True` or `None`; absence of evidence is not reported as a human-made verdict. `ai_source_kind` distinguishes fully generated content from AI-enhanced composites when the source metadata provides that distinction. @@ -379,7 +384,8 @@ Contracts: - `to_bgr` normalizes grayscale and alpha-bearing arrays. - `read_bgr_and_alpha` and `write_bgr_with_alpha` preserve the alpha plane. - `imwrite` returns a success flag; every caller must check it. -- HEIC, HEIF, and AVIF fall back to Pillow plus `pillow-heif`. +- HEIC, HEIF, and AVIF pixel reads fall back to Pillow plus `pillow-heif` from + the independent `heif` extra. Metadata scanning does not require that plugin. - A visible no-op can preserve the original file bytes. Regression coverage: diff --git a/docs/python-api.md b/docs/python-api.md index 5cf04fc..19bed61 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -3,8 +3,17 @@ Use the high level API for normal application integration. Low level detector and pipeline modules are intended for maintainers and specialized workflows. +Dependency groups are identical for the CLI and Python API. The default install +covers metadata extraction, normalization, verdict logic, and stripping. +Array/pixel APIs use `pixels`; visible removal uses `visible`; DWT-DCT detection +uses `detect`; and diffusion removal uses `diffusion`. Add `heif` independently +when path-based pixel APIs must decode HEIC, HEIF, or AVIF. See the complete +[feature-extra matrix](installation.md#feature-extras). + ## Remove visible marks +Install `remove-ai-watermarks[visible]` before using the visible-removal API. + ```python import remove_ai_watermarks as raiw @@ -68,6 +77,9 @@ result, removed = raiw.remove_visible(image, backend="cv2") ## Inspect provenance +The default installation evaluates file metadata. Add `visible`, `detect`, or +`trustmark` to enable the corresponding optional pixel signals. + Get the vendor keys used by visible removal: ```python @@ -172,6 +184,9 @@ as proof that metadata was removed. ## Remove invisible watermarks +Install `remove-ai-watermarks[diffusion]` for the standard pipelines or +`remove-ai-watermarks[qwen-zimage]` for the CUDA-only high-fidelity profile. + ```python from pathlib import Path diff --git a/docs/release-and-distribution.md b/docs/release-and-distribution.md index 1c98000..4265603 100644 --- a/docs/release-and-distribution.md +++ b/docs/release-and-distribution.md @@ -55,8 +55,9 @@ manual Homebrew formula update is the fallback when its automation is blocked. The conda job uses the published artifact rather than a locally built archive as the hash source and commits the resulting recipe change to `main`. Runtime -dependency mapping remains review-controlled: keep it aligned with the core -dependencies in `pyproject.toml`, and document any conda-forge package that is +dependency mapping remains review-controlled: keep it aligned with the default +metadata dependencies in `pyproject.toml`, do not copy optional pixel extras +into the default recipe, and document any conda-forge package that is unavailable and must be omitted. ## Source distribution boundary diff --git a/docs/supported-signals.md b/docs/supported-signals.md index d893e61..7fb5f2b 100644 --- a/docs/supported-signals.md +++ b/docs/supported-signals.md @@ -33,7 +33,7 @@ when you can select the affected area yourself. | Backend | Install | Behavior | | --- | --- | --- | -| `cv2` | Core package | Classical OpenCV inpainting | +| `cv2` | `remove-ai-watermarks[visible]` | Classical OpenCV inpainting | | `migan` | `remove-ai-watermarks[migan]` | MI-GAN through ONNX Runtime | | `lama` | `remove-ai-watermarks[lama]` | big-LaMa through ONNX Runtime | | `auto` | Depends on installed extras | Selects LaMa, then MI-GAN, then OpenCV | @@ -69,6 +69,9 @@ Pixel based image commands discover these extensions: - HEIC and HEIF; - AVIF. +HEIC, HEIF, and AVIF pixel decoding requires the independent `heif` extra in +addition to the selected pixel feature. Metadata scanning does not. + Metadata inspection and removal additionally have container paths for: - JPEG XL metadata; diff --git a/docs/synthid-robust-identity-research.md b/docs/synthid-robust-identity-research.md index b49efa9..3877a63 100644 --- a/docs/synthid-robust-identity-research.md +++ b/docs/synthid-robust-identity-research.md @@ -224,7 +224,7 @@ from the test set + this doc). ## 6. Integration cost (rough) -- New deps: `diffusers` already in the gpu extra; PhotoMaker ships as a `.bin` +- New deps: `diffusers` already in the diffusion extra; PhotoMaker ships as a `.bin` loaded via `pipeline.load_photomaker_adapter(...)`. The OpenCLIP encoder is the same one diffusers already pulls. No new heavy pip dep. - Weight download: PhotoMaker-V1 weights are ~3 GB. Add to the Modal HF volume diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index f68c5f9..89c61bb 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -25,13 +25,10 @@ requirements: run: - python >=${{ python_min }} - pillow >=10.0.0 - - pillow-heif >=0.13.0 - piexif >=1.1.3 - - numpy >=1.24.0 - - py-opencv >=4.8.0 - click >=8.0.0 - python-dotenv >=1.0.0 - # c2pa-python is a core PyPI dependency but is not packaged on conda-forge. + # c2pa-python is a default PyPI dependency but is not packaged on conda-forge. # The guarded import falls back to the built-in C2PA byte scanner when it is # absent. Add it here once a c2pa-python feedstock exists. @@ -52,11 +49,9 @@ about: homepage: https://github.com/wiltodelta/remove-ai-watermarks summary: Remove visible and invisible AI watermarks from images description: | - Detect and remove registered visible AI-provenance marks and strip - AI-provenance metadata (C2PA, EXIF, IPTC, and PNG text chunks) from images. - The core package covers the identify, metadata, visible, and erase command - surface. Optional pip extras add SynthID diffusion removal and additional - invisible-watermark detectors. + Inspect and strip AI-provenance metadata (C2PA, EXIF, IPTC, and PNG text + chunks) from images. Optional pip extras add visible watermark removal, + SynthID diffusion removal, and additional invisible-watermark detectors. license: Apache-2.0 license_file: LICENSE repository: https://github.com/wiltodelta/remove-ai-watermarks diff --git a/pyproject.toml b/pyproject.toml index 8242281..81e0175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.21.2" +version = "0.22.0" description = "AI watermark remover: strip visible and invisible AI watermarks (Gemini / Nano Banana sparkle, SynthID) and provenance metadata (C2PA, EXIF) from images" readme = "README.md" requires-python = ">=3.10.1" @@ -46,15 +46,7 @@ classifiers = [ ] dependencies = [ "pillow>=10.0.0", - # HEIC/AVIF pixel decode for the removal path (iPhone photos, modern exports): - # OpenCV cannot decode these containers, so image_io.imread falls back to Pillow - # and pillow-heif (bundled libheif, prebuilt wheels) registers the HEIF+AVIF - # openers. The metadata path already handles them via a plugin-free binary scan; - # this closes the same gap for the pixel path so `visible`/`all` work on them. - "pillow-heif>=0.13.0", "piexif>=1.1.3", - "numpy>=1.24.0", - "opencv-python-headless>=4.8.0", "click>=8.0.0", "python-dotenv>=1.0.0", # Official C2PA reader (Content Authenticity Initiative, MIT/Apache-2.0). The @@ -67,7 +59,25 @@ dependencies = [ ] [project.optional-dependencies] -gpu = [ +pixels = [ + "numpy>=1.24.0", + "opencv-python-headless>=4.8.0", +] +# Optional HEIC/AVIF pixel decode. Metadata scanning handles these containers +# without this plugin; combine `heif` with any pixel feature only when needed. +heif = [ + "pillow-heif>=0.13.0", +] +visible = ["remove-ai-watermarks[pixels]"] +# Open DWT-DCT watermarks used by Stable Diffusion / SDXL / FLUX. The in-tree +# decoder avoids the upstream invisible-watermark package's mandatory torch and +# non-headless OpenCV dependencies. +detect = [ + "remove-ai-watermarks[pixels]", + "PyWavelets>=1.1.1", +] +diffusion = [ + "remove-ai-watermarks[pixels]", "torch>=2.0.0", # The default PyPI torch wheel is a CPU/CUDA build. To drive an Intel GPU # (Arc / Data Center) via ``--device xpu`` you need an XPU-enabled torch @@ -75,7 +85,7 @@ gpu = [ # XPU build). Install that build first, then this extra (torch is then # already satisfied and won't be re-pulled): # pip install torch --index-url https://download.pytorch.org/whl/xpu - # pip install 'remove-ai-watermarks[gpu]' + # pip install 'remove-ai-watermarks[diffusion]' # uv users can target the ``pytorch-xpu`` index declared under [tool.uv]: # uv pip install torch --index-url https://download.pytorch.org/whl/xpu "diffusers>=0.38.0", @@ -95,23 +105,13 @@ gpu = [ ] # Full two-stage high-fidelity profile: Qwen-Image-2512 Lightning + DiffSynth # Canny ControlNet for the frame, then SAM-masked Z-Image Turbo face repair. -# CUDA-only and intentionally separate from the normal gpu extra because the +# CUDA-only and intentionally separate from the normal diffusion extra because the # additional model stack and DiffSynth runtime are large. qwen-zimage = [ - "remove-ai-watermarks[gpu]", + "remove-ai-watermarks[diffusion]", "diffsynth>=2.0.17,<3", "torchvision>=0.20.0", ] -# Open invisible-watermark (imwatermark) decoder for detecting the DWT-DCT -# watermarks embedded by Stable Diffusion / SDXL / FLUX. Optional because it -# pulls non-headless opencv AND torch (invisible-watermark declares torch a hard -# dependency, and WatermarkDecoder eagerly imports rivaGan -> torch at import -# time, so the dwtDct-only detect path still needs torch present even though it -# never runs on GPU). So `detect` alone pulls torch -- no need to add `gpu` for -# detection. identify() guards the import and skips the signal when absent. -detect = [ - "invisible-watermark>=0.2.0", -] # Adobe TrustMark decoder -- the open, keyless watermark behind Adobe Durable # Content Credentials (soft-binding alg ``com.adobe.trustmark.P``). Optional # because it pulls torch and downloads model weights on first use. identify() @@ -124,6 +124,7 @@ trustmark = [ # cached by huggingface_hub; it is never bundled in this repo. The default cv2 # eraser backend needs none of this. lama = [ + "remove-ai-watermarks[visible]", "onnxruntime>=1.16.0", "huggingface-hub>=0.20.0", ] @@ -133,6 +134,7 @@ lama = [ # memory-tight learned tier (vs big-LaMa's ~4.7 GB). Select it explicitly when # LaMa, the quality-first `auto` choice, is too large. Same runtime as `lama`. migan = [ + "remove-ai-watermarks[visible]", "onnxruntime>=1.16.0", "huggingface-hub>=0.20.0", ] @@ -146,12 +148,16 @@ migan = [ # weights are fetched with torch.hub (bundled with spandrel's torch), so no extra # download dependency is needed. esrgan = [ + "remove-ai-watermarks[pixels]", "spandrel>=0.3.0", ] dev = [ + "remove-ai-watermarks[visible]", + "remove-ai-watermarks[detect]", "pytest>=8.0.0", "pytest-cov>=4.1.0", "pytest-xdist>=3.5.0", + "packaging>=24.0", "ruff>=0.4.0", "pyright>=1.1.0", "invisible-watermark>=0.2.0", @@ -160,11 +166,11 @@ dev = [ "uv-outdated>=0.1.0; python_version >= '3.12'", "uv-secure>=0.12.0; python_version >= '3.12'", ] -all = ["remove-ai-watermarks[gpu,detect,trustmark,lama,migan,dev]"] +all = ["remove-ai-watermarks[visible,heif,detect,trustmark,diffusion,qwen-zimage,lama,migan,esrgan]"] # PyTorch Intel-GPU (XPU) wheel index. ``explicit = true`` keeps it inert for # the default CPU/CUDA install: uv consults it only when a torch install -# explicitly targets it (see the ``gpu`` extra comment), so it does not alter +# explicitly targets it (see the ``diffusion`` extra comment), so it does not alter # the locked CPU/CUDA resolution. Linux/Windows only -- no macOS XPU build. [[tool.uv.index]] name = "pytorch-xpu" diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index a96a921..a724efe 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -25,7 +25,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") _warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*") -__version__ = "0.21.2" +__version__ = "0.22.0" __all__ = ["__version__", "remove_visible", "visible_provenance"] diff --git a/src/remove_ai_watermarks/cli.py b/src/remove_ai_watermarks/cli.py index 3f8676c..a680640 100644 --- a/src/remove_ai_watermarks/cli.py +++ b/src/remove_ai_watermarks/cli.py @@ -183,7 +183,7 @@ _upscaler_option = click.option( "--upscaler", type=click.Choice(["lanczos", "esrgan"]), default="lanczos", - help="How to upscale a small input to the --min-resolution floor: lanczos (default, cv2, no deps) or " + help="How to upscale a small input to the --min-resolution floor: lanczos (default, cv2, no model) or " "esrgan (Real-ESRGAN via the 'esrgan' extra; better detail, slower on CPU). Best for photo/texture " "content -- as a generic GAN with no face/glyph prior it can degrade faces (diffusion mitigates) and " "thin text, so lanczos stays the default. Falls back to lanczos if the extra is absent. Only when upscaling.", @@ -331,7 +331,7 @@ _visible_backend_option = click.option( default="auto", help="Fill backend for visible-mark removal (localize -> fill). auto: best available, " "LaMa > MI-GAN > cv2 (a learned backend needs the 'lama' or 'migan' extra; else cv2, " - "with a warning). cv2: classical inpaint (no deps, smears texture). migan: MI-GAN ONNX " + "with a warning). cv2: classical inpaint (no model download, smears texture). migan: MI-GAN ONNX " "(light, ~1 GB, the memory-tight pick). lama: big-LaMa ONNX (best quality, ~4.7 GB).", ) @@ -800,7 +800,7 @@ def _parse_region(spec: str) -> tuple[int, int, int, int]: "--backend", type=click.Choice(["cv2", "migan", "lama"]), default="cv2", - help="Inpaint backend. cv2: instant, no deps. migan: light ONNX MI-GAN, ~1 GB RAM, " + help="Inpaint backend. cv2: instant, no model download. migan: light ONNX MI-GAN, ~1 GB RAM, " "near-LaMa quality (extra 'migan'). lama: big-LaMa, best quality but ~4.7 GB RAM (extra 'lama').", ) @click.option("--inpaint-method", type=click.Choice(["telea", "ns"]), default="telea", help="cv2 inpaint method.") @@ -941,13 +941,14 @@ def cmd_invisible( """Remove invisible AI watermarks (SynthID, StableSignature, TreeRing). Uses diffusion-based regeneration. Requires GPU for reasonable speed. - Requires the [gpu] extra: pip install 'remove-ai-watermarks[gpu]' + Requires the [diffusion] extra: pip install 'remove-ai-watermarks[diffusion]' """ from remove_ai_watermarks.invisible_engine import is_available as invisible_available if not invisible_available(): console.print( - "Error: GPU dependencies not installed.\n Install them with: pip install 'remove-ai-watermarks[gpu]'" + "Error: Diffusion dependencies not installed.\n" + " Install them with: pip install 'remove-ai-watermarks[diffusion]'" ) raise SystemExit(1) @@ -1298,7 +1299,7 @@ def cmd_all( synthid_skipped = True console.print( " Warning: Skipped - GPU dependencies not installed.\n" - " Install them with: pip install 'remove-ai-watermarks[gpu]'" + " Install them with: pip install 'remove-ai-watermarks[diffusion]'" ) elif _should_skip_invisible_scrub(force, source): # No locally-detectable invisible watermark -> skip the destructive @@ -1404,7 +1405,7 @@ def cmd_all( " visible mark and metadata were stripped.\n" "\n" " Install the extra and rerun to remove it:\n" - " pip install 'remove-ai-watermarks[gpu]'\n" + " pip install 'remove-ai-watermarks[diffusion]'\n" " =====================================================================" ) raise SystemExit(1) @@ -1766,7 +1767,7 @@ def cmd_batch( f"\n WARNING: the invisible (SynthID) watermark was NOT removed on " f"{synthid_skipped_count} image(s) -- the GPU dependencies are not installed, " f"so those outputs still carry the invisible watermark.\n" - f" Install the extra and rerun: pip install 'remove-ai-watermarks[gpu]'" + f" Install the extra and rerun: pip install 'remove-ai-watermarks[diffusion]'" ) # Non-zero exit so a wrapping service detects an incomplete/failed run (batch used diff --git a/src/remove_ai_watermarks/dwt_dct.py b/src/remove_ai_watermarks/dwt_dct.py new file mode 100644 index 0000000..005b4d6 --- /dev/null +++ b/src/remove_ai_watermarks/dwt_dct.py @@ -0,0 +1,95 @@ +"""DWT-DCT decoder compatible with invisible-watermark's ``dwtDct`` path. + +Derived from ShieldMnt/invisible-watermark ``imwatermark/maxDct.py`` (MIT), +trimmed to the matrix path used by Stable Diffusion, SDXL, and FLUX. + +Copyright (c) 2021 ShieldMnt + +The complete upstream license is distributed in +``licenses/invisible-watermark-MIT.txt``. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportMissingTypeStubs=false + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import cv2 +import numpy as np +import pywt + +if TYPE_CHECKING: + from numpy.typing import NDArray + +_DEFAULT_SCALES = (0, 36, 36) +_DEFAULT_BLOCK = 4 + + +class _DecodeMaxDct: + """Extract frequency-domain bits using the upstream matrix algorithm.""" + + def __init__( + self, + wm_lengths: tuple[int, ...], + scales: tuple[int, int, int] = _DEFAULT_SCALES, + block: int = _DEFAULT_BLOCK, + ) -> None: + self._wm_lengths = wm_lengths + self._scales = scales + self._block = block + + def decode(self, bgr: NDArray[Any]) -> dict[int, NDArray[Any]]: + row, col, _channels = bgr.shape + yuv = cv2.cvtColor(bgr, cv2.COLOR_BGR2YUV) + + scores_by_length = {wm_len: [[] for _ in range(wm_len)] for wm_len in self._wm_lengths} + for channel in range(2): + if self._scales[channel] <= 0: + continue + ca1, _detail = pywt.dwt2(yuv[: row // 4 * 4, : col // 4 * 4, channel], "haar") + self._decode_frame(ca1, self._scales[channel], scores_by_length) + + return { + wm_len: np.asarray([float(np.asarray(score).mean()) if score else 0.0 for score in scores]) * 255 > 127 + for wm_len, scores in scores_by_length.items() + } + + def _decode_frame( + self, + frame: NDArray[Any], + scale: int, + scores_by_length: dict[int, list[list[int]]], + ) -> None: + row, col = frame.shape + bit_index = 0 + for i in range(row // self._block): + for j in range(col // self._block): + block = frame[ + i * self._block : i * self._block + self._block, + j * self._block : j * self._block + self._block, + ] + inferred = self._infer_bit(block, scale) + for wm_len, scores in scores_by_length.items(): + scores[bit_index % wm_len].append(inferred) + bit_index += 1 + + def _infer_bit(self, block: NDArray[Any], scale: int) -> int: + position = int(np.argmax(np.abs(block.flatten()[1:]))) + 1 + i, j = position // self._block, position % self._block + value = abs(float(block[i][j])) + return int((value % scale) > 0.5 * scale) + + +def decode_dwt_dct(bgr: NDArray[Any], wm_len: int) -> NDArray[Any]: + """Extract ``wm_len`` watermark bits from a BGR image.""" + return decode_dwt_dct_lengths(bgr, (wm_len,))[wm_len] + + +def decode_dwt_dct_lengths(bgr: NDArray[Any], wm_lengths: tuple[int, ...]) -> dict[int, NDArray[Any]]: + """Extract several watermark lengths with one DWT and block scan.""" + if bgr.size == 0 or min(bgr.shape[:2]) * max(bgr.shape[:2]) < 256 * 256: + raise RuntimeError("image too small, should be larger than 256x256") + if not wm_lengths or any(wm_len <= 0 for wm_len in wm_lengths): + raise ValueError("watermark lengths must be positive") + return _DecodeMaxDct(wm_lengths=tuple(dict.fromkeys(wm_lengths))).decode(bgr) diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index cfb4fdb..8683d90 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -722,8 +722,8 @@ def _visible_text_marks(image_path: Path, *, image: NDArray[Any] | None = None) def _invisible_watermark(image_path: Path) -> str | None: """Open invisible-watermark scheme name (SD/SDXL/FLUX) or None. - Optional: needs the imwatermark decoder (extra ``detect``). Returns None if - it is not installed or no known watermark decodes. + Optional: needs the torch-free DWT-DCT decoder (extra ``detect``). Returns + None if it is not installed or no known watermark decodes. """ from remove_ai_watermarks.invisible_watermark import detect_invisible_watermark @@ -761,6 +761,9 @@ def _collect_visible_signals( image = imread(image_path) except Exception as exc: # cv2 missing - detectors fall back / no-op logger.debug("visible-mark decode unavailable: %s", exc) + return platform + if image is None: + return platform sparkle_conf = _visible_sparkle(image_path, image=image) if sparkle_conf is not None and sparkle_conf >= _SPARKLE_THRESHOLD: @@ -1087,8 +1090,8 @@ def identify( image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container). check_visible: Also run the registered visible-mark detectors through cv2. Set False for a metadata-only, dependency-light scan. - check_invisible: Also decode open invisible watermarks (SD/SDXL/FLUX) via - the optional imwatermark library. No-op when it is not installed. + check_invisible: Also decode optional open invisible watermarks + (SD/SDXL/FLUX). No-op when the decoder extra is not installed. File-backed metadata extraction runs first. The extracted evidence is then evaluated independently, followed by the optional pixel-backed visible and diff --git a/src/remove_ai_watermarks/invisible_engine.py b/src/remove_ai_watermarks/invisible_engine.py index de6ac59..0e38d9e 100644 --- a/src/remove_ai_watermarks/invisible_engine.py +++ b/src/remove_ai_watermarks/invisible_engine.py @@ -4,7 +4,7 @@ Wraps the vendored noai-watermark code for removing invisible AI watermarks (SynthID, StableSignature, TreeRing) via diffusion-based regeneration. This module requires the 'gpu' extra dependencies: - uv pip install 'remove-ai-watermarks[gpu]' + uv pip install 'remove-ai-watermarks[diffusion]' """ # cv2/torch boundary: this engine wraps cv2 (resize/imwrite/cvtColor) and the @@ -226,7 +226,7 @@ class InvisibleEngine: input size, so this is a transparent quality boost; it adds time and memory on small inputs. Ignored on a min > max misconfig. upscaler: How to upscale a small input to the ``min_resolution`` floor: - ``"lanczos"`` (default, cv2, no deps) or ``"esrgan"`` (Real-ESRGAN + ``"lanczos"`` (default, cv2, no model download) or ``"esrgan"`` (Real-ESRGAN via the ``esrgan`` extra). Only applies when UPscaling (the floor case); a ``max_resolution`` downscale always uses Lanczos. Falls back to Lanczos if the extra is absent. diff --git a/src/remove_ai_watermarks/invisible_watermark.py b/src/remove_ai_watermarks/invisible_watermark.py index 45cb57a..bf74538 100644 --- a/src/remove_ai_watermarks/invisible_watermark.py +++ b/src/remove_ai_watermarks/invisible_watermark.py @@ -14,21 +14,20 @@ source: The watermark is fragile: it does NOT survive JPEG re-encoding or resizing (verified -- gone after JPEG q90), so detection works only on pristine PNG -originals. Absence is never proof. Requires the optional ``invisible-watermark`` -package (extra: ``detect``); ``detect_invisible_watermark`` returns None when it -is not installed. +originals. Absence is never proof. Requires the optional ``detect`` extra; +``detect_invisible_watermark`` returns None when it is not installed. """ -# imwatermark ships no type stubs (like cv2); its decoder returns are Unknown. -# Relax the untyped-library diagnostics for this thin wrapper module only. +# The optional numeric libraries do not provide complete types for this path. # pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false from __future__ import annotations import logging -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING if TYPE_CHECKING: + from collections.abc import Iterable from pathlib import Path logger = logging.getLogger(__name__) @@ -49,10 +48,10 @@ _MATCH_SD1_FRAC = 0.92 # fraction of the 136 string bits that must match def is_available() -> bool: - """True if the optional imwatermark decoder is installed.""" + """True when all dependencies for the optional DWT-DCT decoder exist.""" from .optional_deps import module_available - return module_available("imwatermark") + return module_available("cv2", "numpy", "pywt") def _bits_match(value: int, ref: int, width: int = 48) -> int: @@ -68,6 +67,20 @@ def _bytes_match_frac(a: bytes, b: bytes) -> float: return 1.0 - diff / (8 * len(b)) +def _bits_to_int(bits: Iterable[object]) -> int: + value = 0 + for bit in bits: + value = (value << 1) | int(bool(bit)) + return value + + +def _bits_to_bytes(bits: Iterable[object], nbytes: int) -> bytes: + import numpy as np + + packed = np.packbits([int(bool(bit)) for bit in bits]) + return bytes(int(value) for value in packed[:nbytes]) + + def detect_invisible_watermark(image_path: Path) -> str | None: """Return the embedding scheme name if a known open watermark is decoded. @@ -78,32 +91,26 @@ def detect_invisible_watermark(image_path: Path) -> str | None: """ if not is_available(): return None - from imwatermark import WatermarkDecoder - from remove_ai_watermarks import image_io + from remove_ai_watermarks.dwt_dct import decode_dwt_dct_lengths img = image_io.imread(image_path) if img is None: return None - # 48-bit fixed-message watermarks (SDXL, FLUX.2). try: - bits = WatermarkDecoder("bits", 48).decode(img, "dwtDct") - value = 0 - for bit in bits: - value = (value << 1) | (1 if bit else 0) - for name, ref in _BITS_48.items(): - if _bits_match(value, ref) >= _MATCH_48: - return name + decoded = decode_dwt_dct_lengths(img, (48, 8 * len(_SD1_STRING))) except Exception as exc: # decode can fail on tiny images - logger.debug("48-bit watermark decode failed for %s: %s", image_path, exc) + logger.debug("watermark decode failed for %s: %s", image_path, exc) + return None - # 136-bit default string watermark (SD 1.x / 2.x). - try: - raw = cast("bytes", WatermarkDecoder("bytes", 8 * len(_SD1_STRING)).decode(img, "dwtDct")) - if _bytes_match_frac(raw, _SD1_STRING) >= _MATCH_SD1_FRAC: - return "Stable Diffusion 1.x / 2.x" - except Exception as exc: - logger.debug("string watermark decode failed for %s: %s", image_path, exc) + value = _bits_to_int(decoded[48]) + for name, ref in _BITS_48.items(): + if _bits_match(value, ref) >= _MATCH_48: + return name + + raw = _bits_to_bytes(decoded[8 * len(_SD1_STRING)], len(_SD1_STRING)) + if _bytes_match_frac(raw, _SD1_STRING) >= _MATCH_SD1_FRAC: + return "Stable Diffusion 1.x / 2.x" return None diff --git a/src/remove_ai_watermarks/licenses/invisible-watermark-MIT.txt b/src/remove_ai_watermarks/licenses/invisible-watermark-MIT.txt new file mode 100644 index 0000000..e09163b --- /dev/null +++ b/src/remove_ai_watermarks/licenses/invisible-watermark-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 ShieldMnt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/remove_ai_watermarks/noai/__init__.py b/src/remove_ai_watermarks/noai/__init__.py index b8e241e..399ebef 100644 --- a/src/remove_ai_watermarks/noai/__init__.py +++ b/src/remove_ai_watermarks/noai/__init__.py @@ -7,7 +7,7 @@ is exposed **lazily** via PEP 562 ``__getattr__``: importing a light submodule (e.g. ``noai.c2pa`` / ``noai.constants`` from ``identify``) must NOT eagerly pull ``watermark_remover``, which imports torch + diffusers at module top. Keeping this lazy is what lets ``import remove_ai_watermarks.identify`` stay cheap (~36 MB, no -torch) even in a full install where the ``gpu``/``detect`` extras are present -- +torch) even in a full install where the ``diffusion`` extra is present -- otherwise the mere presence of torch in the env inflated identify to ~420 MB and risked OOM on a 512 MB host. """ diff --git a/src/remove_ai_watermarks/noai/c2pa.py b/src/remove_ai_watermarks/noai/c2pa.py index 6005476..81da1de 100644 --- a/src/remove_ai_watermarks/noai/c2pa.py +++ b/src/remove_ai_watermarks/noai/c2pa.py @@ -43,7 +43,7 @@ from remove_ai_watermarks.noai.constants import ( logger = logging.getLogger(__name__) -# Official C2PA reader (c2pa-python, a core dependency). It is the primary, +# Official C2PA reader (c2pa-python, a default dependency). It is the primary, # spec-tracking manifest parser; the hand-rolled caBX/CBOR scanner below stays as # a fallback for synthetic/partial blobs the validator rejects. The import is # guarded so a partially-broken install degrades to the byte-scan rather than diff --git a/src/remove_ai_watermarks/noai/constants.py b/src/remove_ai_watermarks/noai/constants.py index 2431c75..541a77f 100644 --- a/src/remove_ai_watermarks/noai/constants.py +++ b/src/remove_ai_watermarks/noai/constants.py @@ -7,7 +7,7 @@ so adding a new AI tool or metadata key requires updating only this file. from typing import NamedTuple # Supported image formats for the pixel/removal path (CLI input validation + batch -# discovery). PNG/JPEG/WebP decode+encode via cv2; HEIC/HEIF/AVIF via the core +# discovery). PNG/JPEG/WebP decode+encode via cv2; HEIC/HEIF/AVIF via the optional # pillow-heif dep (image_io.imread Pillow fallback + imwrite _pil_write), so batch # now picks them up and the CLI no longer warns on an iPhone HEIC. JPEG-XL is left # out on purpose -- it is metadata/strip-only (no pixel decoder without pillow-jxl). diff --git a/src/remove_ai_watermarks/noai/watermark_remover.py b/src/remove_ai_watermarks/noai/watermark_remover.py index c1ae671..b5c3031 100644 --- a/src/remove_ai_watermarks/noai/watermark_remover.py +++ b/src/remove_ai_watermarks/noai/watermark_remover.py @@ -476,8 +476,9 @@ class WatermarkRemover: """Turn off the diffusers default invisible watermarker on an SDXL pipeline. diffusers embeds an open "Stable Diffusion XL" DWT-DCT invisible watermark on - EVERY SDXL output whenever ``invisible-watermark`` is installed (the ``detect`` - extra). A watermark REMOVER must not re-stamp a detectable AI watermark, or the + EVERY SDXL output whenever ``invisible-watermark`` is installed (kept as a + development parity dependency). A watermark REMOVER must not re-stamp a + detectable AI watermark, or the cleaned output re-reads as AI (``identify`` -> "Open invisible watermark: Stable Diffusion XL"). Shared by both SDXL loaders; the ``ControlNetModel`` sub-model and the Qwen loader never call it (only the pipeline accepts the kwarg). diff --git a/src/remove_ai_watermarks/upscaler.py b/src/remove_ai_watermarks/upscaler.py index 9a386d8..a545f13 100644 --- a/src/remove_ai_watermarks/upscaler.py +++ b/src/remove_ai_watermarks/upscaler.py @@ -4,7 +4,7 @@ Mirrors ``region_eraser``'s optional-backend pattern: ``is_available()`` guards ``spandrel`` import, a lazy singleton (double-checked lock) holds the loaded model, and the weights download on first use (cached by ``torch.hub``) -- they are never bundled. -The DEFAULT upscaler stays Lanczos (cv2, no deps); this is opt-in via the ``esrgan`` +The DEFAULT upscaler stays Lanczos (cv2, no model download); this is opt-in via the ``esrgan`` extra and feeds the ``--upscaler esrgan`` path. ``spandrel`` is a pure model-loader (MIT) with NO basicsr dependency -- it pulls only torch/torchvision/safetensors/numpy/ einops -- so it sidesteps the basicsr / ``torchvision.transforms.functional_tensor`` diff --git a/tests/test_cli.py b/tests/test_cli.py index ed28efa..95ca22d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -542,7 +542,7 @@ class TestAllCommand: result = runner.invoke(main, ["all", str(sample_png), "-o", str(output)]) assert result.exit_code != 0, result.output assert "NOT removed" in result.output - assert "remove-ai-watermarks[gpu]" in result.output + assert "remove-ai-watermarks[diffusion]" in result.output assert output.exists() # visible + metadata still produced a file def test_all_reports_metadata_that_survived_stripping(self, runner, sample_png, tmp_path): @@ -933,21 +933,21 @@ class TestBatchCommand: class TestGpuHintMarkup: - """The GPU-extra install hint must reach the user with the ``[gpu]`` token + """The diffusion install hint must reach the user with the ``[diffusion]`` token intact (plain output prints it verbatim, with no markup parsing).""" def test_invisible_install_hint_keeps_gpu_extra(self, runner, sample_png): with patch("remove_ai_watermarks.invisible_engine.is_available", return_value=False): result = runner.invoke(main, ["invisible", str(sample_png)]) assert result.exit_code != 0 - assert "remove-ai-watermarks[gpu]" in result.output + assert "remove-ai-watermarks[diffusion]" in result.output def test_all_install_hint_keeps_gpu_extra(self, runner, sample_png): # The `all` pipeline skips the invisible step with a warning that carries - # the same hint; it must keep the [gpu] extra too. + # the same hint; it must keep the [diffusion] extra too. with patch("remove_ai_watermarks.invisible_engine.is_available", return_value=False): result = runner.invoke(main, ["all", str(sample_png)]) - assert "remove-ai-watermarks[gpu]" in result.output + assert "remove-ai-watermarks[diffusion]" in result.output class TestEraseCommand: diff --git a/tests/test_identify.py b/tests/test_identify.py index c683d21..fe6171f 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -782,6 +782,15 @@ class TestIdentifyVisibleTextMarks: identify(tmp_clean_png, check_visible=True, check_invisible=False) assert mock_imread.call_count == 1 + def test_missing_pixel_extra_preserves_metadata_verdict(self, tmp_png_with_ai_metadata: Path): + import remove_ai_watermarks.image_io as image_io + + with patch.object(image_io, "imread", side_effect=ModuleNotFoundError("No module named 'cv2'")): + report = identify(tmp_png_with_ai_metadata, check_visible=True, check_invisible=False) + + assert report.is_ai_generated is True + assert report.confidence == "high" + # ── Caveats and serialization ─────────────────────────────────────── @@ -989,7 +998,7 @@ class TestIdentifyC2paDevice: from remove_ai_watermarks.invisible_watermark import is_available as _wm_available # noqa: E402 -@pytest.mark.skipif(not _wm_available(), reason="invisible-watermark not installed") +@pytest.mark.skipif(not _wm_available(), reason="detect extra not installed") class TestIdentifyInvisibleWatermark: def _sdxl_watermarked(self, tmp_path: Path) -> Path: import cv2 diff --git a/tests/test_invisible_engine.py b/tests/test_invisible_engine.py index 94888d2..b44f8ce 100644 --- a/tests/test_invisible_engine.py +++ b/tests/test_invisible_engine.py @@ -18,9 +18,9 @@ class TestIsAvailable: assert isinstance(result, bool) def test_available_reflects_dependencies(self): - """is_available() is True iff torch + diffusers (the gpu extra) import. + """is_available() is True iff torch + diffusers (the diffusion extra) import. - Must not assume the full stack: the core+dev CI env has no diffusers. + Must not assume the full stack: the default+dev CI env has no diffusers. """ import importlib.util @@ -212,7 +212,7 @@ class TestCannyControlImage: def test_edge_map_is_3channel_rgb(self): if not is_available(): - pytest.skip("gpu extra (torch/diffusers) not installed") + pytest.skip("diffusion extra (torch/diffusers) not installed") import numpy as np from remove_ai_watermarks.noai.watermark_remover import WatermarkRemover diff --git a/tests/test_invisible_watermark.py b/tests/test_invisible_watermark.py index 2cf77fb..a516146 100644 --- a/tests/test_invisible_watermark.py +++ b/tests/test_invisible_watermark.py @@ -1,8 +1,7 @@ -"""Tests for open invisible-watermark (imwatermark) detection. +"""Tests for open DWT-DCT watermark detection. -Each known scheme is round-tripped: embed its exact upstream pattern with the -encoder, then assert the detector names it. Skipped entirely if the optional -``invisible-watermark`` package is not installed. +The upstream encoder supplies known watermarks, while the in-tree decoder must +both identify them and match the upstream decoder bit for bit. """ from __future__ import annotations @@ -25,7 +24,7 @@ from remove_ai_watermarks.invisible_watermark import ( is_available, ) -pytestmark = pytest.mark.skipif(not is_available(), reason="invisible-watermark not installed") +pytestmark = pytest.mark.skipif(not is_available(), reason="detect extra not installed") def _base_image() -> np.ndarray: @@ -61,6 +60,20 @@ class TestHelpers: class TestDetect: + def test_in_tree_decoder_matches_upstream(self, tmp_path: Path): + from imwatermark import WatermarkDecoder + + from remove_ai_watermarks.dwt_dct import decode_dwt_dct + from remove_ai_watermarks.image_io import imread + + path = _write_bits_watermark(tmp_path, _BITS_48["Stable Diffusion XL"]) + image = imread(path) + assert image is not None + + upstream = np.asarray(WatermarkDecoder("bits", 48).decode(image, "dwtDct"), dtype=bool) + ours = np.asarray(decode_dwt_dct(image, wm_len=48), dtype=bool) + assert np.array_equal(ours, upstream) + def test_detects_sdxl(self, tmp_path: Path): path = _write_bits_watermark(tmp_path, _BITS_48["Stable Diffusion XL"]) assert detect_invisible_watermark(path) == "Stable Diffusion XL" diff --git a/tests/test_noai.py b/tests/test_noai.py index 25f47cc..f0d707c 100644 --- a/tests/test_noai.py +++ b/tests/test_noai.py @@ -52,8 +52,8 @@ class TestConstants: assert ".jpg" in SUPPORTED_FORMATS def test_supported_formats_include_heic_avif(self): - # HEIC/AVIF are first-class on the pixel path now (read+write via pillow-heif), - # so batch discovers them and the CLI does not warn. + # HEIC/AVIF are first-class when the visible pixel extra is installed + # (read+write via pillow-heif), so batch discovers them without a warning. assert {".heic", ".heif", ".avif"} <= SUPPORTED_FORMATS def test_supported_formats_exclude_jpeg_xl(self): diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..28fd6b8 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,67 @@ +"""Published dependency boundaries.""" + +from __future__ import annotations + +from importlib.metadata import metadata, requires + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + + +def _requirement_names(extra: str | None = None) -> set[str]: + selected_extra = extra or "" + parsed = (Requirement(value) for value in requires("remove-ai-watermarks") or []) + return { + canonicalize_name(requirement.name) + for requirement in parsed + if requirement.marker is None or requirement.marker.evaluate({"extra": selected_extra}) + } + + +def test_default_install_is_metadata_focused(): + default = _requirement_names() + + assert { + "c2pa-python", + "click", + "piexif", + "pillow", + "python-dotenv", + } <= default + assert { + "invisible-watermark", + "numpy", + "opencv-python-headless", + "pillow-heif", + "torch", + "trustmark", + }.isdisjoint(default) + + +def test_pixels_extra_owns_shared_numeric_dependencies(): + assert { + "numpy", + "opencv-python-headless", + } <= _requirement_names("pixels") + + +def test_file_format_and_detector_dependencies_are_independent(): + assert "pillow-heif" in _requirement_names("heif") + assert "pywavelets" in _requirement_names("detect") + + +def test_extras_use_capability_names_without_legacy_aliases(): + extras = set(metadata("remove-ai-watermarks").get_all("Provides-Extra") or []) + + assert {"pixels", "heif", "visible", "detect", "diffusion"} <= extras + assert {"gpu", "remove", "detect-pywavelets"}.isdisjoint(extras) + + +def test_production_all_does_not_include_development_tools(): + assert { + "pyright", + "pytest", + "pytest-cov", + "pytest-xdist", + "ruff", + }.isdisjoint(_requirement_names("all")) diff --git a/tests/test_platform.py b/tests/test_platform.py index 37799ba..2464649 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -238,7 +238,7 @@ class TestQwenKwargs: """_build_qwen_kwargs is pure (no torch); guards the Qwen-Image call shape. watermark_remover imports torch under a try/except, so the module (and this pure - helper) imports fine in the core+dev CI env where torch is absent. + helper) imports fine in the default+dev CI env where torch is absent. """ def test_uses_true_cfg_not_guidance_scale(self): @@ -431,7 +431,7 @@ class TestAvailability: def test_watermark_removal_available(self): # Reflects the actual environment: True iff torch + diffusers (the gpu - # extra) are importable. The core+dev CI env has no diffusers, so this + # extra) are importable. The default+dev CI env has no diffusers, so this # must not assume the full stack is present. import importlib.util diff --git a/uv.lock b/uv.lock index 8366acd..5f299e9 100644 --- a/uv.lock +++ b/uv.lock @@ -3187,78 +3187,100 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.21.2" +version = "0.22.0" source = { editable = "." } dependencies = [ { name = "c2pa-python" }, { name = "click" }, - { name = "numpy" }, - { name = "opencv-python-headless" }, { name = "piexif" }, { name = "pillow" }, - { name = "pillow-heif" }, { name = "python-dotenv" }, ] [package.optional-dependencies] all = [ { name = "accelerate" }, + { name = "diffsynth" }, { name = "diffusers" }, { name = "huggingface-hub" }, - { name = "invisible-watermark" }, + { name = "numpy" }, { name = "onnxruntime", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pyright" }, - { name = "pytest" }, - { name = "pytest-cov" }, - { name = "pytest-xdist" }, - { name = "ruff" }, + { name = "opencv-python-headless" }, + { name = "pillow-heif" }, + { name = "pywavelets", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pywavelets", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "safetensors" }, + { name = "spandrel" }, { name = "tokenizers" }, { name = "torch" }, + { name = "torchvision" }, { name = "transformers" }, { name = "trustmark" }, - { name = "uv-outdated", marker = "python_full_version >= '3.12'" }, - { name = "uv-secure", marker = "python_full_version >= '3.12'" }, ] detect = [ - { name = "invisible-watermark" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "pywavelets", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pywavelets", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] dev = [ { name = "invisible-watermark" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "packaging" }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-xdist" }, + { name = "pywavelets", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pywavelets", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "ruff" }, { name = "uv-outdated", marker = "python_full_version >= '3.12'" }, { name = "uv-secure", marker = "python_full_version >= '3.12'" }, ] -esrgan = [ - { name = "spandrel" }, -] -gpu = [ +diffusion = [ { name = "accelerate" }, { name = "diffusers" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, { name = "safetensors" }, { name = "tokenizers" }, { name = "torch" }, { name = "transformers" }, ] +esrgan = [ + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "spandrel" }, +] +heif = [ + { name = "pillow-heif" }, +] lama = [ { name = "huggingface-hub" }, + { name = "numpy" }, { name = "onnxruntime", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "opencv-python-headless" }, ] migan = [ { name = "huggingface-hub" }, + { name = "numpy" }, { name = "onnxruntime", version = "1.24.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "onnxruntime", version = "1.27.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "opencv-python-headless" }, +] +pixels = [ + { name = "numpy" }, + { name = "opencv-python-headless" }, ] qwen-zimage = [ { name = "accelerate" }, { name = "diffsynth" }, { name = "diffusers" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, { name = "safetensors" }, { name = "tokenizers" }, { name = "torch" }, @@ -3268,44 +3290,57 @@ qwen-zimage = [ trustmark = [ { name = "trustmark" }, ] +visible = [ + { name = "numpy" }, + { name = "opencv-python-headless" }, +] [package.metadata] requires-dist = [ - { name = "accelerate", marker = "extra == 'gpu'", specifier = ">=0.25.0" }, + { name = "accelerate", marker = "extra == 'diffusion'", specifier = ">=0.25.0" }, { name = "c2pa-python", specifier = ">=0.35.0" }, { name = "click", specifier = ">=8.0.0" }, { name = "diffsynth", marker = "extra == 'qwen-zimage'", specifier = ">=2.0.17,<3" }, - { name = "diffusers", marker = "extra == 'gpu'", specifier = ">=0.38.0" }, + { name = "diffusers", marker = "extra == 'diffusion'", specifier = ">=0.38.0" }, { name = "huggingface-hub", marker = "extra == 'lama'", specifier = ">=0.20.0" }, { name = "huggingface-hub", marker = "extra == 'migan'", specifier = ">=0.20.0" }, - { name = "invisible-watermark", marker = "extra == 'detect'", specifier = ">=0.2.0" }, { name = "invisible-watermark", marker = "extra == 'dev'", specifier = ">=0.2.0" }, - { name = "numpy", specifier = ">=1.24.0" }, + { name = "numpy", marker = "extra == 'pixels'", specifier = ">=1.24.0" }, { name = "onnxruntime", marker = "extra == 'lama'", specifier = ">=1.16.0" }, { name = "onnxruntime", marker = "extra == 'migan'", specifier = ">=1.16.0" }, - { name = "opencv-python-headless", specifier = ">=4.8.0" }, + { name = "opencv-python-headless", marker = "extra == 'pixels'", specifier = ">=4.8.0" }, + { name = "packaging", marker = "extra == 'dev'", specifier = ">=24.0" }, { name = "piexif", specifier = ">=1.1.3" }, { name = "pillow", specifier = ">=10.0.0" }, - { name = "pillow-heif", specifier = ">=0.13.0" }, + { name = "pillow-heif", marker = "extra == 'heif'", specifier = ">=0.13.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "remove-ai-watermarks", extras = ["gpu"], marker = "extra == 'qwen-zimage'" }, - { name = "remove-ai-watermarks", extras = ["gpu", "detect", "trustmark", "lama", "migan", "dev"], marker = "extra == 'all'" }, + { name = "pywavelets", marker = "extra == 'detect'", specifier = ">=1.1.1" }, + { name = "remove-ai-watermarks", extras = ["detect"], marker = "extra == 'dev'" }, + { name = "remove-ai-watermarks", extras = ["diffusion"], marker = "extra == 'qwen-zimage'" }, + { name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'detect'" }, + { name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'diffusion'" }, + { name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'esrgan'" }, + { name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'visible'" }, + { name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'dev'" }, + { name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'lama'" }, + { name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'migan'" }, + { name = "remove-ai-watermarks", extras = ["visible", "heif", "detect", "trustmark", "diffusion", "qwen-zimage", "lama", "migan", "esrgan"], marker = "extra == 'all'" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" }, - { name = "safetensors", marker = "extra == 'gpu'" }, + { name = "safetensors", marker = "extra == 'diffusion'" }, { name = "spandrel", marker = "extra == 'esrgan'", specifier = ">=0.3.0" }, - { name = "tokenizers", marker = "extra == 'gpu'", specifier = ">=0.22,<0.23" }, - { name = "torch", marker = "extra == 'gpu'", specifier = ">=2.0.0" }, + { name = "tokenizers", marker = "extra == 'diffusion'", specifier = ">=0.22,<0.23" }, + { name = "torch", marker = "extra == 'diffusion'", specifier = ">=2.0.0" }, { name = "torchvision", marker = "extra == 'qwen-zimage'", specifier = ">=0.20.0" }, - { name = "transformers", marker = "extra == 'gpu'", specifier = ">=5,<6" }, + { name = "transformers", marker = "extra == 'diffusion'", specifier = ">=5,<6" }, { name = "trustmark", marker = "extra == 'trustmark'", specifier = ">=0.8.0" }, { name = "uv-outdated", marker = "python_full_version >= '3.12' and extra == 'dev'", specifier = ">=0.1.0" }, { name = "uv-secure", marker = "python_full_version >= '3.12' and extra == 'dev'", specifier = ">=0.12.0" }, ] -provides-extras = ["gpu", "qwen-zimage", "detect", "trustmark", "lama", "migan", "esrgan", "dev", "all"] +provides-extras = ["pixels", "heif", "visible", "detect", "diffusion", "qwen-zimage", "trustmark", "lama", "migan", "esrgan", "dev", "all"] [[package]] name = "requests" From 63d7cfa0a5787d4c3c3881ed4477ecd9a91efd2e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:43:40 +0000 Subject: [PATCH 12/12] Sync conda recipe with v0.22.0 --- packaging/conda/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index 89c61bb..4aa6c26 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -1,7 +1,7 @@ schema_version: 1 context: - version: "0.21.2" + version: "0.22.0" python_min: "3.10" package: @@ -10,7 +10,7 @@ package: source: url: https://pypi.org/packages/source/r/remove-ai-watermarks/remove_ai_watermarks-${{ version }}.tar.gz - sha256: a013af322d40c137302c500638806c8864d5945699bdc2cdcbdcda3e2f37b3ac + sha256: 00549927083c2b60444dd8404830c7333899a9345aacc41d43b21a77c63b8dd0 build: noarch: python