mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-20 16:07:17 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8bc4b7c68 | ||
|
|
a8e218acf6 | ||
|
|
ad7e4ee08b | ||
|
|
22bc171806 | ||
|
|
d763581ed3 | ||
|
|
0d99f403fb | ||
|
|
e78e5f1154 | ||
|
|
0a77d3198e | ||
|
|
9aea5f240f | ||
|
|
c3ddf8a801 | ||
|
|
5777458296 | ||
|
|
295e7ada2b |
No files matched your search
@@ -0,0 +1,106 @@
|
||||
name: Distribute on release
|
||||
|
||||
# Fans a published GitHub Release out to the channels that need a nudge.
|
||||
# PyPI is handled by publish.yml; conda-forge is handled by its autotick bot.
|
||||
# This workflow event-drives the two channels that would otherwise be manual:
|
||||
# - Homebrew tap: rewrite the formula's url + sha256 to the new sdist.
|
||||
# - HF Space: factory-rebuild so it reinstalls the latest sdist from PyPI.
|
||||
# Both wait for the freshly published sdist to appear on PyPI first, since the
|
||||
# Release event fires in parallel with publish.yml's upload.
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to distribute (no leading v), e.g. 0.10.3. Blank = latest on PyPI."
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
url: ${{ steps.sdist.outputs.url }}
|
||||
sha: ${{ steps.sdist.outputs.sha }}
|
||||
steps:
|
||||
- name: Resolve target version
|
||||
id: v
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "release" ]; then
|
||||
v="${GITHUB_REF_NAME#v}"
|
||||
elif [ -n "${{ github.event.inputs.version }}" ]; then
|
||||
v="${{ github.event.inputs.version }}"
|
||||
else
|
||||
v=$(curl -fsSL https://pypi.org/pypi/remove-ai-watermarks/json \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin)['info']['version'])")
|
||||
fi
|
||||
echo "version=$v" >> "$GITHUB_OUTPUT"
|
||||
echo "Target version: $v"
|
||||
|
||||
- name: Wait for the sdist on PyPI, capture url + sha256
|
||||
id: sdist
|
||||
run: |
|
||||
set -euo pipefail
|
||||
v="${{ steps.v.outputs.version }}"
|
||||
for i in $(seq 1 30); do
|
||||
if rel=$(curl -fsSL "https://pypi.org/pypi/remove-ai-watermarks/$v/json" 2>/dev/null); then
|
||||
url=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
sha=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
if [ -n "$url" ] && [ -n "$sha" ]; then
|
||||
echo "url=$url" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "Found sdist for $v"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "sdist for $v not on PyPI yet (attempt $i), waiting..."
|
||||
sleep 20
|
||||
done
|
||||
echo "::error::sdist for $v did not appear on PyPI in time"
|
||||
exit 1
|
||||
|
||||
homebrew:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout the tap
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: wiltodelta/homebrew-tap
|
||||
token: ${{ secrets.HOMEBREW_TAP_TOKEN }}
|
||||
- name: Bump the formula
|
||||
env:
|
||||
URL: ${{ needs.resolve.outputs.url }}
|
||||
SHA: ${{ needs.resolve.outputs.sha }}
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
F=Formula/remove-ai-watermarks.rb
|
||||
sed -i -E "s|url \"[^\"]*\"|url \"$URL\"|" "$F"
|
||||
sed -i -E "s|sha256 \"[^\"]*\"|sha256 \"$SHA\"|" "$F"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add "$F"
|
||||
git commit -m "remove-ai-watermarks $VERSION" || { echo "Formula already current"; exit 0; }
|
||||
git push
|
||||
|
||||
hf-space:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Factory-rebuild the HuggingFace Space
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: |
|
||||
python3 -m pip install --quiet huggingface_hub
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
from huggingface_hub import HfApi
|
||||
HfApi(token=os.environ["HF_TOKEN"]).restart_space(
|
||||
"wiltodelta/remove-ai-watermarks", factory_reboot=True
|
||||
)
|
||||
print("HF Space factory rebuild triggered")
|
||||
PY
|
||||
+3
-5
@@ -18,9 +18,6 @@ Thumbs.db
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Test results
|
||||
data/results/
|
||||
|
||||
# SynthID corpus reference fills (synthetic black/white calibration tiles,
|
||||
# regenerable; the labeled pos/neg/cleaned images ARE tracked, see README)
|
||||
data/synthid_corpus/refs/
|
||||
@@ -49,6 +46,7 @@ data/gemini_capture/captures/gemini_content_*.png
|
||||
data/samsung_capture/seeds/
|
||||
data/samsung_capture/captures/samsung_content_*
|
||||
|
||||
# GFPGAN downloads its RetinaFace/parsing weights to a CWD ./gfpgan/weights/
|
||||
# working dir on first use (the restore extra). Runtime artifact, never committed.
|
||||
# Leftover GFPGAN weights dir from the retired face-restore experiments
|
||||
# (GFPGAN wrote RetinaFace/parsing weights to a CWD ./gfpgan/weights/ working
|
||||
# dir on first use). Runtime artifact, never committed.
|
||||
gfpgan/
|
||||
@@ -4,7 +4,7 @@ You are a **principal Python engineer** maintaining a CLI tool and library for r
|
||||
|
||||
## How to run
|
||||
|
||||
- `uv run remove-ai-watermarks all <image.png> -o <output.png>` — full pipeline (visible + invisible + metadata). Same diffusion knobs as `invisible` below, plus the visible-pass `--inpaint/--no-inpaint`/`--inpaint-method`.
|
||||
- `uv run remove-ai-watermarks all <image.png> -o <output.png>` — full pipeline (visible + invisible + metadata). Same diffusion knobs as `invisible` below, plus the visible-pass `--inpaint/--no-inpaint`/`--inpaint-method`. **When the `[gpu]` extra is absent, step 2 (invisible/SynthID) is skipped** — `all` still writes an output (visible mark + metadata stripped) but prints a prominent end-of-run banner ("the invisible (SynthID) watermark was NOT removed") AND exits **non-zero** (1), so a skipped SynthID pass is not mistaken for a clean result (the recurring #14/#47 trap, where the old quiet inline warning was missed). `invisible` already hard-errors without the extra; only `all` continued, hence the loud end-banner. Regression-guarded by `tests/test_cli.py::TestAllCommand::test_all_loud_warning_and_nonzero_exit_when_gpu_missing`.
|
||||
- `uv run remove-ai-watermarks invisible <image.png> -o <out.png>` — diffusion SynthID removal. **Full knob set** (kept identical across `invisible`/`all`/`batch`): `--strength` (vendor-adaptive default), `--steps`, `--guidance-scale` (CFG, default 7.5), `--pipeline sdxl|controlnet` (default `controlnet`), `--controlnet-scale`, `--model` (HF model id, default SDXL base), `--device`, `--seed`, `--hf-token`, `--max-resolution`/`--min-resolution`, `--upscaler lanczos|esrgan`, `--humanize` (Analog Humanizer grain), `--unsharp` (final sharpen), and `--adaptive-polish/--no-adaptive-polish` (**ON by default**; detail-targeted polish that self-gates to a no-op where there is no deficit). `--auto` is deprecated and now a no-op that only warns (the polish it used to enable is ON by default).
|
||||
- `uv run remove-ai-watermarks visible <image.png> -o <out.png>` — known-visible-mark removal, CPU, no GPU. Reverse-alpha based: each mark is removed by inverting its captured alpha map. `--mark auto` (default) picks the strongest detected of the Gemini sparkle, the Doubao "豆包AI生成" text strip, the Jimeng "★ 即梦AI" wordmark, and the Samsung Galaxy AI "✦ Contenuti generati dall'AI" strip (bottom-LEFT, locale-specific — Italian variant calibrated); `--mark gemini` / `--mark doubao` / `--mark jimeng` / `--mark samsung` force one (choices come from the registry). Gemini/Doubao recover pixels exactly with no inpaint at native; **Jimeng and Samsung add an always-on thin residual inpaint over the glyph footprint** (their marks re-rasterize per image, so reverse-alpha alone leaves a faint outline). For arbitrary logos/objects use `erase`.
|
||||
- `uv run remove-ai-watermarks erase <image.png> --region x,y,w,h -o <out.png>` — universal region eraser (any logo/object, any position). `--backend cv2` (default, no deps) or `--backend lama` (big-LaMa via onnxruntime, extra `lama`); `--region` is repeatable.
|
||||
@@ -15,11 +15,12 @@ You are a **principal Python engineer** maintaining a CLI tool and library for r
|
||||
|
||||
## Test and lint
|
||||
|
||||
- **CI** (`.github/workflows/test.yml`): runs on push to `main` + every PR. A `lint` job (ubuntu: `ruff check` + `ruff format --check`) plus a `test` matrix (ubuntu/macos/windows x py3.10/3.12) that does `uv sync --frozen --extra dev` then `pytest`. The matrix installs only core + dev (no `gpu` extra), so the GPU/model-running tests skip there and it exercises the metadata/identify/visible/cv2-eraser surface on all three OSes. Keep `uv.lock` valid (don't break `--frozen`) when editing `pyproject.toml`. `publish.yml` stays release-only and now verifies the release tag matches the `pyproject.toml` version (fails the build on a mismatch) before building, then uploads via `uv publish` (PyPI trusted publishing over OIDC, no token — replaced the `pypa/gh-action-pypi-publish` action so the upload no longer depends on that action's bundled twine accepting the Metadata-Version; the `id-token: write` permission + `pypi` environment + workflow filename are unchanged, so PyPI's trusted-publisher entry still matches). **Release flow:** bump the version in `pyproject.toml` + `src/remove_ai_watermarks/__init__.py` + `uv.lock` (the project's own `[[package]]` entry — find it with `grep -n 'name = "remove-ai-watermarks"' uv.lock`, the `version =` line right below it, ~line 2246), commit `chore(release): vX.Y.Z`, `git tag -a vX.Y.Z -m vX.Y.Z` (annotated — `git tag` without `-m` errors here), push `main` + the tag, then `gh release create vX.Y.Z` — **PyPI publish triggers on the GitHub Release `published` event, NOT on the tag push**, so the tag alone does not publish. **Sdist must exclude `data/`** (`[tool.hatch.build.targets.sdist] exclude = ["/data"]`): hatchling's default sdist bundles all VCS-tracked files, so the committed `data/` test corpora (the multi-hundred-MB synthid_corpus images + the visible-mark captures) pushed the **0.8.0** sdist past PyPI's per-project file-size limit (400 "File too large") — the wheel uploaded but the sdist was rejected, so 0.8.0 shipped wheel-only and 0.8.1 carried the fix. The wheel only ships `src/` (via `[tool.hatch.build.targets.wheel] packages`), so it was never affected. **A failed PyPI upload of one artifact still leaves the other live and you cannot re-upload the same version** — fix the build and cut the next patch. **Build backend is pinned `hatchling<1.31`** (`[build-system] requires`): hatchling **1.30.0** made **Metadata-Version 2.5** (PEP 794) the default, which the twine bundled in `pypa/gh-action-pypi-publish@release/v1` rejects (`"'2.5' is not a valid Metadata-Version"`) — this **failed the v0.8.3 PyPI upload on 2026-06-01** (tag-match + build passed, the upload step failed; nothing was uploaded, so the version stayed empty on PyPI), when unpinned `requires = ["hatchling"]` pulled 1.30.0. **hatchling 1.30.1 reverted the default back to 2.4** ("kept at 2.4 until more tools support 2.5"), and 1.27-1.29 always emitted 2.4 — so `<1.31` keeps `uv build` on a 2.4-emitting hatchling (it resolves to the latest allowed, **1.30.1**, which uploads fine). (The earlier "1.28+ emits 2.5" note was imprecise: the 2.5 default landed only in 1.30.0, verified against hatch's changelog.) The publish workflow **now uses `uv publish`** (its uploader handles 2.5), so this pin is no longer load-bearing — it stays as belt-and-suspenders so the first uv-publish release ships 2.4 metadata (isolating the uploader swap from the metadata-version bump); drop it to `requires = ["hatchling"]` once that release confirms the path.
|
||||
- `bash maintain.sh` — uv-outdated, uv-secure, ruff check/fix, ruff format, pyright, pytest -n auto
|
||||
- **Strict pyright is clean across `src/` (0 errors).** The cv2/torch/diffusers boundary files (`gemini_engine`, `region_eraser`, `doubao_engine`, `humanizer`, `invisible_engine`, `noai/watermark_remover`) carry a documented per-file `# pyright:` relax pragma that turns off only the unknown-type / untyped-third-party rules — those libs ship no usable types, so strict typing there fights the ecosystem. Pure-logic files stay fully strict; `typings/piexif/__init__.pyi` is a local stub so `metadata.py`/`extractor.py` resolve piexif. Public ndarray-returning signatures on the relaxed engines are still annotated `NDArray[Any]` so strict consumers (`cli.py`) stay clean. When touching a relaxed file, prefer fixing real issues over widening the pragma; keep the pragma scoped to genuinely-untyped boundaries. (`uv-secure` is clean since idna was bumped 3.11 -> 3.16, fixing GHSA-65pc-fj4g-8rjx, and aiohttp 3.13.5 -> 3.14.0 via `uv lock --upgrade-package aiohttp`, fixing GHSA-hg6j-4rv6-33pg + GHSA-jg22-mg44-37j8. (The basicsr Dependabot alert (GHSA-86w8-vhw6-q9qq, command injection, no patch, <= 1.4.2): accepted, not fixable -- basicsr is the optional `restore` extra pinned to 1.4.2 as the only buildable version, experimental and off by default.)
|
||||
- **CI** (`.github/workflows/test.yml`): runs on push to `main` + every PR. A `lint` job (ubuntu: `ruff check` + `ruff format --check`) plus a `test` matrix (ubuntu/macos/windows x py3.10/3.12) that does `uv sync --frozen --extra dev` then `pytest`. The matrix installs only core + dev (no `gpu` extra), so the GPU/model-running tests skip there and it exercises the metadata/identify/visible/cv2-eraser surface on all three OSes. Keep `uv.lock` valid (don't break `--frozen`) when editing `pyproject.toml`. `publish.yml` stays release-only and now verifies the release tag matches the `pyproject.toml` version (fails the build on a mismatch) before building, then uploads via `uv publish` (PyPI trusted publishing over OIDC, no token — replaced the `pypa/gh-action-pypi-publish` action so the upload no longer depends on that action's bundled twine accepting the Metadata-Version; the `id-token: write` permission + `pypi` environment + workflow filename are unchanged, so PyPI's trusted-publisher entry still matches). **Release flow:** bump the version in `pyproject.toml` + `src/remove_ai_watermarks/__init__.py` + `uv.lock` (the project's own `[[package]]` entry — find it with `grep -n 'name = "remove-ai-watermarks"' uv.lock`, the `version =` line right below it, ~line 2246), commit `chore(release): vX.Y.Z`, `git tag -a vX.Y.Z -m vX.Y.Z` (annotated — `git tag` without `-m` errors here), push `main` + the tag, then `gh release create vX.Y.Z` — **PyPI publish triggers on the GitHub Release `published` event, NOT on the tag push**, so the tag alone does not publish. **After the PyPI sdist is live, bump the Homebrew formula** in the separate public tap repo `wiltodelta/homebrew-tap` (`Formula/remove-ai-watermarks.rb`): update `url` to the new sdist URL (from `https://pypi.org/pypi/remove-ai-watermarks/<version>/json`, the `sdist` entry's `url`) and `sha256` to its hash, commit + push there — otherwise `brew install wiltodelta/tap/remove-ai-watermarks` keeps installing the old version. The formula is a core-only venv that pip-installs the sdist (no vendored resources, so pip pulls the binary numpy/opencv wheels per platform at install time); only those two lines change per release. **This is now AUTOMATED:** the main repo's `.github/workflows/distribute.yml` fires on the GitHub Release `published` event, waits for the sdist to appear on PyPI (poll loop, the Release event races publish.yml's upload), rewrites the formula's `url`+`sha256`, and pushes to the tap using the `HOMEBREW_TAP_TOKEN` repo secret (a fine-grained PAT with Contents:write on `homebrew-tap`). The SAME workflow also factory-rebuilds the HF Space (`HfApi.restart_space(..., factory_reboot=True)`, `HF_TOKEN` secret) so the Space reinstalls the new sdist (it pins `remove-ai-watermarks>=...` and only re-resolves on a rebuild). The manual Homebrew steps above are the fallback / what the workflow automates — a normal release needs no Homebrew or HF action. **Other distribution channels:** (1) **conda-forge** — recipe source of truth committed at `packaging/conda/recipe.yaml` (v1 `recipe.yaml`, noarch core-only: pillow/piexif/numpy/py-opencv/click/python-dotenv); the initial submission is `conda-forge/staged-recipes` PR #33674 (went green only after **`pip_check: false`** in the python test — rattler-build's `pip check` defaults to ON and fails on the ancient conda-forge `piexif py_2` build's stale metadata with "piexif 1.1.3 is not supported on this platform", though the package installs/imports/works; keep it disabled). Once that merges and the `remove-ai-watermarks-feedstock` exists, the `regro-cf-autotick-bot` auto-opens a version-bump PR on the feedstock when each new PyPI sdist is detected — just review + merge it (hand-edit only if run-deps changed; keep `packaging/conda/recipe.yaml` in sync as the reference copy). (2) **ComfyUI Registry** — the node package is a SEPARATE repo `wiltodelta/ComfyUI-remove-ai-watermarks` with its OWN `pyproject.toml` `version` (independent of the library version). Publish a new node version by bumping that `version` in the node repo's `pyproject.toml` and pushing to `main` — the node repo's `.github/workflows/publish.yml` (`Comfy-Org/publish-node-action@main`, triggered on a push that touches `pyproject.toml`, secret `COMFY_REGISTRY_TOKEN`) **auto-publishes** it; `comfy node publish --token <registry-PAT>` is the manual/local fallback. It is NOT auto-published on a library release (the node has its own version), so only bump it when the node code or its `remove-ai-watermarks>=` dependency floor changes. **Sdist must exclude `data/`** (`[tool.hatch.build.targets.sdist] exclude = ["/data"]`): hatchling's default sdist bundles all VCS-tracked files, so the committed `data/` test corpora (the multi-hundred-MB synthid_corpus images + the visible-mark captures) pushed the **0.8.0** sdist past PyPI's per-project file-size limit (400 "File too large") — the wheel uploaded but the sdist was rejected, so 0.8.0 shiLine truncated
|
||||
- `bash maintain.sh` — uv-outdated, uv-secure, ruff check/fix, ruff format, pyright (scoped `src/`, see the OOM note below), pytest -n auto. The helper tools live in the `dev` extra (`pytest-xdist`, plus `uv-outdated`/`uv-secure` marker-gated to py3.12+ so the py3.10 resolution stays solvable) — a bare env without `--extra dev` does not have them.
|
||||
- **Strict pyright is clean across `src/` (0 errors).** The cv2/torch/diffusers boundary files (`gemini_engine`, `region_eraser`, `doubao_engine`, `humanizer`, `invisible_engine`, `noai/watermark_remover`) carry a documented per-file `# pyright:` relax pragma that turns off only the unknown-type / untyped-third-party rules — those libs ship no usable types, so strict typing there fights the ecosystem. Pure-logic files stay fully strict; `typings/piexif/__init__.pyi` is a local stub so `metadata.py`/`extractor.py` resolve piexif. Public ndarray-returning signatures on the relaxed engines are still annotated `NDArray[Any]` so strict consumers (`cli.py`) stay clean. When touching a relaxed file, prefer fixing real issues over widening the pragma; keep the pragma scoped to genuinely-untyped boundaries. (`uv-secure` is clean since idna was bumped 3.11 -> 3.16, fixing GHSA-65pc-fj4g-8rjx, and aiohttp 3.13.5 -> 3.14.0 via `uv lock --upgrade-package aiohttp`, fixing GHSA-hg6j-4rv6-33pg + GHSA-jg22-mg44-37j8. (The old basicsr Dependabot alert (GHSA-86w8-vhw6-q9qq) is resolved by removal: the experimental `restore` extra was retired and basicsr is no longer anywhere in the dependency tree.) The torch Dependabot alert **GHSA-rrmf-rvhw-rf47** (`torch.jit.script` memory corruption, vulnerable `<= 2.12.0`) is **dismissed as `not_used`** (2026-06-10): torch is a transitive dep of the optional `gpu` extra only, the codebase never calls `torch.jit` (grep-verified), and **no patched torch version exists** (`first_patched_version` is null), so it cannot be closed by an upgrade — do not re-triage it.
|
||||
- **Full-project `uv run pyright` (no path) OOMs/crashes node on this ML-heavy repo** (emits a `libnode` stack frame, no summary) — a known environment limit, not a code error. Gate with `uv run --extra dev --extra gpu pyright src/` (completes, authoritative) or scope to changed files; also run `uv run ruff check` and `uv run pytest` directly.
|
||||
- Run `uv run` from the repo root — from another cwd it falls back to a bare env without numpy/cv2/torch.
|
||||
- **Stale `trustmark` remnant in site-packages after an extras change:** the `trustmark` package downloads model weights INTO its own package dir, so when a narrower `uv sync` prunes the package, a `trustmark/models/` directory survives as an empty namespace package. Symptom: pyright `"TrustMark" is unknown import symbol` on `trustmark_detector.py` and `find_spec("trustmark")` returning a loader-less spec (so `is_available()` lies True). Fix: `rm -rf .venv/lib/python3.12/site-packages/trustmark` (regenerable weights cache).
|
||||
- To add a dev tool (pytest/ruff/pyright) into the env, use `uv sync --frozen --extra dev --extra gpu`, **never `uv pip install`** — `uv pip install` re-resolves and rewrites `uv.lock`, which silently bumped `transformers` to a build incompatible with the pinned `diffusers` (`cannot import name 'Qwen3VLForConditionalGeneration'`) and broke every `identify`/metadata import. Recovery: `git checkout uv.lock && uv sync --frozen --extra gpu --extra dev`. The `gpu` extra holds `diffusers`/`transformers`/`torch`, so a bare `uv sync` (no extras) removes them; `noai/__init__` is now **lazy** (PEP 562 `__getattr__`, so importing `identify`/`metadata` no longer pulls `watermark_remover`/torch), so a bare env breaks only when the removal pipeline is actually invoked, not on import. `maintain.sh`'s `uv sync --all-extras` also pulls the heavy `trustmark`/`lama` wheels (pytorch-lightning, onnxruntime) — fine on a good connection, but on flaky DNS sync only `--extra gpu --extra dev` and run the lint/test steps by hand.
|
||||
- Metadata/C2PA tests assert against real committed fixtures in `data/samples/` (`chatgpt-*.png` = OpenAI C2PA, `firefly-1.png` = Adobe, `mj-*` = Midjourney IPTC, `doubao-1.png` = ByteDance Doubao with the China TC260 `<TC260:AIGC>` XMP label **and** a visible "豆包AI生成" text mark bottom-right; `grok-1.jpg` = xAI Grok with its EXIF-only `Signature:` blob + UUID `Artist` and no C2PA/SynthID/IPTC); synthetic byte blobs cover the JPEG/ISOBMFF format paths. The "non-AI / clean photo" control is no longer in `data/samples/` -- the `clean_photo` conftest fixture serves a verified-negative image from the corpus `neg/` set (skips if the corpus is absent).
|
||||
- SynthID reference corpus: `scripts/synthid_corpus.py` ingests labeled images into `data/synthid_corpus/`. The labeled `images/` (`pos/` `neg/` `cleaned/`) are **committed** (public repo -- review every image for private content before adding; `manifest.csv` is kept in sync with the files on disk, one row per tracked image); only the synthetic `refs/` calibration fills are gitignored. See its README for the collection protocol and verification oracles. **`cleaned/` examples must be produced by a CURRENT shipped removal method** -- the default SDXL img2img pass (optionally `--max-resolution`). Do NOT archive cleaned outputs from methods that are no longer in the pipeline (ctrlregen, the old text/face-protection, IP-Adapter FaceID, CodeFormer) or from the experimental opt-in paths (controlnet, face restore) as corpus examples; a cleaned reference should represent the canonical removal, and a removed method's output is not a reproducible example. Keep those experiment outputs in a local working dir, never in the committed corpus.
|
||||
@@ -36,9 +37,9 @@ You are a **principal Python engineer** maintaining a CLI tool and library for r
|
||||
- `noai/c2pa.py` — PNG chunk parser; use `extract_c2pa_chunk(path)` to get raw caBX payload, `has_c2pa_metadata(path)` to detect. Do not reimplement chunk parsing. `extract_c2pa_info(path)` sets `synthid_watermark`/`synthid_vendors` when the manifest is signed by a SynthID-using vendor, and `soft_binding`/`soft_binding_vendors` when a `c2pa.soft-binding` `alg` names a forensic-watermark vendor (`soft_binding_vendors_in(buffer)` is the shared byte-scan, used by both the PNG parser and the non-PNG binary path). PNG/caBX chunk reads are clamped to the remaining file size (`safe_length = min(length, remaining)`; skipped chunks use seek) so a malformed huge `length` cannot drive a multi-GB allocation (shared safety discipline matching `isobmff.scan_c2pa_region`).
|
||||
- `noai/constants.py` — PNG_SIGNATURE, C2PA_CHUNK_TYPE, C2PA_SIGNATURES, and `C2PA_AI_VENDORS` — the single `C2paAiVendor` registry of C2PA-signing vendors (issuer byte, resolved org name, the `identify` platform label, and a `synthid` flag), from which `C2PA_ISSUERS`, `SYNTHID_C2PA_ISSUERS` (issuers that pair SynthID with C2PA: Google, OpenAI), and `identify._ISSUER_PLATFORM` are all **derived** — plus `C2PA_SOFT_BINDINGS` (soft-binding `alg` prefix → forensic-watermark vendor: Adobe TrustMark, Digimarc, Imatag, Steg.AI, Microsoft, ...). Add a new C2PA vendor as one `C2PA_AI_VENDORS` entry (never edit the derived dicts), a new soft-binding to `C2PA_SOFT_BINDINGS`; not inline.
|
||||
- `metadata.py` — `scan_head(path, size=1MB)` is the shared input for every C2PA/AIGC/IPTC byte scan: first `size` bytes plus the payloads of any provenance metadata found beyond that window — for ISOBMFF, the late provenance boxes from `isobmff.scan_c2pa_region` (catches a manifest after a large `mdat`); for **PNG**, the late `tEXt`/`iTXt`/`zTXt`/`eXIf`/`iCCP` chunks from `_png_late_metadata` (catches an XMP/EXIF packet appended after a large `IDAT`, e.g. a TC260 AIGC label at ~2.7 MB). Behavior-neutral (`f.read(size)`) for non-ISOBMFF inputs and for any file that fits within `size`. Use it instead of `open().read(1MB)` for any new marker scan. **Memoized per (path, size, mtime)** (added 2026-06-09, `_scan_head_cached` lru_cache, `maxsize=8`): one `identify`/`get_ai_metadata` call fans out to ~8 byte-scan detectors that each re-read the same file head, so the cache turns those into a single read; the mtime key invalidates on change, a stat failure falls back to an uncached read. `synthid_source(path)` returns the vendor name(s) if the C2PA manifest implies a SynthID pixel watermark, else None. Format-agnostic: PNG via the caBX parser, JPEG/WebP/AVIF/HEIF/JXL via a binary scan (C2PA marker + SynthID issuer + AI-source marker). `get_ai_metadata` surfaces the verdict, and `metadata --check` prints it as a callout. Both `get_ai_metadata` and `has_ai_metadata` guard the PIL open with `except Exception` (HEIC/unknown formats raise non-OSError) and fall through to the binary scan. `xai_signature(path)` detects xAI/Grok's EXIF-only scheme (`ImageDescription` = `Signature: <base64>` + UUID `Artist`); it feeds `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify`. `iptc_ai_system(path)` detects the IPTC Photo Metadata 2025.1 AI-disclosure XMP properties (`IPTC_AI_FIELD_MARKERS` = `AISystemUsed`/`AISystemVersionUsed`/`AIPromptInformation`/`AIPromptWriterName`) and returns the `AISystemUsed` generator name (or `"fields present"`). `remove_ai_metadata` routes **ISOBMFF video** (`.mp4`/`.mov`/`.m4v`) through the same `isobmff.strip_c2pa_boxes` as AVIF/HEIF (MP4 is ISOBMFF), and `_scrub_ai_exif` removes the xAI signature + AI-generator EXIF tags on JPEG output. `strip_c2pa_boxes` is **fail-safe** on a malformed box: it returns the original bytes unchanged with a logged warning instead of truncating the tail to EOF (detection-only `scan_c2pa_region` still stops at a malformed box). `_png_late_metadata` clamps each late-chunk read to the remaining file size (`safe_length = min(length, remaining)`) so a malformed `length` cannot drive a multi-GB allocation, AND advances the cursor by `safe_length` (not the raw `length`) so an inflated length cannot jump past EOF and abort the scan, silently skipping a genuine AI-label chunk after it.
|
||||
- `identify.py` — the OpenAI rollout caveat is keyed on `_vendor_of(synthid) == "OpenAI"` (not a raw substring over the issuer + verdict blob). `identify(path)` aggregates every locally-readable signal (C2PA issuer→platform, C2PA soft-binding forensic-watermark vendor, IPTC "Made with AI" + IPTC 2025.1 `AISystemUsed`, embedded SD/ComfyUI params, SynthID proxy, xAI/Grok EXIF signature via `metadata.xai_signature`, the China TC260 AIGC label via `metadata.aigc_label`, the HuggingFace `hf-job-id` job marker via `metadata.huggingface_job`, the Samsung Galaxy AI editing marker via `metadata.samsung_genai`, the visible marks — Gemini sparkle plus the ByteDance Doubao 豆包AI生成 / Jimeng 即梦AI / Samsung Galaxy AI "Contenuti generati dall'AI" text marks via the `watermark_registry` — open invisible watermark, Adobe TrustMark via `trustmark_detector`) into one `ProvenanceReport`. `is_ai_generated` is True or None (never asserted False — stripped metadata is not proof of clean origin). The `hf_job`, visible-mark, and Samsung `samsung_genai` signals are **medium** confidence: each lifts an otherwise-Unknown verdict to a tentative AI (`hf_only` / `visible_only` / `samsung_only`, parallel branches; `visible_only` fires on any `visible_*` signal) but is excluded from the high-confidence `ai_from_metadata` set, so none overrides a hard metadata signal. **Visible-mark detection** (`check_visible`, signals `visible_sparkle` / `visible_doubao` / `visible_jimeng` / `visible_samsung`): the Gemini sparkle keeps its own file-level path (`_visible_sparkle` → `gemini_engine.detect_sparkle_confidence`, promoted only at confidence ≥ `_SPARKLE_THRESHOLD` 0.5; corpus-tuned to separate Gemini sparkles ≥0.56 from non-sparkle ≤0.49), while Doubao/Jimeng/Samsung reuse the registry detectors (`_visible_text_marks` → `watermark_registry`, iterating `_VISIBLE_MARK_PLATFORM`), each gated by its own engine NCC threshold via `MarkDetection.detected` (Doubao 0.4, Jimeng 0.45, Samsung 0.4). Doubao/Jimeng are normally also caught by the TC260 AIGC metadata label and Samsung by its C2PA + `genAIType` marker, so the visible path is their stripped-metadata fallback. Visible marks set `platform` only when no harder signal already did, and (like the sparkle) are excluded from integrity-clash vendor claims. The cv2 dependency lives in the engines, not here. **`import identify` is deliberately light** (~21 MB; ~36 MB with cv2 loaded by a visible-mark run, ~106 MB for a full `check_visible` run): it imports only the pure `noai.c2pa`/`noai.constants` submodules, and `noai/__init__` is lazy (see "Test and lint"), so torch/diffusers are NOT pulled at import even in a full `gpu`/`detect` install — fits a 512 MB host. The heavy paths are opt-in: `check_invisible=True` needs the `detect`/`trustmark` extras (each pulls **torch**; TrustMark also **downloads weights**), so on a core-only deploy leave `check_invisible` off (it is a no-op there anyway). Before the lazy `__init__`, the mere presence of torch in the env inflated `import identify` to ~420 MB. **C2PA platform attribution is device-token-first, issuer-scan fallback** (`_device_platform` scans manifest bytes for `_DEVICE_C2PA_PLATFORM` tokens, then `_attribute_platform`/`_ISSUER_PLATFORM`). **Why, verified on real signed files 2026-05-26:** the old issuer-only byte-scan matched ANY issuer substring anywhere, so multi-entity manifests mis-attributed -- Leica→"Truepic" (a signing authority in the trust chain), Nikon→"Adobe Firefly" (XMP-toolkit "Adobe" + the sample's "Adobe_MAX" name), Pixel→"Google (Gemini)" ("Google LLC" cert org), Truepic→"Google". A distinctive device token wins instead. **Token distinctiveness is load-bearing:** bare `b"Truepic"` mis-fires (it appears in unrelated trust chains -- it mis-attributed the OpenAI `chatgpt-1.png` fixture), so the token is the specific `b"Truepic_Lens"` from the Lens SDK claim generator; likewise `b"Pixel Camera"` (cert CN) not bare `b"Pixel"`. `_DEVICE_C2PA_PLATFORM` lists ONLY tokens **verified against a real C2PA file**: Leica (`lc_c2pa`/`Leica Camera`), Nikon (`NIKON`), Pixel (`Pixel Camera` -- from a real Pixel 10 Pro file attached to c2pa-rs issue #1609/#1554), Sony (`sony.sig`/`sony.cert` -- Sony's own C2PA assertion namespace, verified on a real Sony PXW-Z300 file; NOT bare "Sony" which is a common EXIF Make), Truepic (`Truepic_Lens`). Canon/Bria have **no public direct-download C2PA sample** (checked exhaustively: GitHub issue/PR attachments, contentcredentials gallery, HF datasets -- all upload-to-verify or token-gated; Canon's only public file was a self-signed hobbyist CR3, not factory), so they stay unmapped until a real file is captured (same fixture discipline as Grok/Doubao). The Sony sample is video (MP4) -- our ISOBMFF C2PA path detects it; Sony Alpha stills likely share the `sony.*` namespace but are not separately verified. **Samsung Galaxy + ASUS Gallery live in a separate `_SIGNER_C2PA_PLATFORM` (scaLine truncated
|
||||
- `identify.py` — the OpenAI rollout caveat is keyed on `_vendor_of(synthid) == "OpenAI"` (not a raw substring over the issuer + verdict blob). `identify(path)` aggregates every locally-readable signal (C2PA issuer→platform, C2PA soft-binding forensic-watermark vendor, **C2PA cloud-manifest reference** via `metadata.c2pa_cloud_manifest` — signal `c2pa_cloud`, **medium**, provenance-only (does NOT set `is_ai`, excluded from `ai_from_metadata` + clash vendors): a C2PA 2.4 Durable-Content-Credentials case where the embedded manifest is stripped but an XMP `dcterms:provenance` pointer to the vendor's cloud manifest store (`_C2PA_MANIFEST_REPOSITORIES`, today `cai-manifests.adobe.com` → "Adobe Content Authenticity") survives, so the credentials stay recoverable server-side; only emitted when no embedded manifest already attributed the file — surfaced on 2 corpus PNGs 2026-06-10 that read fully `unknown` before, IPTC "Made with AI" + IPTC 2025.1 `AISystemUsed`, embedded SD/ComfyUI params, SynthID proxy, xAI/Grok EXIF signature via `metadata.xai_signature`, the China TC260 AIGC label via `metadata.aigc_label`, the HuggingFace `hf-job-id` job marker via `metadata.huggingface_job`, the Samsung Galaxy AI editing marker via `metadata.samsung_genai`, the visible marks — Gemini sparkle plus the ByteDance Doubao 豆包AI生成 / Jimeng 即梦AI / Samsung Galaxy AI "Contenuti generati dall'AI" text marks via the `watermark_registry` — open invisible watermark, Adobe TrustMark via `trustmark_detector`) into one `ProvenanceReport`. `is_ai_generated` is True or None (never asserted False — stripped metadata is not proof of clean origin). The `hf_job`, visible-mark, and Samsung `samsung_genai` signals are **medium** confidence: each lifts an otherwise-Unknown verdict to a tentative AI (`hf_only` / `visible_only` / `samsung_only`, parallel branches; `visible_only` fires on any `visible_*` signal) but is excluded from the high-confidence `ai_from_metadata` set, so none overrides a hard metadata signal. **Visible-mark detection** (`check_visible`, signals `visible_sparkle` / `visible_doubao` / `visible_jimeng` / `visible_samsung`): the Gemini sparkle keeps its own file-level path (`_visible_sparkle` → `gemini_engine.detect_sparkle_confidence`, promoted only at confidence ≥ `_SPARKLE_THRESHOLD` 0.5; corpus-tuned to separate Gemini sparkles ≥0.56 from non-sparkle ≤0.49), while Doubao/Jimeng/Samsung reuse the registry detectors (`_visible_text_marks` → `watermark_registry`, iterating `_VISIBLE_MARK_PLATFORM`), each gated by its own engine NCC threshold via `MarkDetection.detected` (Doubao 0.4, Jimeng 0.45, Samsung 0.4). Doubao/Jimeng are normally also caught by the TC260 AIGC metadata label and Samsung by its C2PA + `genAIType` marker, so the visible path is their stripped-metadata fallback. Visible marks set `platform` only when no harder signal already did, and (like the sparkle) are excluded from integrity-clash vendor claims. The cv2 dependency lives in the engines, not here. **`import identify` is deliberately light** (~21 MB; ~36 MB with cv2 loaded by a visible-mark run, ~106 MB for a full `check_visible` run): it imports only the pure `noai.c2pa`/`noai.constants` submodules, and `noai/__init__` is lazy (see "Test and lint"), so torch/diffusers are NOT pulled at import even in a full `gpu`/`detect` install — fits a 512 MB host. The heavy paths are opt-in: `check_invisible=True` needs the `detect`/`trustmark` extras (each pulls **torch**; TrustMark also **downloads weights**), so on a core-only deploy leave `check_invisible` off (it is a no-op there anyway). Before the lazy `__init__`, the mere presence of torch in the env inflated `import identify` to ~420 MB. **C2PA platform attribution is device-token-first, issuer-scan fallback** (`_device_platform` scans manifest bytes for `_DEVICE_C2PA_PLATFORM` tokens, then `_attribute_platform`/`_ISSUER_PLATFORM`). **Why, verified on real signed files 2026-05-26:** the old issuer-only byte-scan matched ANY issuer substring anywhere, so multi-entity manifests mis-attributed -- Leica→"Truepic" (a signing authority in the trust chain), Nikon→"Adobe Firefly" (XMP-toolkit "Adobe" + the sample's "Adobe_MAX" name), Pixel→"Google (Gemini)" ("Google LLC" cert org), Truepic→"Google". A distinctive device token wins instead. **Token distinctiveness is load-bearing:** bare `b"Truepic"` mis-fires (it appears in unrelated trust chains -- it mis-attributed the OpenAI `chatgpt-1.png` fixture), so the token is the specific `b"Truepic_Lens"` from the Lens SDK claim generator; likewise `b"Pixel Camera"` (cert CN) not bare `b"Pixel"`. `_DEVICE_C2PA_PLATFORM` lists ONLY tokens **verified against a real C2PA file**: Leica (`lc_c2pa`/`Leica Camera`), Nikon (`NIKON`), Pixel (`Pixel Camera` -- from a real Pixel 10 Pro file attached to c2pa-rs issue #1609/#1554), Sony (`sony.sig`/`sony.cert` -- Sony's own C2PA assertion namespace, verified on a real Sony PXW-Z300 file; NOT bLine truncated
|
||||
- `watermark_registry.py` — **single catalog of known visible watermarks**, the unified "find known marks in their usual places, recognize, remove" entry. **Reverse-alpha based by policy**: a mark is listed only once a real alpha map has been captured for it, and removal inverts that map (`original = (wm - a*logo)/(1-a)`) — Gemini recovers cleanly with no inpaint (its sparkle alpha comes from a pure-black capture, so it is near-exact), while **Doubao, Jimeng, and Samsung all add an always-on THIN residual inpaint** over the glyph footprint (their text marks re-rasterize + jitter a few px per image, so a single capture cannot pixel-cancel them; the inpaint blends into the reverse-alpha-recovered pixels). Arbitrary-region inpainting still lives in `region_eraser`/`erase`. Each `KnownMark` ties a key to {usual `location`, `in_auto` flag, `recovery` (="reverse-alpha"), a `detect` adapter → uniform `MarkDetection`, a `remove` adapter}. Entries today: `gemini` (bottom-right sparkle), `doubao` (bottom-right "豆包AI生成"), `jimeng` (bottom-right "★ 即梦AI"), and `samsung` (bottom-**LEFT** "✦ Contenuti generati dall'AI", Samsung Galaxy AI, Italian locale). `detect_marks` scans all; `best_auto_mark` picks the highest-confidence detection. **Cross-engine confidences aren't directly comparable**, so the gemini adapter applies the corpus-validated 0.5 sparkle threshold (`_GEMINI_AUTO_MIN_CONF`) for its `detected` flag — otherwise the gemini engine's loose internal threshold weakly fires (~0.36) on the Doubao text and hijacks `auto`. The shape-keyed Doubao/Jimeng/Samsung NCC detectors don't cross-fire (jimeng scores ~0.22 on the Doubao strip, well under its 0.45 threshold; Samsung is bottom-left so it shares no corner with the others, and scored 0.0 on Doubao/Jimeng captures and they 0.0 on a real Samsung photo), so `auto` picks the right one. `cli.cmd_visible` is registry-driven: `--mark auto` → `best_auto_mark`, `--mark <key>` → that mark; `--mark` choices come from `mark_keys()`. **`cli._remove_visible_auto` is the shared visible-removal helper used by `cmd_all`/`cmd_batch` too** (they no longer hardcode `GeminiEngine`), so `all`/`batch` remove Doubao/Jimeng/Samsung text marks, not just the Gemini sparkle (regression-guarded by `test_all_visible_step_uses_registry`). The three text-mark adapters were consolidated 2026-06-09: a single `_text_mark(key, label, location)` builds the registry row from one parameterized `_text_mark_detect`/`_text_mark_remove` pair (reverse-alpha only when detected/forced AND `reverse_alpha_available`, else skipped — no inpaint); the gemini adapters stay bespoke. Add a new visible mark = one `_text_mark(...)` row + its `TextMarkConfig` (with a captured alpha map); do not re-add per-mark `if` branches or copy-paste adapters. **Alpha-on-save policy (issue #30):** `cli._write_bgr_with_alpha` rejoins the input's alpha plane **unchanged** — it must NOT zero alpha in the watermark bbox. Reverse-alpha (and `erase` inpaint) recover real pixels there, so zeroing alpha punched a transparent hole that renders as a solid **white box** on any non-transparent viewer (Gemini app exports are opaque RGBA, so every user hit it; regression-guarded by `test_visible_keeps_alpha_opaque_in_watermark_region`). The registry `remove()` still returns its region (used for `inpaint_residual` positioning), but the CLI no longer uses it to clear alpha.
|
||||
- `gemini_engine.py` — visible Gemini-sparkle remover/detector (cv2/numpy, no GPU). `detect_sparkle_confidence(path)` is the file-level entry point used by `identify.py`. The public entry points normalize a grayscale (2D) or RGBA (4-channel) input to BGR up front so a non-BGR image does not crash the cv2 pipeline. **Detection localization (issue #36):** `detect_watermark`'s global multi-scale NCC search applies a size weight (`(scale/96)**0.5`) that suppresses tiny-patch false positives but can let a larger, mediocre match (e.g. a bright collar in a portrait) outrank a small, near-perfect sparkle in the corner — so a faint sparkle on a busy background scored below threshold and read as clean (the regression osachub reported from widening the search window 256px->512px between v0.7.2 and v0.8.8). `_corner_promote` adds a bottom-right-corner raw-NCC pass on top of the global search: a match with raw NCC >= `_CORNER_PROMOTE_NCC` 0.85 that beats the global pick overrides it (it only ever replaces a lower-fidelity pick, so it cannot weaken an existing detection), rescuing the buried sparkle without reverting the wider window. The corner side is **relative-clamped** (`_CORNER_PROMOTE_FRAC` 0.20 of the short side, clamped to `[_CORNER_PROMOTE_MIN` 96, `_CORNER_PROMOTE_MAX` 384`]`): a fixed 256px is a true corner on a large image but covers ~70% of a small portrait, where a real photo raw-matches the star at ~0.81 (relative tightening drops that worst case to ~0.69, while the upper clamp stops the corner ballooning on huge images where a real photo reached ~0.83 at 512px). The 0.85 gate sits midway between the worst real-photo corner match (~0.78 across native + downscaled negatives) and a genuine faint sparkle (~0.93), so promotion adds true detections with zero corpus false positives (Gemini's sparkle sits ~60-160px from the corner at fixed margins, covered by the [96, 384] band at every measured size). Regression-guarded by `test_gemini_engine.py::TestCornerPromotion`. **Removal is reverse-alpha with an over-subtraction guard** (`remove_watermark` → `_reverse_alpha_blend`, else `_inpaint_footprint`): the sparkle alpha is computed (`alpha = max(R,G,B)/255`) from the bundled sparkle-on-black captures `assets/gemini_bg_{96,48}.png` (the capture max is ~130, NOT 255 — the sparkle is a ~51%-opaque white overlay, so `alpha` maxes at ~0.51, which is CORRECT for the capture, not under-exposed). The alpha is near-exact only when the real mark's effective opacity matches the capture, which holds on bright/flat backgrounds — re-verified clean on `demo_banana_before.png` 2026-05-31. **Issue #30 (dark-background black pit):** on a dark/textured background (e.g. grass, ~73) the real sparkle's effective opacity is LOWER than the captured 0.51, so the fixed-alpha reverse blend OVER-subtracts (`watermarked - a*logo` goes negative) and drives the footprint to black — the white sparkle becomes a black diamond. `remove_watermark` now detects this via `_reverse_alpha_oversubtracts` (fraction of footprint pixels with `alpha >= _FOOTPRINT_ALPHA` 0.1 whose numerator < 0 exceeds `_OVERSUB_FOOTPRINT_FRAC` 0.05) and **inpaints the footprint** (`_inpaint_footprint`, cv2 NS over the dilated alpha mask) from the surrounding pixels instead. **Behavior-neutral on the working case:** a bright background over-subtracts at ~0% so reverse-alpha is used and the output is byte-identical to before (verified: demo_banana 0.0 frac vs issue-#30 grass 0.61 frac; regression-guarded by `test_gemini_engine.py::TestOverSubtractionGuard`, which composites the sparkle at a reduced effective alpha to reproduce the mismatch). **Under-subtraction (the symmetric case, fixed 2026-06-03):** some real Gemini sparkles are rendered MORE opaque than the captured ~0.51, so the fixed-alpha reverse blend UNDER-subtracts and leaves a bright sparkle residual the detector still fires on (measured on the spaces corpus: a visible-removal audit through the registry path left a detectable sparkle on a meaningful fraction of marks, all under-removals, NOT a background-brightness class — failures and successes had the same input confidence and the same background-luma distribution; the discriminator was the removal delta itself). `remove_watermark` now estimates a per-image alpha gain (`_estimate_alpha_gain`: effective sparkle opacity at the bright core vs the local background ring, `a_eff/a_cap`, clamped `[1.0, _ALPHA_GAIN_MAX` 1.94`]`) and scales the alpha to match before the over-sub/blend branch. The gain cleanly separates on the corpus (under-removed marks ~1.47, cleanly-removed ~1.00), and a deadband (`_ALPHA_GAIN_DEADBAND` 1.05) keeps a matching sparkle **byte-identical** to the pre-fix output, so the fix is purely additive (0 regressions on the audit set; the over-sub guard still runs on the scaled alpha as the safety net for an over-shooting estimate). Regression-guarded by `test_gemini_engine.py::TestUnderSubtractionGain` (composites a more-opaque-than-captLine truncated
|
||||
- `gemini_engine.py` — visible Gemini-sparkle remover/detector (cv2/numpy, no GPU). `detect_sparkle_confidence(path)` is the file-level entry point used by `identify.py`. The public entry points normalize a grayscale (2D) or RGBA (4-channel) input to BGR up front so a non-BGR image does not crash the cv2 pipeline. **Detection localization (issue #36):** `detect_watermark`'s global multi-scale NCC search applies a size weight (`(scale/96)**0.5`) that suppresses tiny-patch false positives but can let a larger, mediocre match (e.g. a bright collar in a portrait) outrank a small, near-perfect sparkle in the corner — so a faint sparkle on a busy background scored below threshold and read as clean (the regression osachub reported from widening the search window 256px->512px between v0.7.2 and v0.8.8). `_corner_promote` adds a bottom-right-corner raw-NCC pass on top of the global search: a match with raw NCC >= `_CORNER_PROMOTE_NCC` 0.85 that beats the global pick overrides it (it only ever replaces a lower-fidelity pick, so it cannot weaken an existing detection), rescuing the buried sparkle without reverting the wider window. The corner side is **relative-clamped** (`_CORNER_PROMOTE_FRAC` 0.20 of the short side, clamped to `[_CORNER_PROMOTE_MIN` 96, `_CORNER_PROMOTE_MAX` 384`]`): a fixed 256px is a true corner on a large image but covers ~70% of a small portrait, where a real photo raw-matches the star at ~0.81 (relative tightening drops that worst case to ~0.69, while the upper clamp stops the corner ballooning on huge images where a real photo reached ~0.83 at 512px). The 0.85 gate sits midway between the worst real-photo corner match (~0.78 across native + downscaled negatives) and a genuine faint sparkle (~0.93), so promotion adds true detections with zero corpus false positives (Gemini's sparkle sits ~60-160px from the corner at fixed margins, covered by the [96, 384] band at every measured size). Regression-guarded by `test_gemini_engine.py::TestCornerPromotion`. **Square-image residual misses are NOT fixable by lowering the detector threshold (measured + REJECTED 2026-06-11):** osachub (#36 follow-up) reported the corner-promote still misses Gemini sparkles on Google **square (1:1)** outputs. Reproduced on the spaces corpus: of 330 square Google-C2PA images, 140 score below the identify 0.5 threshold, and visual review confirmed a real class -- faint white sparkles on dark/textured/colored backgrounds (raw NCC 0.46-0.73, below the 0.85 promote gate) landing at fusion conf 0.41-0.47. A margin-gated promote (promote when raw NCC >= 0.50 AND `_core_ring_margin` >= 40) rescued 32/33 confirmed misses at an apparent 0 FP, but that 0 was a **measurement artifact** -- the negative set was the margin<40 misses, which a margin>=40 gate excludes by construction. On an honest 518-image non-Google pool the same gate fired on **~174 (≈33%)**, visually content (screenshots, Chinese "AI生成" Doubao/Jimeng text marks, logos, bright textures), not sparkles. Adding an achromatic-core constraint (`chroma <= 15`) did not separate them either (kept 15/33 POS, 41 NEG still firing). Root cause is the documented contrast-invariant-NCC wall: a faint sparkle on a busy background is indistinguishable from a bright/ornate content corner at the (shape-NCC, brightness-margin, core-chroma) feature level. **Conclusion: keep the 0.85 corner gate; do NOT add a margin/chroma-gated lower promote.** The cost (mislabel ~8-33% of non-Gemini content as Gemini) outweighs the benefit -- the visible sparkle is a medium-confidence stripped-metadata fallback, and intact Gemini is caught by C2PA in `identify` regardless. Remaining square misses are an accepted known limitation; a real fix would need a sparkle-specific discriminator (template match on a background-subtracted image, or a hard fixed-margin position prior), which is open research, not a threshold tweak. **Removal is reverse-alpha with an over-subtraction guard** (`remove_watermark` → `_reverse_alpha_blend`, else `_inpaint_footprint`): the sparkle alpha is computed (`alpha = max(R,G,B)/255`) from the bundled sparkle-on-black captures `assets/gemini_bg_{96,48}.png` (the capture max is ~130, NOT 255 — the sparkle is a ~51%-opaque white overlay, so `alpha` maxes at ~0.51, which is CORRECT for the capture, not under-exposed). The alpha is near-exact only when the real mark's effective opacity matches the capture, which holds on bright/flat backgrounds — re-verified clean on `demo_banana_before.png` 2026-05-31. **Issue #30 (dark-background black pit):** on a dark/textured background (e.g. grass, ~73) the real sparkle's effective opacity is LOWER than the captured 0.51, so the fixed-alpha reverse blend OVER-subtracts (`watermarked - a*logo` goes negative) and drives the footprint to black — the white sparkle becomes a black diamond. `remove_watermark` now detects this via `_reverse_alpha_oversubtracts` (fraction of footprint pixels with `alpha >= _FOOTPRINT_ALPHA` 0.1 whose numeLine truncated
|
||||
- `_text_mark_engine.py` — **shared base for the three reverse-alpha text-mark engines (Doubao/Jimeng/Samsung), extracted 2026-06-09** (they were ~90% byte-identical clones). `TextMarkEngine(config: TextMarkConfig)` owns the whole `locate → extract_mask → detect → _fixed/_aligned_alpha_map → _apply_reverse_alpha → remove_watermark_reverse_alpha` pipeline (+ the asset-keyed `load_alpha_template`/`glyph_silhouette`/`template_match_score` caches). Each engine module is now a thin subclass: it supplies only its `TextMarkConfig` (the tuned constants, the bundled asset, and the bounded structural deltas — `corner` br/bl, `margin_floor` 4/2, `morph_open_size` 5/3, `min_gw` 8/16) plus the test-facing module shims (`_alpha_template`/`_glyph_silhouette`/`_template_match_score` + the constants). Behavior is byte-exact vs the old per-engine code (the three engine test suites pass unchanged). Gemini stays a SEPARATE engine (its multi-size fixed-slot sparkle model is genuinely different). Add a new text mark = a new `TextMarkConfig` + a thin subclass + one registry `_text_mark(...)` row. The engine bullets below describe each mark's calibration history; the LOGIC lives here.
|
||||
- `doubao_engine.py` — **a thin `_text_mark_engine.TextMarkEngine` subclass (config only) since 2026-06-09.** visible Doubao "豆包AI生成" remover/detector (cv2/numpy, no GPU). `DoubaoEngine.locate` anchors a bottom-right box by **geometry** (mark scales with image WIDTH), `extract_mask` pulls the light, low-chroma glyphs (the detection candidate) using a per-pixel channel-spread proxy `sat = roi.max(axis=2) - roi.min(axis=2)` (no HSV conversion). `detect` is **shape-consistent**: it matches the bundled alpha glyph silhouette (`assets/doubao_alpha.png`) against the candidate via zero-mean normalized correlation (`_template_match_score`, cv2 `TM_CCOEFF_NORMED`), gated at `DETECT_NCC_THRESHOLD` 0.4 over a small `DETECT_MIN_COVERAGE` floor. Keying on glyph SHAPE (not coverage heuristics) fixed #23 (corpus FP 7/1243). **Removal = reverse-alpha + thin residual inpaint** (`remove_watermark_reverse_alpha`): `original = (wm - a*logo)/(1-a)` from the bundled alpha map + `_ALPHA_LOGO_BGR` (pure white) + `_ALPHA_*_FRAC` geometry, then a deliberately THIN inpaint (`_RESIDUAL_*`, `INPAINT_NS`) over the glyph footprint clears leftover edges without smearing. **Alpha is rebuilt by `scripts/visible_alpha_solve.py` (the careful gray-self solve: cubic background fit, mean over channels, full halo, unblurred), same recipe as Jimeng** — the captures are committed in `data/doubao_capture/captures/`. **Removal aligns ALWAYS** (no `_ALPHA_NATIVE_BAND` fast-path): it tries fixed geometry AND `_aligned_alpha_map`'s `TM_CCOEFF_NORMED` scale+position search and keeps the lower-residual one — the mark is re-rasterized and a few px off per image, so fixed geometry alone leaves a visible outline even at 2048. **The locate box (`WM_*`) is generous (0.22 wide, margins 0.004) and reaches close to the corner** — a tight box (the old 0.185 / margin 0.012) let a corner-ward shift fall OUTSIDE the alignment search, so the align missed and a readable outline survived; regression-guarded by `test_recovers_shifted_mark_on_texture` (composes the alpha shifted on a known texture; old box ~29 vs new ~1 mean residual). **Issue #13 follow-up defect (found 2026-05-31): the SHIPPED Doubao removal left a clearly READABLE "豆包AI生成" outline on the real `doubao-1.png` sample, while `detect` returned conf 0.0 (it is fooled by a thin outline) so `test_reverse_alpha_removes_mark` passed and the old "56/56 clean" claim was detector-measured, not visual.** Root cause: bad alpha (under-estimated, max ~0.65) + fixed-no-inpaint + tight box; the careful rebuild + always-align + thin inpaint + wide box takes it from a readable outline to faint texture-level traces (parity with Jimeng — a single capture cannot pixel-cancel a per-image re-rasterized mark). **Lesson: a detector-only removal test is insufficient; assert visual residual (the textured-shift test).** **`extract_mask` guards a degenerate ROI (`bh < 16 or bw < 16` -> empty mask, skips cv2):** the always-align removal scores each placement with a residual `detect(out)`, and on an extremely wide/short image (e.g. 2048x1, `test_wide_short_does_not_raise`) that fed cv2's GaussianBlur a ~1-px-tall ROI and **faulted natively on Windows py3.12 (access violation, non-deterministic — one CI cell went red while a re-run passed)**; the old at-native path never ran `detect` on degenerate sizes. Real images always clear the guard (the `WM_*` box floors are `max(16, …)` height / `max(40, …)` width), so it only short-circuits slivers. `reverse_alpha_available` is just "asset present"; the registry gates removal on `detect`. The shipped third-party `_refs/zhengsuanfa_doubao_alpha_120x20.png` is NOT a usable alpha (verified 2026-05-29). Arbitrary-region inpainting is `region_eraser`/`erase`.
|
||||
- `jimeng_engine.py` — **a thin `TextMarkEngine` subclass (config only) since 2026-06-09.** visible Jimeng / Dreamina "★ 即梦AI" remover/detector (cv2/numpy, no GPU), built 2026-05-30 from issue #13's solid captures (@powersee). Shares the base with `doubao_engine`: `locate` anchors a bottom-right box by **geometry** (scales with WIDTH), `extract_mask` pulls the light low-chroma glyphs (white top-hat + grayish + min-luma), `detect` matches the bundled "即梦AI" glyph silhouette (`assets/jimeng_alpha.png`) via `TM_CCOEFF_NORMED` over a coverage floor. Threshold `DETECT_NCC_THRESHOLD` **0.45** cleanly separates real Jimeng marks (>=0.81) from the Doubao strip (0.21) and other AI output (0.0), so the two ByteDance marks don't cross-fire in `--mark auto`. **Logo is pure white (255,255,255)** (`_ALPHA_LOGO_BGR`; the white capture + an L-pair-solve confirm ~254.6); compositing is **sRGB, not linear** (a linear-light solve tripled the cross-residual). **Alpha rebuilt by `scripts/visible_alpha_solve.py` from the GRAY capture** (`data/jimeng_capture/captures/`, the solid captures now committed): `a = (I - B)/(255 - B)`, B a per-capture **cubic** background fit over the non-glyph pixels, **averaged over channels, full halo extent (down to a~0.02), unblurred**. Gray (bg ~132) is the deliberate choice over black: it is the best proxy for real content (the mark sits on bright photo areas, not on black), and the careful build drops the gray self-residual to ~1.3. **The mask quality, not the method, was the earlier limit** — a max-channel / quadratic-bg / blurred / halo-truncated build (and a black-dominated LS) left a visible outline (lesson from issue #13: when reverse-alpha leaves a ghost, suspect the captured alpha map before adding heuristics or switching method). Geometry emitted by the solver at `_ALPHA_NATIVE_WIDTH` 2048: `_ALPHA_WIDTH_FRAC` 0.202, `_ALPHA_HEIGHT_FRAC` 0.058, margins ~0.029. **Removal = reverse-alpha + a deliberately THIN residual inpaint** (`remove_watermark_reverse_alpha`, `_RESIDUAL_DILATE` 5 over the `_RESIDUAL_ALPHA_FLOOR` 0.05 footprint, `_RESIDUAL_INPAINT_RADIUS` 2, `INPAINT_NS`): a single 2048 alpha cannot pixel-cancel the mark re-rasterized at another resolution (alpha maps from independent captures correlate 0.998, not 1.0; off-native reverse-alpha alone only halves the mark), so a tight inpaint clears the residual edges WITHOUT the texture/edge smear a wide full-footprint pass caused. **Placement ALWAYS tries fixed geometry AND `_aligned_alpha_map`'s NCC scale+position search, keeping the lower-residual** — the mark re-rasterizes + jitters a few px per image even at the captured width, so fixed geometry alone misses (there is no `_ALPHA_NATIVE_BAND` fast-path; the scale search `_ALPHA_ALIGN_SEARCH` is fine-stepped, and the `WM_*` locate box is generous so a corner-ward shift stays inside the search — the same widen that fixed Doubao). Verified clean on the solid captures (native 2048; faint self-residual ~1.3 visible only on a dead-flat field, hidden by real texture) and a real 1440-wide Jimeng download (off-native, table edge preserved). `reverse_alpha_available` is just "asset present"; the registry gates on `detect`. **No committed real sample** (the real content download stays gitignored; only the solid calibration captures are committed) — `tests/test_jimeng_engine.py` synthesizes a mark from the bundled alpha asset, and `test_recovers_shifted_mark_on_texture` guards the align-on-shift path that the Doubao defect exposed. Jimeng images are independently caught by the China TC260 AIGC label in `metadata`/`identify`, so this engine is the visible-mark *removal* path, not a new `identify` signal.
|
||||
@@ -70,7 +71,7 @@ Who embeds what, and whether it is locally detectable (so we know which gaps are
|
||||
- **C2PA / IPTC (covered by the issuer/marker scan):** OpenAI, Google, Adobe Firefly, Microsoft (Designer + **Bing Image Creator** — collected 2026-05-24; Bing now runs Microsoft's own **MAI-Image** model, signs C2PA as "Microsoft", NOT OpenAI/DALL-E), and **Stability AI** (collected from Brand Studio / DreamStudio successor; signs C2PA as "Stability AI Ltd", no SynthID, no imwatermark on its current Stable Image model — issuer added to `C2PA_ISSUERS`). Still unsampled: Canva (its downloads are re-encoded design *exports* that strip C2PA, so a Canva "positive" is inconclusive — skipped), Getty, Shutterstock. Midjourney embeds NO C2PA and no invisible watermark (our `mj-*` sample carried only the IPTC tag). **Samsung Galaxy AI** (Generative Edit / Sketch to Image / Portrait Studio on Galaxy S23 FE / S24 / S25, One UI 7+) signs C2PA as "Samsung Galaxy" with the standard `trainedAlgorithmicMedia` source type AND a proprietary `genAIType` marker; verified on real signed files 2026-05-29 (the standard scan catches the source type; `genAIType` additionally catches a Galaxy S24 file that omits it). It ALSO burns a **visible** localized wordmark into the pixels — a sparkle + "generated with AI" string in the bottom-LEFT corner (issue #37; the Italian "✦ Contenuti generati dall'AI" variant is calibrated) — removed by `samsung_engine.py` / `visible --mark samsung` (reverse-alpha, see the engine bullet); detection feeds `identify` as the medium `visible_samsung` signal. The string is locale-specific, so each locale needs its own captured alpha template. **ASUS Gallery** also signs edited photos as C2PA (`com.asus.gallery`) but with no AI source type — a signer, not an AI marker. **Black Forest Labs (FLUX)** API output signs C2PA: `claim_generator_info "Black Forest Labs API"` + a `c2pa.ai_generated_content` assertion + `trainedAlgorithmicMedia` (issuer `b"Black Forest Labs"` added to `C2PA_ISSUERS`, platform "Black Forest Labs (FLUX)"). **ByteDance Volcano Engine (Volcengine)** — the cloud behind Doubao / Jimeng — signs its AI image output with a cert from `certificate_center@volcengine.com` + `trainedAlgorithmicMedia` (issuer `b"volcengine"` → "ByteDance (Volcano Engine)", platform "ByteDance (Doubao / Jimeng / Volcano Engine)"); note this is the C2PA-signed surface, distinct from the XMP/PNG TC260 `AIGC` label Doubao also uses. All three verified on real signed files 2026-05-29.
|
||||
- **EXIF/XMP generator tag (caught by `exif_generator`):** **Ideogram** writes EXIF `Make="Ideogram AI"` (collected 2026-05-24 — no C2PA, no SynthID, no imwatermark; the Make tag is the only signal).
|
||||
- **xAI / Grok — its own EXIF signature scheme, NOT C2PA (DETECTED by `metadata.xai_signature`, built 2026-05-26).** Grok JPEG downloads (Aurora model) carry **no C2PA, no XMP, no SynthID, no IPTC** — only EXIF `Artist` = a UUID and EXIF `ImageDescription` = `Signature: <base64>` (a crypto signature, unverifiable locally without xAI's public key). This empirically kills the earlier unverified "xAI signs C2PA as xAI" lead — xAI is not even a C2PA member. `exif_generator` misses it (neither field holds an `AI_GENERATOR_TOKENS` token), so a dedicated detector `xai_signature(path)` matches the pair (`ImageDescription ~ ^Signature: [A-Za-z0-9+/=]{64,}` AND UUID `Artist`); wired into `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify` (signal `xai_signature`, platform "xAI (Grok / Aurora)"). **Format confirmed stable across n=3 genuine generations:** exactly three EXIF tags (`Artist`, `ExifOffset`, `ImageDescription`), `Signature:` prefix constant, base64 payload 300-1004 chars. Two capture facts: (a) the `Artist` UUID **equals the public image id** in the asset URL (`https://imagine-public.x.ai/imagine-public/images/<uuid>.jpg`), so it is NOT a private per-user secret — only the `Signature` blob is; (b) the Grok web-UI image is a re-encoded **WebP with no signature** — the EXIF survives only in the *original* JPEG (download button or that public tokenless URL), which is why screenshots / re-encodes are metadata-stripped. A real fixture `data/samples/grok-1.jpg` plus **synthetic** JPEG fixtures (fake UUID + fake `Signature:` blob) cover the detector; never add a real Grok image carrying private content (the repo is public). **Stripped on removal too:** `remove_ai_metadata` now calls `_scrub_ai_exif` on the JPEG EXIF, which deletes the xAI Signature+UUID-Artist pair **and** any `Software`/`Make`/`Artist`/`ImageDescription` tag holding an `AI_GENERATOR_TOKENS` token (so Ideogram's `Make="Ideogram AI"` is scrubbed too), while keeping genuine camera/editor EXIF. The shared `_is_xai_signature_pair` helper (module-level compiled regexes) is the single source of truth for the pattern, used by both `xai_signature` and `_scrub_ai_exif`. (AVIF/HEIF/JXL still strip only C2PA boxes via `isobmff`, not EXIF — unchanged.)
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). **Doubao** (ByteDance) uses it (verified on the real #13 sample 2026-05-25; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **three serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `<TC260:AIGC>{...}</TC260:AIGC>` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on the corpus 2026-05-30) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on the corpus 2026-05-28, `ContentProducer="doubao"`), **and** a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on the corpus 2026-05-30, `ContentProducer="001191440300708461136T1308L"`). Both generic forms (the PNG chunk and the bare `{"AIGC":...}` object) are gated on at least one TC260 field (`_TC260_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated).
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). **Doubao** (ByteDance) uses it (verified on the real #13 sample 2026-05-25; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **four serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `<TC260:AIGC>{...}</TC260:AIGC>` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on the corpus 2026-05-30) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on the corpus 2026-05-28, `ContentProducer="doubao"`), a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on the corpus 2026-05-30, `ContentProducer="001191440300708461136T1308L"`), **and** a bare `AIGC{...}` blob (the label glued straight to its JSON, no `"AIGC":` key wrapper) embedded in a **JPEG APP segment near the JFIF header** — confirmed on the corpus 2026-06-10 (`ContentProducer="00119144030008867405X210002"`; 3 files read `unknown` before this form was added). The two raw-JSON forms are scanned in one loop (`'"AIGC"'` then `AIGC{`) that **falls through on a non-TC260 / undecodable hit instead of returning** — a quoted `"AIGC"` can appear later in an XMP packet while the real label is a bare `AIGC{...}` earlier in the file, so an unconditional early return on the quoted form would shadow the bare form (the exact bug behind the 06-10 misses). All three generic forms (the PNG chunk, the bare `{"AIGC":...}` object, and the bare `AIGC{...}` blob) are gated on at least one TC260 field (`_TC260_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated).
|
||||
- **HuggingFace-hosted job (caught by `metadata.huggingface_job`, surfaced by `identify` as the `hf_job` signal, MEDIUM confidence):** HuggingFace Jobs / Spaces stamp generated PNGs with an `hf-job-id` tEXt chunk holding the job UUID (3 on the corpus 2026-05-28, no other signal). It marks the *hosting job*, not a model — most commonly diffusion output — so it lifts an Unknown verdict to a tentative AI via `hf_only` (parallel to the visible sparkle) but never overrides a hard metadata signal; `_HF_JOB_CAVEAT` states the limit (job, not model; not proof of AI pixels). Stripped on removal (the PNG save whitelist keeps only `STANDARD_METADATA_KEYS`, so `hf-job-id` and the `AIGC` chunk are both dropped). The exact writer is not authoritatively documented (HF Jobs are generic GPU jobs), hence medium not high.
|
||||
- **No detectable signal on download (correctly reported `unknown`):** **Recraft** (PNG export is a re-encoded design export — strips everything), **Krea hosting FLUX 2** (no imwatermark despite FLUX — the host omits the encoder, same as Stability's hosted SDXL), and Midjourney (embeds nothing). Lesson: the imwatermark detector only fires on *pristine* output from a pipeline that runs the encoder (diffusers default, official BFL), not from re-hosts (Krea/Stability) or re-encoded exports (Recraft/Canva).
|
||||
- **Invisible but NOT locally detectable (proprietary, API/oracle only — same wall as SynthID):** Amazon Titan Image Generator + Nova Canvas (Bedrock `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), NVIDIA Cosmos (SynthID video). No local detector possible; treat like SynthID.
|
||||
|
||||
@@ -8,6 +8,10 @@ Strips SynthID, C2PA Content Credentials, EXIF/XMP "Made with AI" labels, and vi
|
||||
>
|
||||
> No Python, no GPU, no setup. Visible-watermark and metadata removal are **free**. Invisible-watermark removal (SynthID / SDXL regeneration) normally needs a local GPU and ~2 GB of models. On **[raiw.cc](https://raiw.cc)** it runs on cloud GPUs in one click for a small per-image fee.
|
||||
|
||||
[](https://pypi.org/project/remove-ai-watermarks/)
|
||||
[](https://pypi.org/project/remove-ai-watermarks/)
|
||||
[](https://pepy.tech/project/remove-ai-watermarks)
|
||||
[](LICENSE)
|
||||
[](https://github.com/wiltodelta/remove-ai-watermarks/actions/workflows/test.yml)
|
||||
[](https://github.com/sponsors/wiltodelta)
|
||||
|
||||
@@ -26,7 +30,7 @@ If this tool saves you time, consider [sponsoring its development](https://githu
|
||||
- **Text and face preservation (default)** — the default pipeline is a canny ControlNet that keeps text and face structure sharp through the removal pass (without copying original pixels, so SynthID is still removed). Use `--pipeline sdxl` for plain SDXL img2img (lighter, no extra model download) on inputs without text or faces. Canny preserves face *structure*, not *identity* (the regenerated face drifts in likeness). The library does not ship a face-restore extra: every approach evaluated (GFPGAN-on-cleaned, PhotoMaker-V2, InstantID txt2img, InstantID img2img-on-cleaned) regenerated the face via SDXL and made the output look more AI-generated than the cleaned image. The cleaned controlnet output is the least-AI face state achievable without re-introducing SynthID.
|
||||
- **Batch processing** — process entire directories
|
||||
- **Detection** — three-stage NCC watermark detection with confidence scoring
|
||||
- **Provenance detection (`identify`)** — aggregate C2PA issuer, the C2PA soft-binding forensic-watermark vendor (Adobe TrustMark, Digimarc, Imatag, ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, embedded SD/ComfyUI params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the China TC260 AIGC label (XMP, PNG chunk, or EXIF), the HuggingFace `hf-job-id` job marker, the SynthID metadata proxy, the visible marks (Gemini sparkle plus the Doubao "豆包AI生成" / Jimeng "即梦AI" / Samsung Galaxy AI "Contenuti generati dall'AI" text marks), the open SD/SDXL/FLUX invisible watermark, and (with the `trustmark` extra) the open Adobe TrustMark watermark into one origin-platform + watermark-inventory verdict (`--json` for machine output)
|
||||
- **Provenance detection (`identify`)** — aggregate C2PA issuer, the C2PA soft-binding forensic-watermark vendor (Adobe TrustMark, Digimarc, Imatag, ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, embedded SD/ComfyUI params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the China TC260 AIGC label (XMP, PNG chunk, EXIF, or JPEG segment), the HuggingFace `hf-job-id` job marker, the SynthID metadata proxy, the C2PA cloud-manifest reference (Adobe Durable Content Credentials, when the embedded manifest is stripped), the visible marks (Gemini sparkle plus the Doubao "豆包AI生成" / Jimeng "即梦AI" / Samsung Galaxy AI "Contenuti generati dall'AI" text marks), the open SD/SDXL/FLUX invisible watermark, and (with the `trustmark` extra) the open Adobe TrustMark watermark into one origin-platform + watermark-inventory verdict (`--json` for machine output)
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -58,7 +62,7 @@ If this tool saves you time, consider [sponsoring its development](https://githu
|
||||
|
||||
> Visible overlays are used by Google Gemini / Nano Banana (sparkle logo), by ByteDance's Doubao ("豆包AI生成" corner text) and Jimeng / Dreamina ("★ 即梦AI" wordmark), and by Samsung Galaxy AI ("✦ Contenuti generati dall'AI" strip, bottom-left, locale-specific). All are removed on CPU by reverse-alpha against a captured alpha map (Jimeng and Samsung add a thin residual inpaint over the glyph footprint, since their marks re-rasterize per image). Other services rely on invisible watermarks and/or metadata; our diffusion-based regeneration works against any invisible watermark in pixel or frequency domain. For a visible mark from any other source (any position, any colour), use the universal `erase --region` command.
|
||||
|
||||
> **Detection:** `remove-ai-watermarks identify <image>` reports the origin platform and watermark inventory for all the signals above — C2PA issuer, the C2PA soft-binding forensic-watermark vendor (TrustMark / Digimarc / Imatag / ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, the China TC260 AIGC label (XMP, PNG chunk, or EXIF), the HuggingFace `hf-job-id` job marker, embedded generation params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the SynthID metadata proxy, the visible marks (Gemini sparkle plus the Doubao "豆包AI生成" / Jimeng "即梦AI" / Samsung Galaxy AI "Contenuti generati dall'AI" text marks), and (with the `[detect]` / `[trustmark]` extras) the open SD/SDXL/FLUX and Adobe TrustMark invisible watermarks. SynthID and the proprietary soft-binding watermarks (Digimarc etc.) have no local decoder, so they are reported by metadata proxy / vendor name only.
|
||||
> **Detection:** `remove-ai-watermarks identify <image>` reports the origin platform and watermark inventory for all the signals above — C2PA issuer, the C2PA soft-binding forensic-watermark vendor (TrustMark / Digimarc / Imatag / ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, the China TC260 AIGC label (XMP, PNG chunk, EXIF, or JPEG segment), the HuggingFace `hf-job-id` job marker, embedded generation params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the SynthID metadata proxy, the C2PA cloud-manifest reference (Adobe Durable Content Credentials, when the embedded manifest is stripped), the visible marks (Gemini sparkle plus the Doubao "豆包AI生成" / Jimeng "即梦AI" / Samsung Galaxy AI "Contenuti generati dall'AI" text marks), and (with the `[detect]` / `[trustmark]` extras) the open SD/SDXL/FLUX and Adobe TrustMark invisible watermarks. SynthID and the proprietary soft-binding watermarks (Digimarc etc.) have no local decoder, so they are reported by metadata proxy / vendor name only.
|
||||
|
||||
## How it works
|
||||
|
||||
@@ -154,6 +158,34 @@ The cleaner parses each layer, removes AI-related fields, and preserves standard
|
||||
|
||||
## Installation
|
||||
|
||||
### Homebrew (macOS / Linux)
|
||||
|
||||
```bash
|
||||
brew install wiltodelta/tap/remove-ai-watermarks
|
||||
```
|
||||
|
||||
This installs the core command surface (`identify`, `metadata`, `visible`,
|
||||
`erase`) as a self-contained CLI. The diffusion-based `invisible` / `all`
|
||||
pipeline needs heavy ML dependencies (torch, diffusers, multi-GB) and is kept
|
||||
out of the Homebrew build; add it with the `gpu` extra via pip if you need it:
|
||||
|
||||
```bash
|
||||
pip install "remove-ai-watermarks[gpu]"
|
||||
```
|
||||
|
||||
### conda
|
||||
|
||||
A conda-forge recipe is under review
|
||||
([staged-recipes PR](https://github.com/conda-forge/staged-recipes/pull/33674)).
|
||||
Once it merges, the core package installs with:
|
||||
|
||||
```bash
|
||||
conda install -c conda-forge remove-ai-watermarks
|
||||
```
|
||||
|
||||
Like the Homebrew build, this is the core command surface; add the diffusion
|
||||
`invisible` / `all` pipeline with the pip `gpu` extra.
|
||||
|
||||
### Recommended
|
||||
|
||||
Install as an isolated CLI tool — no need to manage virtual environments:
|
||||
@@ -200,6 +232,10 @@ After installation the `remove-ai-watermarks` command is available system-wide.
|
||||
> pip install -e ".[gpu]" # or: uv pip install -e ".[gpu]"
|
||||
> ```
|
||||
>
|
||||
> Without the `[gpu]` extra, `all` still runs the visible and metadata steps, but
|
||||
> it skips the invisible (SynthID) step, prints a clear warning, and exits with a
|
||||
> non-zero status so a skipped step is not mistaken for a clean result.
|
||||
>
|
||||
> To let `identify` decode the open Stable Diffusion / SDXL / FLUX invisible
|
||||
> watermarks, install the `detect` extra (adds the `invisible-watermark` decoder):
|
||||
>
|
||||
@@ -252,6 +288,23 @@ pytest
|
||||
./maintain.sh
|
||||
```
|
||||
|
||||
## ComfyUI
|
||||
|
||||
Custom nodes are available so the watermark tools run inside a ComfyUI graph:
|
||||
[ComfyUI-remove-ai-watermarks](https://github.com/wiltodelta/ComfyUI-remove-ai-watermarks)
|
||||
([registry](https://registry.comfy.org/nodes/remove-ai-watermarks)).
|
||||
|
||||
Install via ComfyUI Manager (search "Remove AI Watermarks") or manually:
|
||||
|
||||
```bash
|
||||
cd ComfyUI/custom_nodes
|
||||
git clone https://github.com/wiltodelta/ComfyUI-remove-ai-watermarks
|
||||
pip install -r ComfyUI-remove-ai-watermarks/requirements.txt
|
||||
```
|
||||
|
||||
Nodes: Remove Visible Watermark, Detect Visible Watermark, Erase Region (by
|
||||
mask), and Remove Invisible Watermark / SynthID (needs the `gpu` extra).
|
||||
|
||||
## Usage
|
||||
|
||||
### CLI
|
||||
@@ -416,6 +469,13 @@ Won't fix:
|
||||
|
||||
- **Nightshade / Glaze / PhotoGuard removal**. These are defensive perturbations used by artists to protect their work from being scraped into AI training sets. Removing them attacks artists, not AI provenance. Out of scope.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Visible-mark and metadata removal is lossless.** Reverse-alpha recovers the original pixels under the mark; metadata stripping never touches image data.
|
||||
- **The invisible (SynthID) path is lossy and not guaranteed.** It runs a low-strength SDXL img2img regeneration, so it softens fine detail and is content-dependent. There is no public SynthID decoder, so the tool cannot verify removal locally; confirm with the Gemini app's "Verify with SynthID" oracle and raise `--strength` if it still detects. A vendor can change the scheme at any time, so treat this as an arms race, not a permanent fix.
|
||||
- **Large images are not tiled yet.** The SynthID path runs at the diffusion model's native resolution and caps or upscales larger inputs instead of tiling them, so very large images lose detail. (Nano Banana 2 is natively 1024px; GPT Image 2 supports 4K experimentally.) Tiled diffusion is the planned fix.
|
||||
- **Out of scope:** defeating trained AI-vs-real classifiers like Hive (see [Threat model](#threat-model)), visible-logo removal from video, and any guarantee that a stripped copy is untraceable server-side.
|
||||
|
||||
## Legal
|
||||
|
||||
Watermarking and provenance for AI-generated content is now regulated in several jurisdictions. The table below summarises the May 2026 status. None of this is legal advice.
|
||||
@@ -433,14 +493,14 @@ Watermarking and provenance for AI-generated content is now regulated in several
|
||||
|
||||
## Threat model
|
||||
|
||||
This tool defends already-distributed AI imagery against automatic detection systems (social-platform "Made with AI" labels, third-party classifiers, content-policy filters). It does **not** retroactively anonymise generation.
|
||||
This tool removes specific, known signals: the embedded SynthID pixel watermark, the visible vendor marks, and the C2PA / EXIF / IPTC provenance metadata that platforms read to apply automatic "Made with AI" labels. It is **not** a general detector-evasion tool. It does **not** defeat trained statistical AI-vs-real classifiers (for example Hive Moderation), and a light diffusion pass will not reliably fool those, so a clean classifier hit after removal is expected, not a bug. It also does **not** retroactively anonymise generation. And watermarking is a weak trust signal in the first place: a marker that is almost always present yet trivially removable can make a cleaned forgery look more trustworthy, not less, which is why durable provenance more likely comes from signing genuine content than from watermarking synthetic content.
|
||||
|
||||
In particular, **SynthID** (Google DeepMind) is embedded across Google's generative media stack — Imagen (images), Veo (video), Lyria (audio) — and Gemini app image outputs (Nano Banana / Gemini 3 Pro, which we verified positive via the Gemini app's SynthID oracle); Google reported over 10 billion items watermarked by December 2025. It carries a **multi-bit payload** — the research paper's SynthID-O variant encodes 136-bit payloads in 512x512 images ([arxiv 2510.09263](https://arxiv.org/abs/2510.09263)). The payload is believed to encode a user / session identifier. If the original watermarked file ever passed through a system controlled by the prompt originator (a saved Gemini account history, a screenshot uploaded to a Google product, a backup), Google retains the ability to link that original to the generating account. Stripping the watermark from a copy you possess does not erase Google's server-side record.
|
||||
|
||||
Use cases where the threat model fits:
|
||||
- You generated the image yourself, want to publish it as your own work, and accept the consequences if Google ever publishes their detector logs.
|
||||
- You are running a security / robustness evaluation.
|
||||
- You are preserving art or historical record against false-positive "AI-generated" labels.
|
||||
- A real photo of yours was lightly AI-edited (a retouch in Gemini or ChatGPT, say) and now carries a SynthID or C2PA label that overstates how AI-generated it is, and you want to clear that label from your own copy.
|
||||
|
||||
Use cases where the threat model **does not** fit:
|
||||
- Generating an image, expecting that removing the watermark anonymises you to Google. It doesn't.
|
||||
|
||||
+3
-1
@@ -7,5 +7,7 @@ uv run uv-outdated
|
||||
uv run uv-secure --ignore-unfixed
|
||||
uv run ruff check --fix
|
||||
uv run ruff format
|
||||
uv run pyright
|
||||
# Scoped to src/: a full-project pyright run OOM-crashes node on this ML-heavy
|
||||
# repo (see CLAUDE.md "Test and lint"); src/ is the authoritative strict gate.
|
||||
uv run pyright src/
|
||||
uv run pytest -n auto
|
||||
@@ -0,0 +1,63 @@
|
||||
schema_version: 1
|
||||
|
||||
context:
|
||||
version: "0.10.1"
|
||||
python_min: "3.10"
|
||||
|
||||
package:
|
||||
name: remove-ai-watermarks
|
||||
version: ${{ version }}
|
||||
|
||||
source:
|
||||
url: https://pypi.org/packages/source/r/remove-ai-watermarks/remove_ai_watermarks-${{ version }}.tar.gz
|
||||
sha256: ec92b450363d947cd897f0b75c18e75c2988ff79314ea247333ba892470b5a77
|
||||
|
||||
build:
|
||||
noarch: python
|
||||
number: 0
|
||||
script: python -m pip install . -vv --no-deps --no-build-isolation
|
||||
|
||||
requirements:
|
||||
host:
|
||||
- python ${{ python_min }}.*
|
||||
- pip
|
||||
- hatchling
|
||||
run:
|
||||
- python >=${{ python_min }}
|
||||
- pillow >=10.0.0
|
||||
- piexif >=1.1.3
|
||||
- numpy >=1.24.0
|
||||
- py-opencv >=4.8.0
|
||||
- click >=8.0.0
|
||||
- python-dotenv >=1.0.0
|
||||
|
||||
tests:
|
||||
- python:
|
||||
imports:
|
||||
- remove_ai_watermarks
|
||||
- remove_ai_watermarks.identify
|
||||
- remove_ai_watermarks.metadata
|
||||
python_version: ${{ python_min }}.*
|
||||
# pip_check defaults to true; disable it explicitly. `pip check` fails on
|
||||
# the conda-forge piexif `py_2` build, whose stale 2019 metadata pip reads
|
||||
# as "piexif 1.1.3 is not supported on this platform" -- even though the
|
||||
# package installs, imports, and works. The imports test verifies loading.
|
||||
pip_check: false
|
||||
|
||||
about:
|
||||
homepage: https://github.com/wiltodelta/remove-ai-watermarks
|
||||
summary: Remove visible and invisible AI watermarks from images
|
||||
description: |
|
||||
Detect and remove visible AI watermarks (Gemini / Nano Banana sparkle,
|
||||
ByteDance Doubao and Jimeng, Samsung Galaxy AI) and strip AI-provenance
|
||||
metadata (C2PA, EXIF, IPTC, PNG text chunks) from images. The core package
|
||||
covers the identify / metadata / visible / erase command surface; optional
|
||||
pip extras add SynthID diffusion removal and additional invisible-watermark
|
||||
detectors.
|
||||
license: Apache-2.0
|
||||
license_file: LICENSE
|
||||
repository: https://github.com/wiltodelta/remove-ai-watermarks
|
||||
|
||||
extra:
|
||||
recipe-maintainers:
|
||||
- wiltodelta
|
||||
+37
-21
@@ -1,19 +1,48 @@
|
||||
[project]
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.10.0"
|
||||
description = "Remove visible and invisible AI watermarks from images (Gemini / Nano Banana, ChatGPT, Stable Diffusion)"
|
||||
version = "0.11.0"
|
||||
description = "AI watermark remover: strip visible and invisible AI watermarks (Gemini / Nano Banana sparkle, SynthID) and provenance metadata (C2PA, EXIF) from images"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = {text = "Apache-2.0"}
|
||||
keywords = [
|
||||
"ai-watermark",
|
||||
"ai-watermark-remover",
|
||||
"watermark-remover",
|
||||
"watermark-removal",
|
||||
"remove-watermark",
|
||||
"synthid",
|
||||
"c2pa",
|
||||
"content-credentials",
|
||||
"nano-banana",
|
||||
"gemini",
|
||||
"gemini-watermark",
|
||||
"ai-metadata",
|
||||
"metadata-removal",
|
||||
"provenance",
|
||||
"exif",
|
||||
"stable-diffusion",
|
||||
"comfyui",
|
||||
"ai-detection",
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: End Users/Desktop",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Multimedia :: Graphics",
|
||||
"Topic :: Multimedia :: Graphics :: Graphics Conversion",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
"Topic :: Scientific/Engineering :: Image Processing",
|
||||
"Topic :: Security",
|
||||
"Topic :: Utilities",
|
||||
]
|
||||
dependencies = [
|
||||
"pillow>=10.0.0",
|
||||
@@ -91,19 +120,17 @@ esrgan = [
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-xdist>=3.5.0",
|
||||
"ruff>=0.4.0",
|
||||
"pyright>=1.1.0",
|
||||
"invisible-watermark>=0.2.0",
|
||||
# maintain.sh helpers; they only support newer Pythons, so gate them by
|
||||
# marker to keep the py3.10 resolution (and CI matrix) solvable.
|
||||
"uv-outdated>=0.1.0; python_version >= '3.12'",
|
||||
"uv-secure>=0.12.0; python_version >= '3.12'",
|
||||
]
|
||||
all = ["remove-ai-watermarks[gpu,detect,trustmark,lama,dev]"]
|
||||
|
||||
# diffusers 0.38.0 (security fix for GHSA-98h9-4798-4q5v) declares a dependency
|
||||
# on safetensors>=0.8.0rc0 — a pre-release. Allow pre-releases globally so the
|
||||
# resolver can satisfy that. Drop once diffusers publishes a release with a
|
||||
# stable safetensors pin (or once safetensors 0.8.0 stable is out).
|
||||
[tool.uv]
|
||||
prerelease = "allow"
|
||||
|
||||
# PyTorch Intel-GPU (XPU) wheel index. ``explicit = true`` keeps it inert for
|
||||
# the default CPU/CUDA install: uv consults it only when a torch install
|
||||
# explicitly targets it (see the ``gpu`` extra comment), so it does not alter
|
||||
@@ -120,18 +147,7 @@ remove-ai-watermarks = "remove_ai_watermarks.cli:main"
|
||||
Repository = "https://github.com/wiltodelta/remove-ai-watermarks"
|
||||
|
||||
[build-system]
|
||||
# Pin hatchling < 1.31. hatchling 1.30.0 made Metadata-Version 2.5 (PEP 794) the
|
||||
# default, which the twine bundled in pypa/gh-action-pypi-publish@release/v1 rejects
|
||||
# ("'2.5' is not a valid Metadata-Version"), failing the v0.8.3 PyPI upload
|
||||
# (2026-06-01) when unpinned requires = ["hatchling"] pulled 1.30.0. hatchling 1.30.1
|
||||
# reverted the default to 2.4 ("kept at 2.4 until more tools support 2.5"), and
|
||||
# 1.27-1.29 were always 2.4 -- so < 1.31 keeps `uv build` on a 2.4-emitting hatchling
|
||||
# (it resolves to the latest allowed, 1.30.1). The publish workflow now uses
|
||||
# `uv publish`, whose uploader accepts 2.5, so this pin is belt-and-suspenders, not
|
||||
# load-bearing: keeping it makes the first uv-publish release ship 2.4 metadata
|
||||
# (isolating the uploader swap from the metadata-version bump). Drop to
|
||||
# `requires = ["hatchling"]` once that release confirms the path.
|
||||
requires = ["hatchling<1.31"]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -11,4 +11,4 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.10.0"
|
||||
__version__ = "0.11.0"
|
||||
@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Literal
|
||||
import click
|
||||
|
||||
from remove_ai_watermarks import __version__, watermark_registry
|
||||
from remove_ai_watermarks.noai.constants import SUPPORTED_FORMATS
|
||||
from remove_ai_watermarks.noai.watermark_profiles import (
|
||||
resolve_strength,
|
||||
strength_default_help,
|
||||
@@ -106,8 +107,6 @@ Progress = _Progress
|
||||
SpinnerColumn = BarColumn = TextColumn = TimeElapsedColumn = _column
|
||||
console = _Console()
|
||||
|
||||
SUPPORTED_FORMATS = {".png", ".jpg", ".jpeg", ".webp"}
|
||||
|
||||
|
||||
def _setup_logging(verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.WARNING
|
||||
@@ -919,6 +918,13 @@ def cmd_all(
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
# Tracks whether step 2 (invisible / SynthID removal) was skipped because the
|
||||
# GPU extra is missing. A skipped step 2 still produces an output file (visible
|
||||
# mark + metadata stripped), so without a loud end-of-run notice + non-zero exit
|
||||
# the user mistakes it for a clean result and ships an image that still carries
|
||||
# the invisible watermark (recurring reports: #14, #47).
|
||||
synthid_skipped = False
|
||||
|
||||
# Use a temp file for intermediate results so the user doesn't see
|
||||
# a partial output file during long model downloads.
|
||||
import tempfile
|
||||
@@ -955,6 +961,7 @@ def cmd_all(
|
||||
from remove_ai_watermarks.invisible_engine import is_available as invisible_available
|
||||
|
||||
if not invisible_available():
|
||||
synthid_skipped = True
|
||||
console.print(
|
||||
" Warning: Skipped - GPU dependencies not installed.\n"
|
||||
" Install them with: pip install 'remove-ai-watermarks[gpu]'"
|
||||
@@ -1029,6 +1036,24 @@ def cmd_all(
|
||||
size_kb = output.stat().st_size / 1024
|
||||
console.print(f"\n Done: {output} ({size_kb:.0f} KB, {elapsed:.1f}s total)")
|
||||
|
||||
# A skipped invisible step is the single most common "it didn't work" report:
|
||||
# the output looks processed but still carries the SynthID watermark. Make that
|
||||
# impossible to miss -- a prominent banner plus a non-zero exit so scripts and
|
||||
# batch callers can detect the incomplete run instead of trusting the file.
|
||||
if synthid_skipped:
|
||||
console.print(
|
||||
"\n =====================================================================\n"
|
||||
" WARNING: the invisible (SynthID) watermark was NOT removed.\n"
|
||||
" Step 2 was skipped because the GPU dependencies are not installed,\n"
|
||||
" so this output still carries the invisible watermark -- only the\n"
|
||||
" visible mark and metadata were stripped.\n"
|
||||
"\n"
|
||||
" Install the extra and rerun to remove it:\n"
|
||||
" pip install 'remove-ai-watermarks[gpu]'\n"
|
||||
" ====================================================================="
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
# -- Batch command ----------------------------------------------------
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
``apply_analog_humanizer`` injects film grain and chromatic aberration to defeat
|
||||
digital AI-perfection classifiers (ported from NeuralBleach); ``unsharp_mask``
|
||||
counters the soft, over-smoothed look that diffusion + face-restoration leave
|
||||
behind (itself a common "this is AI" tell).
|
||||
counters the soft, over-smoothed look that the diffusion pass leaves behind
|
||||
(itself a common "this is AI" tell).
|
||||
"""
|
||||
|
||||
# cv2/numpy boundary: third-party libs ship no usable element types; relax the
|
||||
@@ -63,7 +63,7 @@ def apply_analog_humanizer(image: NDArray, grain_intensity: float = 4.0, chromat
|
||||
def unsharp_mask(image: NDArray, amount: float = 0.5, sigma: float = 1.0) -> NDArray:
|
||||
"""Sharpen via unsharp masking: ``out = image + amount * (image - blur(image))``.
|
||||
|
||||
Counters the soft, over-smoothed look of the diffusion + GFPGAN passes, which
|
||||
Counters the soft, over-smoothed look of the diffusion pass, which
|
||||
reads as an AI tell. ``amount`` 0 = no-op (returns an unchanged copy); ~0.5-0.8
|
||||
is a safe range -- higher risks bright edge halos that are their own artifact.
|
||||
``sigma`` is the Gaussian radius of the unsharp kernel.
|
||||
|
||||
@@ -30,6 +30,7 @@ from remove_ai_watermarks.metadata import (
|
||||
IPTC_AI_FIELD_MARKERS,
|
||||
IPTC_AI_MARKERS,
|
||||
aigc_label,
|
||||
c2pa_cloud_manifest_in,
|
||||
c2pa_marker_in,
|
||||
exif_generator,
|
||||
get_ai_metadata,
|
||||
@@ -96,6 +97,13 @@ _HF_JOB_CAVEAT = (
|
||||
"generation) but names neither the model nor the content type, so it is a "
|
||||
"medium-confidence signal, not proof the pixels are AI-generated."
|
||||
)
|
||||
_C2PA_CLOUD_CAVEAT = (
|
||||
"The embedded C2PA manifest is absent but an XMP provenance pointer to the "
|
||||
"vendor's cloud manifest store survives, so the Content Credentials remain "
|
||||
"recoverable server-side -- stripping the file no longer removes the provenance. "
|
||||
"It marks Content Credentials, not AI origin: the cloud manifest may describe a "
|
||||
"human edit, and reading it needs a network fetch this tool does not make."
|
||||
)
|
||||
_SAMSUNG_GENAI_CAVEAT = (
|
||||
"Samsung's genAIType marker shows a Galaxy AI editing tool (Generative Edit, "
|
||||
"Sketch to Image, ...) touched the image; it is an undocumented proprietary "
|
||||
@@ -285,7 +293,12 @@ def _vendor_of(text: str | None) -> str | None:
|
||||
# chain like Adobe over a Gemini original) legitimately names several vendors in
|
||||
# one valid chain and must not read as spoofing. Families not listed here are each
|
||||
# their own independent source (EXIF/XMP generator, IPTC AISystemUsed, AIGC, ...).
|
||||
_CLASH_SOURCE: dict[str, str] = {"c2pa": "c2pa_manifest", "synthid": "c2pa_manifest"}
|
||||
# The single C2PA-manifest source shared by the issuer attribution and the SynthID
|
||||
# proxy (both inferred from the same embedded manifest). Rule 2 keys off it too:
|
||||
# the camera device label is read from this manifest, so an AI marker is a clash
|
||||
# only when its source differs from this (i.e. it is genuinely independent).
|
||||
_C2PA_MANIFEST_SOURCE = "c2pa_manifest"
|
||||
_CLASH_SOURCE: dict[str, str] = {"c2pa": _C2PA_MANIFEST_SOURCE, "synthid": _C2PA_MANIFEST_SOURCE}
|
||||
|
||||
|
||||
def _integrity_clashes(
|
||||
@@ -326,7 +339,16 @@ def _integrity_clashes(
|
||||
+ " -- one provenance set was likely spoofed, transplanted, or laundered."
|
||||
)
|
||||
|
||||
if camera_label and camera_has_ai_marker:
|
||||
# Rule 2: a camera-capture C2PA device next to an AI-generation marker. Only
|
||||
# an AI marker from a source INDEPENDENT of the camera's own C2PA manifest is
|
||||
# a contradiction. A device that both captures and runs on-device generative
|
||||
# AI (Google Pixel Magic Editor / Pixel Studio) records the capture and the
|
||||
# AI edit in ONE manifest, so the AI vendor is named only from that same
|
||||
# manifest (c2pa issuer + synthid proxy) -- a legitimate edit chain, not a
|
||||
# spoof. An EXIF/XMP generator, IPTC field, TC260 AIGC label, or second
|
||||
# manifest naming AI on a camera capture is the real laundering tell.
|
||||
independent_ai_marker = any(grp != _C2PA_MANIFEST_SOURCE for grp in source.values())
|
||||
if camera_label and camera_has_ai_marker and independent_ai_marker:
|
||||
vendors = ", ".join(sorted(set(ai_vendors.values()))) or "present"
|
||||
clashes.append(
|
||||
f"Camera-capture C2PA credentials ({camera_label}) coexist with AI-generation markers "
|
||||
@@ -483,6 +505,21 @@ def identify(image_path: Path, *, check_visible: bool = True, check_invisible: b
|
||||
if c2pa_is_ai and (v := (_vendor_of(_attribute_platform(issuers, is_ai=True)) or _vendor_of(generator))):
|
||||
ai_vendor_claims["c2pa"] = v
|
||||
|
||||
# ── C2PA cloud-manifest reference (Durable Content Credentials) ─
|
||||
# An XMP dcterms:provenance pointer to a vendor manifest store survives even
|
||||
# when the embedded manifest is stripped, so the credentials stay recoverable
|
||||
# server-side (C2PA 2.4). Provenance only -- it does NOT assert AI (the cloud
|
||||
# manifest may describe a human edit), so it is excluded from ai_from_metadata
|
||||
# and the clash vendors. Skip when an embedded manifest already attributed it.
|
||||
if not has_c2pa and (cloud_vendor := c2pa_cloud_manifest_in(head)):
|
||||
signals.append(Signal("c2pa_cloud", f"cloud manifest store: {cloud_vendor}", "medium"))
|
||||
watermarks.append(
|
||||
f"C2PA Durable Content Credentials (cloud manifest at {cloud_vendor}; embedded manifest absent)"
|
||||
)
|
||||
caveats.append(_C2PA_CLOUD_CAVEAT)
|
||||
if platform is None:
|
||||
platform = f"C2PA signer: {cloud_vendor} (cloud manifest)"
|
||||
|
||||
# ── SynthID metadata proxy ──────────────────────────────────────
|
||||
# get_ai_metadata already sets synthid_watermark for both PNG (caBX parser)
|
||||
# and non-PNG (its own synthid_source fallback), so no extra scan is needed.
|
||||
|
||||
@@ -19,6 +19,8 @@ import warnings
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from .noai.watermark_profiles import DEFAULT_MODEL_ID as DEFAULT_SDXL_MODEL_ID
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
@@ -37,9 +39,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Check if invisible watermark removal dependencies are installed."""
|
||||
import importlib.util
|
||||
from .optional_deps import module_available
|
||||
|
||||
return importlib.util.find_spec("diffusers") is not None and importlib.util.find_spec("torch") is not None
|
||||
return module_available("diffusers", "torch")
|
||||
|
||||
|
||||
def _target_size(width: int, height: int, max_resolution: int, min_resolution: int = 0) -> tuple[int, int] | None:
|
||||
@@ -83,7 +85,7 @@ class InvisibleEngine:
|
||||
|
||||
# SDXL base is the default since May 2026; the vendor-adaptive strength
|
||||
# removes the current SynthID (see watermark_profiles + docs/synthid.md).
|
||||
DEFAULT_MODEL_ID = "stabilityai/stable-diffusion-xl-base-1.0"
|
||||
DEFAULT_MODEL_ID = DEFAULT_SDXL_MODEL_ID
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -176,7 +178,7 @@ class InvisibleEngine:
|
||||
output_path: Output path (None = overwrite source).
|
||||
strength: Denoising strength (0.0-1.0). None -> the vendor-adaptive
|
||||
default.
|
||||
steps: Number of denoising steps.
|
||||
num_inference_steps: Number of denoising steps.
|
||||
guidance_scale: Classifier-free guidance scale.
|
||||
seed: Random seed for reproducibility.
|
||||
humanize: Intensity of Analog Humanizer film grain (0 = off).
|
||||
|
||||
@@ -50,9 +50,9 @@ _MATCH_SD1_FRAC = 0.92 # fraction of the 136 string bits that must match
|
||||
|
||||
def is_available() -> bool:
|
||||
"""True if the optional imwatermark decoder is installed."""
|
||||
import importlib.util
|
||||
from .optional_deps import module_available
|
||||
|
||||
return importlib.util.find_spec("imwatermark") is not None
|
||||
return module_available("imwatermark")
|
||||
|
||||
|
||||
def _bits_match(value: int, ref: int, width: int = 48) -> int:
|
||||
|
||||
@@ -20,6 +20,11 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Smaller scan_head window for the cheap marker checks (has_ai_metadata,
|
||||
# samsung_genai); the full-detail scans use scan_head's 1 MB default. Sharing
|
||||
# one constant also keeps both call sites on the same memoized cache entry.
|
||||
_QUICK_SCAN_BYTES = 512 * 1024
|
||||
|
||||
# ── Known AI metadata keys ──────────────────────────────────────────
|
||||
|
||||
AI_METADATA_KEYS: frozenset[str] = frozenset(
|
||||
@@ -306,7 +311,7 @@ def has_ai_metadata(image_path: Path) -> bool:
|
||||
|
||||
# Binary scan covers C2PA (PNG caBX, JPEG APP11, AVIF/HEIF/JXL uuid boxes)
|
||||
# and IPTC AI markers in XMP. First 512KB (plus late ISOBMFF provenance boxes).
|
||||
data = scan_head(image_path, 512 * 1024)
|
||||
data = scan_head(image_path, _QUICK_SCAN_BYTES)
|
||||
if c2pa_marker_in(data):
|
||||
return True
|
||||
if any(marker in data for marker in AIGC_MARKERS):
|
||||
@@ -338,13 +343,16 @@ def aigc_label(image_path: Path) -> dict[str, str] | None:
|
||||
found by a container-agnostic raw-byte scan (PNG/JPEG/WebP alike); and
|
||||
- a raw-JSON ``{"AIGC":{...}}`` block with no namespace, as embedded in JPEG
|
||||
EXIF (UserComment) by some China-served generators, brace-matched from the
|
||||
scan head.
|
||||
scan head; and
|
||||
- a bare ``AIGC{...}`` blob (the label glued straight to its JSON, no
|
||||
``"AIGC":`` key wrapper) embedded in a JPEG APP segment near the JFIF
|
||||
header by some China-served generators.
|
||||
|
||||
Returns the decoded JSON (e.g. ``{"Label": "1", "ContentProducer": ...}``)
|
||||
or None. The generic forms (the PNG-chunk key ``AIGC`` and the bare
|
||||
``{"AIGC":...}`` object) are accepted only if they carry at least one known
|
||||
TC260 field (``_TC260_FIELDS``); the namespaced XMP element is unambiguous,
|
||||
so any JSON object is accepted.
|
||||
or None. The generic forms (the PNG-chunk key ``AIGC``, the bare
|
||||
``{"AIGC":...}`` object, and the bare ``AIGC{...}`` blob) are accepted only
|
||||
if they carry at least one known TC260 field (``_TC260_FIELDS``); the
|
||||
namespaced XMP element is unambiguous, so any JSON object is accepted.
|
||||
"""
|
||||
import html
|
||||
import json
|
||||
@@ -388,24 +396,76 @@ def aigc_label(image_path: Path) -> dict[str, str] | None:
|
||||
body = match.group(1) if match.group(1) is not None else match.group(2)
|
||||
return _parse(html.unescape(body.decode("utf-8", "replace")), require_tc260_field=False)
|
||||
|
||||
# Raw-JSON {"AIGC":{...}} block (no namespace), as written into JPEG EXIF
|
||||
# (UserComment) by some China-served generators -- the PNG-chunk and XMP
|
||||
# paths above both miss it. The bytes pre-check keeps the common (no-AIGC)
|
||||
# path off the full-buffer decode; raw_decode then brace-matches the inner
|
||||
# object (respecting nested braces / quoted strings) and `_parse` applies the
|
||||
# same dict coercion + TC260-field gate as the generic PNG-chunk path.
|
||||
if b'"AIGC"' in data:
|
||||
text = data.decode("latin-1")
|
||||
brace = text.find("{", text.find('"AIGC"') + len('"AIGC"'))
|
||||
if brace != -1:
|
||||
try:
|
||||
_, end = json.JSONDecoder().raw_decode(text, brace)
|
||||
except ValueError:
|
||||
return None
|
||||
return _parse(text[brace:end], require_tc260_field=True)
|
||||
# Generic raw-JSON forms the PNG-chunk and XMP paths above both miss, each
|
||||
# gated on a TC260 field: the ``"AIGC":{...}`` key wrapper (as written into
|
||||
# JPEG EXIF UserComment) and the bare ``AIGC{...}`` blob (the label glued
|
||||
# straight to its JSON, no key wrapper, in a JPEG APP segment near the JFIF
|
||||
# header). `raw_decode` brace-matches the inner object (respecting nested
|
||||
# braces / quoted strings); `_parse` applies the same dict coercion + TC260
|
||||
# gate as the PNG-chunk path. A non-matching hit (no TC260 field, or an
|
||||
# undecodable brace) must FALL THROUGH to the next form, never short-circuit:
|
||||
# a quoted ``"AIGC"`` can appear later in an XMP packet while the real label
|
||||
# is a bare ``AIGC{...}`` blob earlier in the file, so an unconditional return
|
||||
# on the quoted form would shadow the bare form.
|
||||
text = data.decode("latin-1")
|
||||
for needle in ('"AIGC"', "AIGC{"):
|
||||
start = text.find(needle)
|
||||
if start == -1:
|
||||
continue
|
||||
# First brace at/after the needle: the object brace for ``"AIGC":{`` and
|
||||
# the glued brace (at start+4) for the bare ``AIGC{`` -- one search covers both.
|
||||
brace = text.find("{", start)
|
||||
if brace == -1:
|
||||
continue
|
||||
try:
|
||||
_, end = json.JSONDecoder().raw_decode(text, brace)
|
||||
except ValueError:
|
||||
continue
|
||||
if result := _parse(text[brace:end], require_tc260_field=True):
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
# C2PA "Durable Content Credentials" manifest repositories (C2PA 2.4). When the
|
||||
# embedded manifest is stripped, an XMP ``dcterms:provenance`` URL can still point
|
||||
# at the vendor's cloud manifest store, from which the credentials are recoverable
|
||||
# server-side via the file's soft binding. Host -> vendor label. Verified on real
|
||||
# files: Adobe's Content Authenticity cloud store.
|
||||
_C2PA_MANIFEST_REPOSITORIES: tuple[tuple[bytes, str], ...] = (
|
||||
(b"cai-manifests.adobe.com", "Adobe Content Authenticity"),
|
||||
)
|
||||
|
||||
|
||||
def c2pa_cloud_manifest_in(data: bytes) -> str | None:
|
||||
"""Return a C2PA cloud-manifest vendor label if ``data`` carries an XMP
|
||||
``dcterms:provenance`` pointer to a known manifest repository, else None.
|
||||
|
||||
The shared byte-scan (mirroring ``soft_binding_vendors_in``), so a caller that
|
||||
already holds the scan head (``identify``) reuses it instead of re-reading.
|
||||
"""
|
||||
if b"dcterms:provenance" not in data:
|
||||
return None
|
||||
for host, vendor in _C2PA_MANIFEST_REPOSITORIES:
|
||||
if host in data:
|
||||
return vendor
|
||||
return None
|
||||
|
||||
|
||||
def c2pa_cloud_manifest(image_path: Path) -> str | None:
|
||||
"""Return a C2PA cloud-manifest vendor label if the file carries only an XMP
|
||||
``dcterms:provenance`` pointer to a manifest repository (C2PA 2.4 Durable
|
||||
Content Credentials), else None.
|
||||
|
||||
This fires on the laundering case where the *embedded* manifest was stripped
|
||||
but the XMP cloud reference survives, so the Content Credentials remain
|
||||
recoverable server-side. It is provenance, NOT an AI assertion: the cloud
|
||||
manifest can describe a human edit as easily as an AI generation, and reading
|
||||
its contents needs a network fetch we do not do. ``identify`` surfaces it as a
|
||||
provenance signal without setting ``is_ai_generated``.
|
||||
"""
|
||||
return c2pa_cloud_manifest_in(scan_head(image_path, _QUICK_SCAN_BYTES))
|
||||
|
||||
|
||||
def huggingface_job(image_path: Path) -> str | None:
|
||||
"""Return the HuggingFace job id if the image carries an ``hf-job-id`` PNG
|
||||
text chunk, else None.
|
||||
@@ -453,7 +513,7 @@ def samsung_genai(image_path: Path) -> int | None:
|
||||
gated on the ``PhotoEditor_Re_Edit_Data`` container so an incidental
|
||||
``genAIType`` token cannot false-positive.
|
||||
"""
|
||||
head = scan_head(image_path, 512 * 1024)
|
||||
head = scan_head(image_path, _QUICK_SCAN_BYTES)
|
||||
if _SAMSUNG_EDITOR_MARKER not in head:
|
||||
return None
|
||||
m = _SAMSUNG_GENAI_RE.search(head)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Shared availability guard for optional dependencies.
|
||||
|
||||
A bare ``importlib.util.find_spec(name) is not None`` check lies when only a
|
||||
leftover data directory exists in site-packages: e.g. ``trustmark`` downloads
|
||||
model weights into its own package dir, so after the package is uninstalled
|
||||
(``uv sync`` pruning an extra) a ``trustmark/models/`` remnant survives and
|
||||
``find_spec`` resolves it to a namespace-package spec (``loader is None``)
|
||||
while the actual import fails. Every ``is_available()`` guard routes through
|
||||
``module_available`` so a pure namespace package counts as absent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
|
||||
|
||||
def module_available(*names: str) -> bool:
|
||||
"""True when every named module resolves to a real, importable package.
|
||||
|
||||
A spec with ``loader is None`` is a pure namespace package -- for our
|
||||
optional deps that means a stale directory remnant, not an installed
|
||||
package -- so it is treated as not available.
|
||||
"""
|
||||
for name in names:
|
||||
spec = importlib.util.find_spec(name)
|
||||
if spec is None or spec.loader is None:
|
||||
return False
|
||||
return True
|
||||
@@ -82,9 +82,9 @@ def erase_cv2(
|
||||
|
||||
def lama_available() -> bool:
|
||||
"""True when the optional LaMa-ONNX backend can run (onnxruntime installed)."""
|
||||
import importlib.util
|
||||
from .optional_deps import module_available
|
||||
|
||||
return importlib.util.find_spec("onnxruntime") is not None
|
||||
return module_available("onnxruntime")
|
||||
|
||||
|
||||
def _get_lama_session() -> object:
|
||||
|
||||
@@ -40,9 +40,9 @@ _tm_lock = threading.Lock()
|
||||
|
||||
def is_available() -> bool:
|
||||
"""True if the optional ``trustmark`` package is installed."""
|
||||
import importlib.util
|
||||
from .optional_deps import module_available
|
||||
|
||||
return importlib.util.find_spec("trustmark") is not None
|
||||
return module_available("trustmark")
|
||||
|
||||
|
||||
def _decoder() -> Any:
|
||||
|
||||
@@ -8,8 +8,8 @@ The DEFAULT upscaler stays Lanczos (cv2, no deps); this is opt-in via the ``esrg
|
||||
extra and feeds the ``--upscaler esrgan`` path. ``spandrel`` is a pure model-loader
|
||||
(MIT) with NO basicsr dependency -- it pulls only torch/torchvision/safetensors/numpy/
|
||||
einops -- so it sidesteps the basicsr / ``torchvision.transforms.functional_tensor``
|
||||
breakage that the ``restore`` (GFPGAN) extra has to shim. Real-ESRGAN weights are
|
||||
BSD-3-Clause.
|
||||
breakage that the retired ``restore`` (GFPGAN) extra had to shim. Real-ESRGAN weights
|
||||
are BSD-3-Clause.
|
||||
|
||||
CPU works but is slow on large inputs, so this is meant for the pre-diffusion upscale of
|
||||
SMALL inputs (and the GPU worker). On a memory-constrained host it is a no-op (the extra
|
||||
@@ -21,7 +21,6 @@ is absent), and the caller falls back to Lanczos.
|
||||
# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportUnknownParameterType=false, reportMissingTypeArgument=false, reportMissingTypeStubs=false, reportMissingImports=false, reportArgumentType=false, reportAssignmentType=false, reportReturnType=false, reportCallIssue=false, reportIndexIssue=false, reportOperatorIssue=false, reportAttributeAccessIssue=false, reportPrivateImportUsage=false
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -45,7 +44,9 @@ _lock = threading.Lock()
|
||||
|
||||
def is_available() -> bool:
|
||||
"""True if the ``esrgan`` extra (spandrel + torch) is importable."""
|
||||
return importlib.util.find_spec("spandrel") is not None and importlib.util.find_spec("torch") is not None
|
||||
from .optional_deps import module_available
|
||||
|
||||
return module_available("spandrel", "torch")
|
||||
|
||||
|
||||
def _model_cache_path() -> Path:
|
||||
|
||||
@@ -389,6 +389,19 @@ class TestAllCommand:
|
||||
assert result.exit_code == 0, result.output
|
||||
mock_best.assert_called() # the registry auto-detector drove the visible pass
|
||||
|
||||
def test_all_loud_warning_and_nonzero_exit_when_gpu_missing(self, runner, sample_png, tmp_path):
|
||||
"""Regression (#14/#47): when the GPU extra is absent the invisible step is
|
||||
skipped, but the output still looks processed -- the run must fail loudly
|
||||
(prominent banner + non-zero exit) so a skipped SynthID pass is not mistaken
|
||||
for a clean result. The output file is still written (visible + metadata)."""
|
||||
output = tmp_path / "clean.png"
|
||||
with patch("remove_ai_watermarks.invisible_engine.is_available", return_value=False):
|
||||
result = runner.invoke(main, ["all", str(sample_png), "-o", str(output)])
|
||||
assert result.exit_code != 0, result.output
|
||||
assert "NOT removed" in result.output
|
||||
assert "remove-ai-watermarks[gpu]" in result.output
|
||||
assert output.exists() # visible + metadata still produced a file
|
||||
|
||||
def test_all_preserves_rgba_across_invisible_step(self, runner, tmp_path):
|
||||
"""Regression: ``all`` must keep transparency even when the invisible
|
||||
step writes a 3-channel result (as the real diffusion engine does).
|
||||
|
||||
@@ -772,6 +772,31 @@ class TestIntegrityClashesHelper:
|
||||
# must NOT raise a clash.
|
||||
assert _integrity_clashes({}, "Leica (camera, C2PA capture)", camera_has_ai_marker=False) == []
|
||||
|
||||
def test_pixel_generative_edit_same_manifest_no_clash(self):
|
||||
# A Google Pixel that BOTH captures and runs on-device generative AI
|
||||
# (Magic Editor / Pixel Studio) records the capture and the AI edit in
|
||||
# ONE C2PA manifest -- the AI vendor is named only from that same
|
||||
# manifest (c2pa / synthid), independent of nothing. That is a legitimate
|
||||
# edit chain, NOT a camera-vs-AI contradiction, so rule 2 must stay quiet.
|
||||
assert (
|
||||
_integrity_clashes(
|
||||
{"c2pa": "Google", "synthid": "Google"},
|
||||
"Google Pixel (camera, C2PA capture)",
|
||||
camera_has_ai_marker=True,
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
def test_camera_plus_independent_ai_marker_still_clashes(self):
|
||||
# But a camera capture next to an AI marker from a genuinely INDEPENDENT
|
||||
# source (EXIF/XMP generator, TC260 AIGC, ...) is still a laundering tell.
|
||||
clashes = _integrity_clashes(
|
||||
{"c2pa": "Google", "aigc": "China AIGC (TC260)"},
|
||||
"Google Pixel (camera, C2PA capture)",
|
||||
camera_has_ai_marker=True,
|
||||
)
|
||||
assert any("Camera-capture" in c for c in clashes)
|
||||
|
||||
|
||||
class TestIntegrityClashEndToEnd:
|
||||
def _c2pa_jpeg(self, tmp_path: Path, blob: bytes) -> Path:
|
||||
@@ -806,6 +831,22 @@ class TestIntegrityClashEndToEnd:
|
||||
assert r.platform == "Google Pixel (camera, C2PA capture)"
|
||||
assert any("Camera-capture C2PA credentials" in c and "AI-generation markers" in c for c in r.integrity_clashes)
|
||||
|
||||
def test_pixel_generative_edit_no_clash(self, tmp_path: Path):
|
||||
# A real Google Pixel generative edit (Magic Editor / Pixel Studio) signs
|
||||
# ONE manifest carrying both the Pixel Camera capture and a Google
|
||||
# Generative AI edit (trainedAlgorithmicMedia + "Applied imperceptible
|
||||
# SynthID watermark"). The AI marker lives in the SAME manifest as the
|
||||
# device, so it is an edit chain, not a camera-vs-AI contradiction.
|
||||
path = self._c2pa_jpeg(
|
||||
tmp_path,
|
||||
b"Pixel Camera ... Created by Pixel Camera ... computationalCapture ... "
|
||||
b"Created by Google Generative AI ... trainedAlgorithmicMedia ... "
|
||||
b"Applied imperceptible SynthID watermark",
|
||||
)
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is True
|
||||
assert r.integrity_clashes == []
|
||||
|
||||
def test_clash_serializes_to_json(self, tmp_path: Path):
|
||||
path = self._c2pa_jpeg(tmp_path, b"OpenAI ... trainedAlgorithmicMedia ... TC260:AIGC label")
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
@@ -790,6 +790,42 @@ class TestAIGCLabel:
|
||||
def test_has_ai_metadata_detects_raw_json_exif_form(self, tmp_path: Path):
|
||||
assert has_ai_metadata(self._aigc_exif_jpeg(tmp_path))
|
||||
|
||||
def _aigc_bare_jpeg(self, tmp_path: Path, producer: str = "00119144030008867405X210002") -> Path:
|
||||
"""Some China-served generators glue the TC260 label straight to its JSON
|
||||
as a bare ``AIGC{...}`` blob inside a JPEG APP segment (no ``"AIGC":``
|
||||
key wrapper, no PNG chunk, no namespaced XMP) -- seen near the JFIF
|
||||
header on real 2026-06 downloads."""
|
||||
p = tmp_path / "aigc_bare.jpg"
|
||||
Image.new("RGB", (32, 32)).save(p)
|
||||
raw = p.read_bytes()
|
||||
blob = b'AIGC{"Label":"1","ContentProducer":"' + producer.encode() + b'","ProduceID":"8F995586"}'
|
||||
segment = b"\xff\xe9" + (len(blob) + 2).to_bytes(2, "big") + blob # APP9
|
||||
p.write_bytes(raw[:2] + segment + raw[2:]) # splice after SOI
|
||||
return p
|
||||
|
||||
def test_parses_bare_aigc_jpeg_segment_form(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import aigc_label
|
||||
|
||||
info = aigc_label(self._aigc_bare_jpeg(tmp_path))
|
||||
assert info is not None
|
||||
assert info["Label"] == "1"
|
||||
assert info["ContentProducer"] == "00119144030008867405X210002"
|
||||
|
||||
def test_has_ai_metadata_detects_bare_aigc_jpeg_form(self, tmp_path: Path):
|
||||
assert has_ai_metadata(self._aigc_bare_jpeg(tmp_path))
|
||||
|
||||
def test_bare_aigc_without_tc260_field_ignored(self, tmp_path: Path):
|
||||
"""A bare ``AIGC{...}`` blob with no TC260 field must not false-positive."""
|
||||
from remove_ai_watermarks.metadata import aigc_label
|
||||
|
||||
p = tmp_path / "bare_unrelated.jpg"
|
||||
Image.new("RGB", (32, 32)).save(p)
|
||||
raw = p.read_bytes()
|
||||
blob = b'AIGC{"unrelated":"value"}'
|
||||
segment = b"\xff\xe9" + (len(blob) + 2).to_bytes(2, "big") + blob
|
||||
p.write_bytes(raw[:2] + segment + raw[2:])
|
||||
assert aigc_label(p) is None
|
||||
|
||||
def test_raw_json_without_tc260_field_ignored(self, tmp_path: Path):
|
||||
"""A bare ``{"AIGC":{...}}`` object with no TC260 field must not fire."""
|
||||
import json
|
||||
@@ -1185,3 +1221,49 @@ class TestFfmpegMetadataStrip:
|
||||
remove_ai_metadata(src, out)
|
||||
assert out.exists()
|
||||
assert b"Suno AI generated" not in out.read_bytes() # tag stripped, audio kept
|
||||
|
||||
|
||||
class TestC2paCloudManifest:
|
||||
"""C2PA 2.4 Durable Content Credentials: an XMP dcterms:provenance pointer to
|
||||
a vendor cloud manifest store survives when the embedded manifest is stripped."""
|
||||
|
||||
def _cloud_png(self, tmp_path: Path, host: bytes = b"cai-manifests.adobe.com") -> Path:
|
||||
xmp = (
|
||||
b'<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?><x:xmpmeta xmlns:x="adobe:ns:meta/">'
|
||||
b'<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">'
|
||||
b'<rdf:Description rdf:about="" xmlns:dcterms="http://purl.org/dc/terms/" '
|
||||
b'dcterms:provenance="https://' + host + b'/manifests/urn-c2pa-abc123"> </rdf:Description>'
|
||||
b'</rdf:RDF></x:xmpmeta><?xpacket end="w"?>'
|
||||
)
|
||||
p = tmp_path / "cloud.png"
|
||||
img = Image.new("RGB", (16, 16))
|
||||
meta = PngInfo()
|
||||
meta.add_itxt("XML:com.adobe.xmp", xmp.decode("latin-1"))
|
||||
img.save(p, pnginfo=meta)
|
||||
return p
|
||||
|
||||
def test_detects_adobe_cloud_manifest(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import c2pa_cloud_manifest
|
||||
|
||||
assert c2pa_cloud_manifest(self._cloud_png(tmp_path)) == "Adobe Content Authenticity"
|
||||
|
||||
def test_no_provenance_pointer_is_none(self, tmp_clean_png: Path):
|
||||
from remove_ai_watermarks.metadata import c2pa_cloud_manifest
|
||||
|
||||
assert c2pa_cloud_manifest(tmp_clean_png) is None
|
||||
|
||||
def test_unknown_host_is_none(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import c2pa_cloud_manifest
|
||||
|
||||
# A dcterms:provenance pointer to an unrecognized host is not attributed.
|
||||
assert c2pa_cloud_manifest(self._cloud_png(tmp_path, host=b"manifests.example.com")) is None
|
||||
|
||||
def test_cloud_manifest_does_not_assert_ai(self, tmp_path: Path):
|
||||
# Provenance only -- a cloud manifest can describe a human edit, so the
|
||||
# verdict must stay 'unknown', not 'AI-generated'.
|
||||
from remove_ai_watermarks.identify import identify
|
||||
|
||||
r = identify(self._cloud_png(tmp_path), check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is None
|
||||
assert any("Durable Content Credentials" in w for w in r.watermarks)
|
||||
assert any(s.name == "c2pa_cloud" for s in r.signals)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Tests for the shared optional-dependency availability guard."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
|
||||
from remove_ai_watermarks import optional_deps
|
||||
|
||||
|
||||
def _fake_find_spec(specs: dict[str, importlib.machinery.ModuleSpec | None]):
|
||||
def find_spec(name: str) -> importlib.machinery.ModuleSpec | None:
|
||||
return specs[name]
|
||||
|
||||
return find_spec
|
||||
|
||||
|
||||
def _real_spec(name: str) -> importlib.machinery.ModuleSpec:
|
||||
spec = importlib.util.find_spec(name)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
return spec
|
||||
|
||||
|
||||
class TestModuleAvailable:
|
||||
def test_installed_module_is_available(self):
|
||||
assert optional_deps.module_available("json") is True
|
||||
|
||||
def test_missing_module_is_not_available(self, monkeypatch):
|
||||
monkeypatch.setattr(importlib.util, "find_spec", _fake_find_spec({"ghost": None}))
|
||||
assert optional_deps.module_available("ghost") is False
|
||||
|
||||
def test_namespace_package_remnant_is_not_available(self, monkeypatch):
|
||||
# A leftover data dir in site-packages (e.g. trustmark/models/ surviving
|
||||
# an uninstall) resolves to a namespace-package spec with loader=None;
|
||||
# the guard must not report it as installed.
|
||||
ns_spec = importlib.machinery.ModuleSpec("trustmark", loader=None, is_package=True)
|
||||
assert ns_spec.loader is None
|
||||
monkeypatch.setattr(importlib.util, "find_spec", _fake_find_spec({"trustmark": ns_spec}))
|
||||
assert optional_deps.module_available("trustmark") is False
|
||||
|
||||
def test_any_namespace_member_fails_the_conjunction(self, monkeypatch):
|
||||
ns_spec = importlib.machinery.ModuleSpec("spandrel", loader=None, is_package=True)
|
||||
specs = {"spandrel": ns_spec, "torch": _real_spec("json")}
|
||||
monkeypatch.setattr(importlib.util, "find_spec", _fake_find_spec(specs))
|
||||
assert optional_deps.module_available("torch", "spandrel") is False
|
||||
|
||||
def test_all_real_members_are_available(self):
|
||||
assert optional_deps.module_available("json", "logging") is True
|
||||
|
||||
|
||||
class TestGuardsUseSharedHelper:
|
||||
def test_trustmark_is_available_rejects_namespace_remnant(self, monkeypatch):
|
||||
from remove_ai_watermarks import trustmark_detector
|
||||
|
||||
ns_spec = importlib.machinery.ModuleSpec("trustmark", loader=None, is_package=True)
|
||||
monkeypatch.setattr(importlib.util, "find_spec", _fake_find_spec({"trustmark": ns_spec}))
|
||||
assert trustmark_detector.is_available() is False
|
||||
Reference in new issue
Block a user