mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-18 15:31:06 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df94f2c908 | ||
|
|
b2e46ee8db | ||
|
|
b03a65bcb3 | ||
|
|
f2f840197e | ||
|
|
242d78957b | ||
|
|
78426aeef7 | ||
|
|
d914e3ac08 | ||
|
|
c903346a6b | ||
|
|
7cf067d3a5 | ||
|
|
15cae6fc91 | ||
|
|
d3a042209c | ||
|
|
1386708dec | ||
|
|
67665c5cea | ||
|
|
fc54f25948 | ||
|
|
f020115ca2 | ||
|
|
62be1ef456 | ||
|
|
4d7c2b458e | ||
|
|
96b8ab9ae2 | ||
|
|
5b30f0169a | ||
|
|
003dabfae4 | ||
|
|
4c090c5ae2 | ||
|
|
8c025985f7 | ||
|
|
5f1dea95f4 | ||
|
|
00d9771c6a | ||
|
|
de03748849 | ||
|
|
1cfdf192d8 |
No files matched your search
@@ -6,7 +6,7 @@
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Agent skill and Claude Code plugin for the remove-ai-watermarks CLI.",
|
||||
"version": "1.0.5"
|
||||
"version": "1.0.8"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"WebFetch",
|
||||
"WebSearch"
|
||||
]
|
||||
},
|
||||
"enabledPlugins": {
|
||||
"pyright-lsp@claude-plugins-official": true,
|
||||
"code-simplifier@claude-plugins-official": true,
|
||||
"claude-md-management@claude-plugins-official": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
name: signal-discovery
|
||||
description: Discover missing AI provenance and watermark coverage in the remove-ai-watermarks library from fresh or historical corpora, then validate detection and removal parity. Use for corpus gap mining, periodic signal audits, new-provider discovery, or deciding whether old retained data must be replayed. Do not use for ordinary per-file identification or removal.
|
||||
---
|
||||
|
||||
# Signal discovery
|
||||
|
||||
Work from the `remove-ai-watermarks` repository root. This skill discovers and
|
||||
closes library coverage gaps. Implementation, tests, fixtures, and
|
||||
documentation changes belong in this library; do not patch a downstream
|
||||
consumer to compensate for a detector or cleaner gap.
|
||||
|
||||
Treat retained inputs as sensitive and read-only. Keep reports, contact sheets,
|
||||
sidecars, and derived samples under the library's gitignored `.local-eval/`.
|
||||
Never commit a retained input or an artifact derived from one.
|
||||
Use only publication-cleared or synthetic files for tracked fixtures.
|
||||
|
||||
Before acting, read:
|
||||
|
||||
- `CLAUDE.md` and `.claude/rules/development.md`;
|
||||
- `docs/verification-plan.md`;
|
||||
- `docs/supported-signals.md` and `docs/watermarking-landscape.md`;
|
||||
- [references/coverage-and-cadence.md](references/coverage-and-cadence.md).
|
||||
|
||||
## Choose the run
|
||||
|
||||
Do not call `corpus_gap_scan.py` a complete signal audit. Select every mode the
|
||||
question requires:
|
||||
|
||||
1. **Fresh discovery** finds new providers, serializations, labels, and formats
|
||||
in objects added since the preceding review.
|
||||
2. **Historical replay** reruns current code on the complete retained history.
|
||||
Use it after detection-path or format-support changes and on the periodic
|
||||
backstop defined in the reference.
|
||||
3. **Removal parity** proves that every detected removable signal is cleared
|
||||
without violating the signal family's fidelity invariant.
|
||||
4. **Calibration or provider oracle** is required where the corpus has no
|
||||
self-evident answer key. Never treat a local negative as proof that a
|
||||
proprietary watermark is absent.
|
||||
|
||||
Start by inventorying the current code, not a remembered list. Enumerate image
|
||||
and video visible-mark registries, metadata readers and strippers, local
|
||||
invisible decoders, explicit invisible vendor routes, supported containers, and
|
||||
the stable signal families emitted by `identify`. Reconcile that inventory with
|
||||
the coverage matrix before interpreting any green result. A hand-written marker
|
||||
list or report family map that omits a current registry member is itself a
|
||||
finding.
|
||||
|
||||
## Run protocol
|
||||
|
||||
1. Record the library commit, version, enabled extras, corpus date range, and
|
||||
exact commands in an untracked run directory. Preserve the pre-run Git
|
||||
status so unrelated work is not attributed to the audit.
|
||||
2. Refresh the local retained corpus through its owning, external collection
|
||||
process when the task calls for fresh data. Use an explicit gitignored
|
||||
destination, do not delete local objects, and finish with a zero-delta pass.
|
||||
Fetch paired outputs only for a removal or complaint investigation.
|
||||
3. Run the applicable fresh and historical discovery modes from the reference.
|
||||
Use `corpus_gap_scan.py --since YYYY-MM-DD` for the overlapping fresh window
|
||||
and omit `--since` for the complete history. Report counts per input shape
|
||||
and per outcome; an example does not establish coverage.
|
||||
4. Triage every candidate into one of: parser/serialization gap, missing vendor
|
||||
attribution, new visible mark, possible invisible signal, format/read error,
|
||||
expected non-AI provenance, or false candidate. Preserve unresolved items as
|
||||
unresolved.
|
||||
5. Reproduce an actionable gap with a failing test before changing code. Derive
|
||||
the smallest publication-safe fixture from a controlled source or construct
|
||||
it synthetically; do not move the retained file into a tracked path.
|
||||
6. Implement the fix in the library subsystem that owns it. Update the library
|
||||
registry, parser, remover, public docs, and published agent skill when their
|
||||
contracts move. Updating a downstream consumer to take a released version is
|
||||
a later, separately scoped task.
|
||||
7. Rerun the affected focused audit and the complete historical regression
|
||||
mode. A detection change must name the files expected to move before the
|
||||
after-run; classify lost signals separately from intended new detections.
|
||||
8. Run removal parity for every newly detected removable signal. A quiet
|
||||
post-removal detector is necessary but not always sufficient: check
|
||||
pixel/stream invariants and visually inspect residual-prone visible marks.
|
||||
9. Run `bash maintain.sh`. Report evidence limits, incomplete rows, crashes,
|
||||
unsupported formats, and oracle-unavailable cohorts explicitly.
|
||||
|
||||
External oracle uploads are separate mutations. Use the `provider-oracles`
|
||||
skill only for a bounded, already-selected candidate, and obtain the required
|
||||
action-time upload approval. Do not adapt candidates against an oracle, retry
|
||||
automatically, or substitute one provider's verdict for another's.
|
||||
@@ -0,0 +1,57 @@
|
||||
# Coverage and cadence
|
||||
|
||||
Use this reference to choose the audit modes and decide whether current-only or
|
||||
historical data is sufficient.
|
||||
|
||||
## Coverage matrix
|
||||
|
||||
| Surface | Discovery evidence | Regression check | Removal proof |
|
||||
| --- | --- | --- | --- |
|
||||
| C2PA, EXIF, XMP, IPTC, PNG text, TC260, soft bindings, and container metadata | `scripts/corpus_gap_scan.py` plus review of unattributed and partially attributed reports | `scripts/sidecar_regression.py`; compare record-based and file-based identification when collection changes | `scripts/metadata_removal_audit.py`; signal absent after strip and decoded pixels/streams unchanged |
|
||||
| Registered visible image marks | `scripts/visible_positives.py`; metadata-derived vendor cohorts and blinded sheets for uncovered labels | `scripts/visible_eval.py`, registered examples, and full-corpus crossfire | `scripts/visible_removal_audit.py`; same-mark redetection plus visual residual review and fill-quality checks |
|
||||
| Candidate China-AIGC visible marks | `scripts/vendor_cohort_harvest.py` partitions by producer identity without using pixels; inspect full-width top and bottom sheets | `scripts/vendor_mark_calibrate.py` against independently labelled positives, clean negatives, and neighbouring marks | Register only after the candidate gate survives full-corpus crossfire; then use the normal visible-removal audit |
|
||||
| Registered visible video marks | Review provider-labelled or controlled complete clips, not isolated frames alone | Full-clip tests and the local real-provider audit described in `docs/verification-plan.md` | Preserve sequence, timing, duration, and audio; second-pass detection must be quiet |
|
||||
| Locally decodable invisible image marks, including open DWT-DCT and TrustMark | Positive-control-gated corpus or constructed carriers; a negative on an uncontrolled carrier is inconclusive | `scripts/watermark_benchmark.py` and the relevant decoder tests with the required extras installed | Re-run the same decoder on output and retain a positive control in the same run |
|
||||
| Proprietary image marks: Google SynthID, Microsoft InvisMark, Meta Content Seal | Provider provenance can select a cohort but is not a local pixel decode; use bounded provider-labelled candidates | Hash-bound provider-oracle batches when a new route, profile, or claimed signal is evaluated | Source-positive and output-negative verdicts from that same provider; record C2PA separately from the pixel watermark |
|
||||
| Invisible video and audio marks | Matched provider or open-model cohorts with codec and degradation controls | Benchmark-kernel audio/video rows and temporal aggregation checks | Preserve the non-target stream; use the matching oracle and a control-positive run |
|
||||
| Explicit vendor regeneration with no detectable local signal | This is a user assertion or research cohort, not a discovered signal | Verify routing and profile selection separately from detection | Quality and provider-oracle evidence may certify a cohort, but must not create a local detection claim |
|
||||
|
||||
`classify` is a pixel-origin guess, not provenance, and must not be counted as a
|
||||
new signal. `erase --region` is user-directed editing, not automatic watermark
|
||||
detection, and is outside this audit.
|
||||
|
||||
## Gaps the legacy scanner cannot settle alone
|
||||
|
||||
`corpus_gap_scan.py` is a metadata serialization probe. It reports raw markers,
|
||||
input shapes, identify errors, integrity clashes, unattributed results, and
|
||||
blind-marker candidates, but it still cannot settle other useful classes, so
|
||||
inspect them separately:
|
||||
|
||||
- a known generic signal with missing or wrong vendor attribution;
|
||||
- a newly coexisting marker on a file that already has another AI verdict;
|
||||
- an entirely novel marker absent from the scanner's hand-written list;
|
||||
- visible image or video labels;
|
||||
- proprietary invisible marks without a local decoder;
|
||||
- detection/removal parity failures;
|
||||
- decoder crashes, unsupported extensions, and incomplete reads;
|
||||
- drift between stored identify sidecars and the current library;
|
||||
- crossfire where a new detector fires on an older neighbouring cohort.
|
||||
|
||||
Marker matching must remain metadata-region bounded to avoid random hits in
|
||||
compressed pixels, but its inventory should be reconciled with the current
|
||||
metadata constants and registry on every audit. Do not infer completeness from
|
||||
`No gap candidates` until the other rows of the matrix also ran.
|
||||
|
||||
## When to replay old data
|
||||
|
||||
Yes, old data must be rerun. Current code can learn to parse an old
|
||||
serialization, and a new detector can crossfire on an old provider cohort even
|
||||
when no new object demonstrates either problem. The canonical intervals,
|
||||
change triggers, and sampling rules live in
|
||||
[`docs/verification-plan.md#signal-discovery-cadence`](../../../../docs/verification-plan.md#signal-discovery-cadence).
|
||||
|
||||
Keep dated reports immutable enough to compare runs. Record the library SHA and
|
||||
version, corpus minimum and maximum dates, input counts per media/format shape,
|
||||
enabled extras, command line, and output paths. Never commit operational corpus
|
||||
counts or retained identifiers; summarize only durable conclusions in the
|
||||
canonical research document.
|
||||
@@ -114,10 +114,100 @@ jobs:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Factory-rebuild the HuggingFace Space
|
||||
- name: Bump the Space pin, then factory-rebuild
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RAIW_HF_SPACE_TOKEN }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
repo=wiltodelta/raiw-hf-space
|
||||
workflow=pin-library.yml
|
||||
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "RAIW_HF_SPACE_TOKEN secret is not set; skipping the pin bump."
|
||||
else
|
||||
dispatched_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
gh workflow run "$workflow" \
|
||||
--repo "$repo" \
|
||||
--ref main \
|
||||
--raw-field library_version="$VERSION"
|
||||
|
||||
run_id=""
|
||||
for attempt in $(seq 1 30); do
|
||||
run_id=$(
|
||||
gh run list \
|
||||
--repo "$repo" \
|
||||
--workflow "$workflow" \
|
||||
--event workflow_dispatch \
|
||||
--limit 10 \
|
||||
--json databaseId,createdAt \
|
||||
| python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
dispatched_at = sys.argv[1]
|
||||
runs = json.load(sys.stdin)
|
||||
print(next(
|
||||
(run["databaseId"] for run in runs if run["createdAt"] >= dispatched_at),
|
||||
"",
|
||||
))
|
||||
' "$dispatched_at"
|
||||
)
|
||||
if [ -n "$run_id" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Space pin run not visible yet (attempt $attempt), waiting..."
|
||||
sleep 2
|
||||
done
|
||||
if [ -z "$run_id" ]; then
|
||||
echo "::error::The dispatched Space pin run did not appear"
|
||||
exit 1
|
||||
fi
|
||||
for attempt in $(seq 1 180); do
|
||||
read -r status conclusion <<< "$(
|
||||
gh run view "$run_id" \
|
||||
--repo "$repo" \
|
||||
--json status,conclusion \
|
||||
--jq '[.status, (.conclusion // "")] | @tsv'
|
||||
)"
|
||||
if [ "$status" = "completed" ]; then
|
||||
if [ "$conclusion" = "success" ]; then
|
||||
echo "Space pin run $run_id succeeded"
|
||||
break
|
||||
fi
|
||||
echo "::error::Space pin run $run_id completed with $conclusion"
|
||||
exit 1
|
||||
fi
|
||||
echo "Space pin run $run_id is $status (attempt $attempt), waiting..."
|
||||
sleep 5
|
||||
done
|
||||
if [ "${status:-}" != "completed" ]; then
|
||||
echo "::error::Space pin run $run_id did not finish in time"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting for the Space requirements pin to show $VERSION..."
|
||||
pinned=""
|
||||
for attempt in $(seq 1 40); do
|
||||
reqs=$(curl -fsSL \
|
||||
"https://huggingface.co/spaces/wiltodelta/remove-ai-watermarks/raw/main/requirements.txt" \
|
||||
|| true)
|
||||
pinned=$(printf '%s\n' "$reqs" \
|
||||
| sed -n 's/^remove-ai-watermarks==\([0-9][0-9a-zA-Z.]*\).*/\1/p' \
|
||||
| head -1)
|
||||
echo " attempt $attempt: Space pin=$pinned"
|
||||
if [ "$pinned" = "$VERSION" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 15
|
||||
done
|
||||
if [ "$pinned" != "$VERSION" ]; then
|
||||
echo "::error::Space requirements.txt still pinned to ${pinned:-<none>}, not $VERSION"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
python3 -m pip install --quiet huggingface_hub
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
@@ -217,7 +307,12 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for attempt in $(seq 1 180); do
|
||||
# 0.40.0's node sync finished after this waiter: 180 * 5s is 15 min,
|
||||
# and the node suite plus registry upload ran longer. Keep the
|
||||
# workflow wait, then accept a registry row for this library version
|
||||
# so a late-but-published node is not reported as a missing release.
|
||||
version="${{ needs.resolve.outputs.version }}"
|
||||
for attempt in $(seq 1 360); do
|
||||
read -r status conclusion <<< "$(
|
||||
gh run view "$run_id" \
|
||||
--repo "$repo" \
|
||||
@@ -235,5 +330,39 @@ jobs:
|
||||
echo "ComfyUI sync run $run_id is $status (attempt $attempt), waiting..."
|
||||
sleep 5
|
||||
done
|
||||
echo "::error::ComfyUI sync run $run_id did not finish in time"
|
||||
echo "ComfyUI sync run $run_id did not finish in time; checking the registry."
|
||||
python3 -m pip install --quiet packaging
|
||||
if curl -fsSL "https://api.comfy.org/nodes/remove-ai-watermarks/versions" \
|
||||
| VERSION="$version" python3 -c '
|
||||
import json, os, sys
|
||||
from packaging.requirements import InvalidRequirement, Requirement
|
||||
from packaging.utils import canonicalize_name
|
||||
from packaging.version import Version
|
||||
version = os.environ["VERSION"]
|
||||
target = Version(version)
|
||||
def requires_release(value):
|
||||
try:
|
||||
requirement = Requirement(value)
|
||||
except InvalidRequirement:
|
||||
return False
|
||||
return (
|
||||
canonicalize_name(requirement.name) == "remove-ai-watermarks"
|
||||
and requirement.url is None
|
||||
and requirement.marker is None
|
||||
and target in requirement.specifier
|
||||
and any(s.operator == ">=" and Version(s.version) == target for s in requirement.specifier)
|
||||
)
|
||||
matches = [
|
||||
v for v in json.load(sys.stdin)
|
||||
if any(requires_release(d) for d in (v.get("dependencies") or []))
|
||||
]
|
||||
if not matches:
|
||||
sys.exit(1)
|
||||
newest = max(matches, key=lambda v: v.get("createdAt") or "")
|
||||
print("%s %s" % (newest.get("version"), newest.get("status")))
|
||||
'; then
|
||||
echo "Registry already lists a node for $version; the wait timed out after publish."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::ComfyUI sync run $run_id did not finish in time and the registry has no node for $version"
|
||||
exit 1
|
||||
@@ -6,8 +6,9 @@ name: Publish photo classifier to Hugging Face
|
||||
# Hub model: wiltodelta/raiw-photo-classify
|
||||
# Secret: HF_TOKEN, write-role Hugging Face token (same secret Space sync uses).
|
||||
#
|
||||
# Weights are not in git. Full mode downloads them from a GitHub Release
|
||||
# of this repository, then uploads with the card from docs/photo-classify-hf/.
|
||||
# Weights are not in git. Full mode downloads the frozen PyTorch files from a
|
||||
# GitHub Release, exports the vision-only ONNX graph, then uploads everything
|
||||
# with the card from docs/photo-classify-hf/.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -52,6 +53,14 @@ jobs:
|
||||
gh release download "$TAG" --repo "${{ github.repository }}" --dir weights
|
||||
ls -l weights
|
||||
|
||||
- name: Export the vision-only ONNX graph
|
||||
if: ${{ github.event.inputs.mode == 'full' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
uv run --extra classify --with onnx python scripts/export_photo_classify_onnx.py \
|
||||
--checkpoint weights/clip-l-ft.pt \
|
||||
--out weights/clip-l-ft-vision-fp32.onnx
|
||||
|
||||
- name: Publish to Hugging Face
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
|
||||
@@ -94,7 +94,15 @@ jobs:
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install ffmpeg
|
||||
run: sudo apt-get update && sudo apt-get install --yes ffmpeg
|
||||
# Third-party apt sources baked into the runner image (google-chrome et
|
||||
# al.) can publish a broken index and fail `apt-get update`, while this
|
||||
# job only needs ffmpeg from Ubuntu's own repositories: drop every
|
||||
# non-Ubuntu source definition before updating.
|
||||
run: |
|
||||
sudo find /etc/apt/sources.list.d -mindepth 1 -maxdepth 1 \
|
||||
! -name 'ubuntu.sources' -delete
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes ffmpeg
|
||||
- name: Sync dev environment
|
||||
run: uv sync --frozen --extra dev
|
||||
- name: Run full-clip video test
|
||||
|
||||
@@ -59,9 +59,13 @@ jobs:
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
identities = []
|
||||
for path in Path("distribution-identities").glob("*/release-identity.json"):
|
||||
for path in Path("distribution-identities").rglob("release-identity.json"):
|
||||
item = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert path.parent.name == 'distributed-release-' + item['run_attempt'], "Artifact attempt mismatch"
|
||||
# download-artifact@v8 extracts the only matched artifact directly
|
||||
# into the path, with no artifact-name directory; several matches
|
||||
# each get one, so the attempt is name-checked only in that layout.
|
||||
if path.parent.name.startswith('distributed-release-'):
|
||||
assert path.parent.name == 'distributed-release-' + item['run_attempt'], "Artifact attempt mismatch"
|
||||
identities.append(item)
|
||||
assert identities, "No distributed release identity found"
|
||||
assert all(item["run_id"] == os.environ["DISTRIBUTION_RUN_ID"] for item in identities), "Distribution run mismatch"
|
||||
|
||||
@@ -31,7 +31,7 @@ Optional features and installation groups are documented in [`docs/installation.
|
||||
|
||||
`maintain.sh` runs dependency freshness and security checks, Ruff, Pyright scoped to `src/`, and the parallel test suite. Full-project Pyright is not the project gate because the ML dependency graph can exhaust Node memory.
|
||||
|
||||
The gate does not pass `--ignore-unfixed` to uv-secure, so a transitive CVE with no released fix stops it before Ruff, Pyright and the tests run, which a sibling repository that ignores unfixed findings would not do. The global rule applies: run and report those three separately. Scanner failures are fatal too. Triaged blockers and the recheck procedure: `Known security-gate blocks` in [`docs/development.md`](docs/development.md).
|
||||
The gate does not pass `--ignore-unfixed` to uv-secure, so a transitive CVE with no released fix stops it before Ruff, Pyright and the tests run. This matches the global rule: run and report those three separately. Scanner failures are fatal too. Triaged blockers and the recheck procedure: `Known security-gate blocks` in [`docs/development.md`](docs/development.md).
|
||||
|
||||
Command, gate, typing, and model-test invariants auto-load from [`.claude/rules/development.md`](.claude/rules/development.md). Environment recovery, CI behavior, and fixture policy live in [`docs/development.md`](docs/development.md).
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ its linked C2PA manifest; metadata stripping alone removes only the manifest.
|
||||
| Every production feature available on the active Python | `remove-ai-watermarks[all]` |
|
||||
|
||||
Lower-level and specialized extras include `pixels`, `heif`, `trustmark`,
|
||||
`migan`, `lama`, and `diffusion`. The
|
||||
`migan`, `lama`, `diffusion`, and `classify-onnx`. The
|
||||
[installation guide](docs/installation.md#feature-extras) documents their exact
|
||||
dependency composition, Python compatibility, and model requirements.
|
||||
|
||||
|
||||
@@ -63,16 +63,16 @@ the vendors' direct consumer services.
|
||||
`qwen-create-qwen-image-2.png` is the original 2048 x 2048 PNG downloaded from
|
||||
an authenticated Qwen Create generation on 2026-09-03. It was generated by
|
||||
`Qwen-Image 2.0` at 1:1 using the neutral still-life prompt recorded above.
|
||||
The current global consumer surface differs from the supported Chinese Qwen
|
||||
text mark: it places a small Qwen symbol at the bottom right, not
|
||||
`千问AI生成`. The existing Qwen text detector therefore correctly does not
|
||||
claim this new shape (measured strict confidence 0.178).
|
||||
The current global consumer surface differs from the Chinese Qwen text mark: it
|
||||
places a small Qwen symbol at the bottom right, not `千问AI生成`. The Qwen
|
||||
engine has separate text and symbol templates; the symbol path scores 0.848 on
|
||||
this fixture while the text path alone scores 0.178.
|
||||
|
||||
- SHA-256: `60c39ed8aedb081193c62289be549785e74d31fc8d071961003fabd74712429d`
|
||||
- Expected metadata signal: TC260 AIGC label
|
||||
- Expected `ContentProducer`: `001191440101MA9Y9T4H7A00000`
|
||||
- Expected visible result: no registered visible mark; the Qwen symbol is a
|
||||
newly observed, currently unsupported variant
|
||||
- Expected visible result: Qwen symbol detected and removable under the `qwen`
|
||||
registry key
|
||||
- License: repository license; generated and explicitly cleared by the
|
||||
maintainer
|
||||
- Qwen Create terms: <https://terms.alicdn.com/legal-agreement/terms/c_end_product_protocol/20260519200335649/20260519200335649.html>
|
||||
|
||||
+76
-28
@@ -41,47 +41,72 @@ Do these in order after the skill is on the default branch.
|
||||
Cursor, OpenCode, Pi, Codex, and Copilot have no separate skill store; they
|
||||
consume this repo or a copied `SKILL.md`.
|
||||
2. **SkillsMP.** Aggregates public GitHub `SKILL.md` files. No submit form. A
|
||||
search for `remove-ai-watermarks` was empty before the first public push.
|
||||
Neighboring `watermark` hits are PDF-add and detection skills, not removal.
|
||||
search for `remove-ai-watermarks` now returns this skill. Neighboring
|
||||
`watermark` hits are PDF-add and detection skills, not removal.
|
||||
3. **skills.sh.** No submit form. A row appears on the leaderboard only after
|
||||
people run `npx skills add`. The README install line is the seed, not a
|
||||
registration.
|
||||
people run `npx skills add`. The listing is live and has recorded real
|
||||
installs; the README install line remains the seed, not a registration.
|
||||
4. **Awesome-list PRs.** Hand-curated GitHub lists are how people browse skills
|
||||
today. One accurate PR per list, with the scope sentence intact.
|
||||
- `ComposioHQ/awesome-claude-skills`: submitted 2026-09-05 as PR 1841
|
||||
(Creative & Media); no maturity bar in its contribution rules.
|
||||
(Creative & Media); no maturity bar in its contribution rules. Rechecked
|
||||
2026-09-11: still open, CI green, no review comments, waiting on the
|
||||
list maintainers.
|
||||
- `heilcheng/awesome-agent-skills`: submitted 2026-09-09 as PR 483 under
|
||||
Community Skills > Media. The skill's three concrete input/output examples
|
||||
satisfy the list's working-example requirement. Rechecked 2026-09-11:
|
||||
still open; the only check failure is their Vercel deploy authorization,
|
||||
which a contributor cannot grant.
|
||||
- `VoltAgent/awesome-agent-skills`: DEFERRED. Its CONTRIBUTING rejects
|
||||
brand-new skills ("give your skill time to mature and gain users before
|
||||
submitting"). Re-attempt after the skill has real installs and a
|
||||
release that ships it (0.38.0+).
|
||||
- `VoltAgent/awesome-openclaw-skills`: DEFERRED. Entries are indexed via
|
||||
clawskills.sh, so the skill must be published to ClawHub first, and the
|
||||
list asks contributors not to submit skills "created 3 hours ago".
|
||||
Re-attempt after the ClawHub publish plus visible adoption.
|
||||
submitting"). The skill now has real installs and shipped releases, but
|
||||
adoption remains too small to make the list's maturity claim comfortably.
|
||||
- `VoltAgent/awesome-openclaw-skills`: DEFERRED on adoption only. Entries
|
||||
are indexed via clawskills.sh, and the ClawHub prerequisite in step 6 is
|
||||
now met. The list still requires proven real-world usage. Re-attempt once
|
||||
the ClawHub listing shows organic installs.
|
||||
5. **Claude community marketplace.** Form at
|
||||
[platform.claude.com/plugins/submit](https://platform.claude.com/plugins/submit)
|
||||
(individuals) or the claude.ai directory form (Team/Enterprise). Validate
|
||||
with `claude plugin validate` first. SUBMITTED 2026-09-07 for Claude Code
|
||||
only (strict validation passed; Apache 2.0; description carries the
|
||||
scope boundary; platform selection limited to Claude Code because Cowork
|
||||
was not tested). Pending review; the team may reach out by email. The
|
||||
official Anthropic marketplace is
|
||||
curated and has no application.
|
||||
was not tested). Pending review; the team may reach out by email.
|
||||
Rechecked 2026-09-11: `platform.claude.com/plugins` is 404, and
|
||||
`claude.ai/directory` is the MCP connector catalog, not plugins; a search
|
||||
there does not list this skill. Do not open a second submission. The
|
||||
official Anthropic marketplace is curated and has no application.
|
||||
6. **ClawHub.** `clawhub login` then `clawhub skill publish` from
|
||||
`skills/remove-ai-watermarks/`. PUBLISHED 2026-09-07 (1.0.3) and
|
||||
republished 2026-09-08 (1.0.4, identical content) — both sit in
|
||||
`pending.publication` with CLEAN moderation, succeeded scan frameworks,
|
||||
and null ClawScan / VirusTotal / Skillspector verdicts: a recurring
|
||||
ClawHub pipeline defect tracked as openclaw/clawhub#3643 (prior reports:
|
||||
#3624, #3466, #3351, #3284). Release-time publishing is automated through
|
||||
the `clawhub` job in `distribute.yml` (see
|
||||
[release-and-distribution.md](release-and-distribution.md)); do not bump
|
||||
the version again to re-queue until the issue resolves, and re-check the
|
||||
listing before any listing-dependent step such as the awesome-openclaw
|
||||
submission. The public catalog already has an add-watermark
|
||||
skill; this is the removal counterpart, framed as the user's own
|
||||
content. ClawHub disallows deception, impersonation, and fake-engagement
|
||||
install loops.
|
||||
`skills/remove-ai-watermarks/`. LISTED 2026-09-09: versions 1.0.5, 1.0.6,
|
||||
and 1.0.8, submitted by the `clawhub` job in `distribute.yml` during the
|
||||
0.39.0, 0.39.1, and 0.40.0 releases, are public. A green job proves only a
|
||||
submission, so read the catalog itself:
|
||||
`clawhub search remove-ai-watermarks` returns the row and
|
||||
`https://clawhub.ai/api/v1/skills/remove-ai-watermarks` reports
|
||||
`latestVersion.version` 1.0.8, while its `/versions` endpoint returns
|
||||
1.0.5, 1.0.6, and 1.0.8. The earlier 1.0.3 and 1.0.4 submissions never became
|
||||
public and do not appear in that version list, so they were bypassed rather
|
||||
than recovered. 1.0.7 was a post-0.39.1 bump that never reached ClawHub.
|
||||
The issue that tracked 1.0.3 and 1.0.4, openclaw/clawhub#3643, was closed
|
||||
by the reporter on 2026-09-09 once
|
||||
1.0.5 went through; nothing upstream was confirmed fixed, and it is the
|
||||
fifth report of this shape (#3624, #3466, #3351, #3284), so treat a stalled
|
||||
submission as recurring and file a fresh one rather than assuming the
|
||||
pipeline is repaired. Release-time publishing is automated (see
|
||||
[release-and-distribution.md](release-and-distribution.md)), and the job
|
||||
fires only on a published GitHub Release: a skill version bumped after a
|
||||
release reaches ClawHub with the next one. Do not publish by
|
||||
hand to close that gap. The listing records the license as MIT-0 while
|
||||
`SKILL.md` declares Apache-2.0, and that is not a listing error to repair:
|
||||
MIT-0 is a platform-wide constant in ClawHub's own client
|
||||
(`dist/schema/license.js`, and its version schema accepts only `"MIT-0"`
|
||||
or null), so every skill published there is offered under MIT No
|
||||
Attribution. Publishing to ClawHub means offering this skill on those
|
||||
terms in addition to the repository's Apache-2.0; do not try to edit the
|
||||
field, and do not read the mismatch as drift. The public catalog
|
||||
already has an add-watermark skill; this is the removal counterpart, framed
|
||||
as the user's own content. ClawHub disallows deception, impersonation, and
|
||||
fake-engagement install loops.
|
||||
7. **OpenAI ChatGPT / Codex plugin directory.** Skills-only plugins are a
|
||||
supported submission type at
|
||||
[platform.openai.com/plugins](https://platform.openai.com/plugins). This is
|
||||
@@ -93,6 +118,29 @@ Do these in order after the skill is on the default branch.
|
||||
execution, arbitrary file access, and optional hardware access, which the
|
||||
official guide routes through that product-specific review. Convert with
|
||||
[Submit your Claude Code plugin to OpenAI](https://developers.openai.com/plugins/guides/submit-claude-plugin).
|
||||
8. **AgenticSkills.** SUBMITTED 2026-09-09 as review issue 174 under AI/ML
|
||||
Development. The submission carries the user-owned-content boundary and
|
||||
links to the repository rather than hosting a separate copy. Rechecked
|
||||
2026-09-11: the public index still does not list it (`/skills/remove-ai-watermarks`
|
||||
is 404; search for `wiltodelta` returns 0 of 192). Their form promises a
|
||||
48-hour review; that window has passed. Issue 174 is not a public GitHub
|
||||
issue. Do not open a second submission. Followed up 2026-09-11 through
|
||||
their contact form as a listing correction.
|
||||
9. **AI Skills Catalog.** SUBMITTED 2026-09-09 as reference 2 under Video Media,
|
||||
with Codex as the primary agent and every supported harness marked compatible.
|
||||
Rechecked 2026-09-11: the public catalog at skillscatalog.ai shows 0 skills
|
||||
from 0 publishers, `/api/featured` returns 401, and `/api/auth/session`
|
||||
returns 500, so a stranger cannot confirm the listing. Do not open a
|
||||
duplicate while that initial review is pending. Once accepted, treat
|
||||
material skill-version updates as manual because the catalog stores
|
||||
compatible packages in addition to source metadata and does not promise
|
||||
that a GitHub change refreshes those packages. Use the update route
|
||||
supplied with acceptance; if none is supplied, contact the catalog or
|
||||
resubmit, then verify the live result.
|
||||
|
||||
The conditional update actions for every catalog are part of the canonical
|
||||
release checklist in
|
||||
[Release and distribution](release-and-distribution.md#agent-skill).
|
||||
|
||||
Do not treat Smithery or other MCP directories as skill catalogs. Do not open
|
||||
a second near-identical skill to farm installs. The conversion path that
|
||||
|
||||
+8
-5
@@ -78,6 +78,9 @@ no-signal metadata scan. Install the extra first:
|
||||
uv tool install --force "remove-ai-watermarks[classify]"
|
||||
```
|
||||
|
||||
The optional CPU ONNX backend is library-only
|
||||
(`classify_pixels(..., backend="onnx")`, extra `classify-onnx`), not a CLI flag.
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
@@ -465,10 +468,10 @@ remove-ai-watermarks invisible image.png -o clean.png --force
|
||||
### Choose a strength cohort
|
||||
|
||||
`--vendor` selects the cohort the default strength resolves from. `auto`
|
||||
(the default) derives it from provenance: the C2PA issuer for
|
||||
OpenAI/Google/Microsoft, and the standalone AI IPTC tag for Meta Content Seal
|
||||
(Muse output carries no C2PA; the tag is a standard code, so C2PA evidence
|
||||
always wins first). An explicit value both names the cohort for stripped files
|
||||
(the default) derives it from product-specific C2PA provenance for
|
||||
OpenAI/Google/Microsoft. The standalone AI IPTC tag is a shared standard and
|
||||
does not select Meta; use `--vendor meta` only when the source is independently
|
||||
known. An explicit value both names the cohort for stripped files
|
||||
and implies the scrub runs -- naming the cohort asserts the pixel watermark is
|
||||
present -- exactly like `--force` plus a measured floor:
|
||||
|
||||
@@ -486,7 +489,7 @@ The same option exists on `all` and `batch`, and as
|
||||
| `qwen-zimage` | Default. Qwen-Image-2512 global pass plus a SAM-masked Z-Image face stage |
|
||||
| `sdxl-zimage` | The same recipe and face stage on an SDXL global pass, at a higher denoise |
|
||||
| `chroma-zimage` | The same face stage on an Apache-2.0 Chroma1 global pass; explicit OpenAI use needs a higher floor than qwen-zimage |
|
||||
| `auto` | Pick the engine from provenance: chroma-zimage for Microsoft, qwen-zimage for OpenAI, Google, Meta, and unknown |
|
||||
| `auto` | Pick the engine from provenance: chroma-zimage for Microsoft, qwen-zimage for OpenAI, Google, unknown, and an explicitly selected Meta cohort |
|
||||
|
||||
**All four are CUDA-only.** There is no CPU or MPS profile for invisible-watermark
|
||||
removal. The former `controlnet`, `sdxl`, `qwen` and `default` profiles were removed
|
||||
|
||||
+19
-16
@@ -29,26 +29,29 @@ Do not add advisory ignores or suppress the scanner's exit status. Run and
|
||||
report Ruff, Pyright scoped to `src/`, and tests separately when security blocks
|
||||
the gate.
|
||||
|
||||
Rechecked on 2026-09-08 against PyPI release metadata and the advisory sources:
|
||||
Rechecked on 2026-09-10 against the locked versions and the advisory sources:
|
||||
|
||||
- `lightning`, pulled by the optional `trustmark` extra, remains affected by
|
||||
[PYSEC-2026-3624](https://github.com/pypa/advisory-database/blob/main/vulns/lightning/PYSEC-2026-3624.yaml)
|
||||
and its alias [GHSA-qqmf-gpg7-g8gw](https://github.com/advisories/GHSA-qqmf-gpg7-g8gw).
|
||||
These describe one checkpoint-loading vulnerability. The latest PyPI release
|
||||
is still affected; the advisory's `2022.6.15` fix entry is not a newer release
|
||||
on the supported 2.x line. The adapter selects TrustMark's supplied weights,
|
||||
rather than accepting a caller-supplied Lightning checkpoint.
|
||||
- `lightning` and `pytorch-lightning`, pulled by the optional `trustmark`
|
||||
extra, are pinned to 2.6.6. That release clears PYSEC-2026-3624 and
|
||||
PYSEC-2026-3967; do not reintroduce an older transitive resolution.
|
||||
- `accelerate`, required by the diffusion stack, is affected by
|
||||
[GHSA-4j2p-28q2-5m79](https://github.com/advisories/GHSA-4j2p-28q2-5m79).
|
||||
The sharded-checkpoint loader accepts unsafe `weight_map` paths; the advisory
|
||||
lists no patched release. The project has no direct call to
|
||||
`load_checkpoint_in_model` or `load_checkpoint_and_dispatch`, but this does
|
||||
not prove indirect model-loading paths are unaffected.
|
||||
[GHSA-4j2p-28q2-5m79](https://github.com/advisories/GHSA-4j2p-28q2-5m79)
|
||||
and PYSEC-2026-3804. The sharded-checkpoint loader accepts unsafe
|
||||
`weight_map` paths; neither advisory lists a patched release. 1.15.0 is
|
||||
on PyPI and is outside the advisory's `<= 1.14.0` range, but its
|
||||
`load_checkpoint_in_model` still joins `weight_map` entries with no
|
||||
containment check, and the sanitizer PRs
|
||||
([#4138](https://github.com/huggingface/accelerate/pull/4138),
|
||||
[#4214](https://github.com/huggingface/accelerate/pull/4214)) did not
|
||||
land. Do not bump solely to silence the scanner. The project has no
|
||||
direct call to `load_checkpoint_in_model` or
|
||||
`load_checkpoint_and_dispatch`, but this does not prove indirect
|
||||
model-loading paths are unaffected.
|
||||
|
||||
Before carrying either block forward, rerun `uvx uv-secure uv.lock` and check
|
||||
Before carrying the remaining block forward, rerun `uvx uv-secure uv.lock` and check
|
||||
PyPI for a fixed release. Upgrade a fixed package with
|
||||
`uv lock --upgrade-package <package>` and rerun the gate. Removing either
|
||||
current dependency would remove a supported optional feature, so that is not a
|
||||
`uv lock --upgrade-package <package>` and rerun the gate. Removing the current
|
||||
dependency would remove a supported optional feature, so that is not a
|
||||
maintenance-only repair.
|
||||
|
||||
Minor-only lock refreshes must retain each package's current major and lower
|
||||
|
||||
@@ -126,6 +126,7 @@ application actually uses:
|
||||
| `detect` | Open DWT-DCT detection for Stable Diffusion, SDXL, and FLUX | `pixels`, PyWavelets | No |
|
||||
| `trustmark` | Adobe TrustMark detection on Python 3.11-3.12 | trustmark | Yes |
|
||||
| `classify` | Metadata-free photo AI-versus-camera classifier plus gated provider | `pixels`, Torch, Transformers | Yes |
|
||||
| `classify-onnx` | Optional CPU-only ONNX vision runtime for `classify` | `classify`, ONNX Runtime | Yes |
|
||||
| `diffusion` | Torch and Diffusers runtime; video SynthID regeneration | `pixels`, Torch, Diffusers | Yes |
|
||||
| `migan` | MI-GAN ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
| `lama` | big-LaMa ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
@@ -152,12 +153,14 @@ flowchart LR
|
||||
heif
|
||||
trustmark
|
||||
classify --> pixels
|
||||
onnx["classify-onnx"] --> classify
|
||||
```
|
||||
|
||||
`heif`, `trustmark`, and `text-draft` are independent branches. Combine them
|
||||
explicitly with another feature when required. `text-draft` is excluded from
|
||||
`all` because it proposes unverified OCR annotations and is not a production
|
||||
removal path. The TrustMark branch remains available only on Python 3.11-3.12.
|
||||
removal path. `classify-onnx` is excluded from `all`; install it only for `backend="onnx"`.
|
||||
The TrustMark branch remains available only on Python 3.11-3.12.
|
||||
Its supported dependency floor includes releases requiring NumPy 1.x, which has
|
||||
no CPython 3.13 or 3.14 wheels; newer TrustMark releases relaxing that requirement
|
||||
do not by themselves widen this package's declared support range. The `all` bundle contains every production branch compatible with
|
||||
|
||||
@@ -388,7 +388,11 @@ OpenCV does not decode these formats in the project. `image_io.imread` falls
|
||||
back to Pillow with `pillow-heif` when the `heif` extra is installed alongside
|
||||
a pixel feature. The
|
||||
default metadata path scans these containers without that plugin. A corrupt or
|
||||
truncated file may still fail to decode.
|
||||
truncated file may still fail to decode. A camera-style AVIF's EXIF
|
||||
orientation is exposed by this plugin neither through `getexif()` nor
|
||||
`info["original_orientation"]`, so the display-tag carry cannot re-declare it
|
||||
(HEIC can; see the display-tags contract in
|
||||
[`module-internals.md`](module-internals.md)).
|
||||
|
||||
### Some metadata removal requires ffmpeg
|
||||
|
||||
@@ -512,11 +516,9 @@ remain identical because isolated decoder hits can otherwise be content noise.
|
||||
It accepts Variant P schemas 0-2. Variant Q requires a different model, and
|
||||
schema 3 is rejected at the measured precision threshold. Its NumPy 1.x runtime
|
||||
limits the extra to Python 3.11-3.12; the rest of the package remains supported
|
||||
through Python 3.14. The current TrustMark dependency line also resolves lightning 2.6.5
|
||||
(PYSEC-2026-3624, no fixed release yet); the vulnerable `load_from_checkpoint`
|
||||
path is unreachable here because TrustMark loads its checksummed checkpoints
|
||||
with plain `torch.load`. Bump lightning and cut a patch release when a fix
|
||||
ships. The calibration history is in
|
||||
through Python 3.14. The current TrustMark dependency line resolves
|
||||
`lightning` and `pytorch-lightning` 2.6.6, which contain the published fixes
|
||||
for their checkpoint-loading advisories. The calibration history is in
|
||||
[module internals](module-internals.md#metadata-and-provenance).
|
||||
|
||||
A generic metadata-free AI-generated-image classifier is not shipped. It is a
|
||||
|
||||
+102
-28
@@ -1084,13 +1084,53 @@ gate without downloads and pin that `identify` does not import this
|
||||
module.
|
||||
|
||||
Weights stay out of git. The Hub snapshot is `wiltodelta/raiw-photo-classify`.
|
||||
`RAIW_CLASSIFY_WEIGHTS` overrides it. The extra is `classify`. The runtime
|
||||
resolves the snapshot through `WEIGHTS_ALLOW_PATTERNS`, exported so an
|
||||
offline deploy's pre-cache installs the exact runtime request as one name
|
||||
(`tests/test_classify.py` pins the seam). User guide:
|
||||
`RAIW_CLASSIFY_WEIGHTS` overrides it. The extra is `classify`. The optional
|
||||
CPU ONNX vision runtime is the `classify-onnx` extra. The runtime
|
||||
narrows its snapshot download to the selected backend's model and shared heads.
|
||||
`WEIGHTS_ALLOW_PATTERNS` exports the union so an offline deploy can pre-cache
|
||||
both backends explicitly (`tests/test_classify.py` pins the seam). User guide:
|
||||
[photo-classify.md](photo-classify.md). Hub card:
|
||||
[photo-classify-hf/README.md](photo-classify-hf/README.md).
|
||||
|
||||
The CLIP loader suppresses discarded random parameter initialization with
|
||||
Transformers' `no_init_weights` context before applying the complete frozen
|
||||
state dict. Normal `CLIPModel(config)` construction initialized 427 million
|
||||
parameters that the checkpoint immediately replaced. On 2026-09-10, three
|
||||
fresh local CPU processes over the same synthetic 512x384 PNG fell from
|
||||
9.024/7.558/7.783 seconds to 2.124/2.059/2.065 seconds. This is a process-cold,
|
||||
OS-cache-warm comparison, not a hosted-container startup estimate. All 42
|
||||
tracked image fixtures kept byte-identical classification records before and
|
||||
after. `tests/test_clip_l_ft.py` rejects a loader that invokes random Linear
|
||||
initialization and verifies that checkpoint parameters and constructor-created
|
||||
buffers are fully materialized.
|
||||
|
||||
The optional CPU-only `backend="onnx"` path reads a static batch-one,
|
||||
vision-only FP32 graph produced by `scripts/export_photo_classify_onnx.py`.
|
||||
The graph is published in the pinned Hub snapshot; PyTorch remains the default.
|
||||
It is 1,216,102,506 bytes versus the 1,710,697,163-byte full
|
||||
PyTorch checkpoint; the removed text tower alone is 494,601,216 bytes. A
|
||||
2026-09-10 stress comparison covered 500 locally available catalog photographs
|
||||
closest to either Model 1 threshold plus 500 deterministic family-stratified
|
||||
photographs. The minimum embedding cosine was 0.999999979, mean absolute delta
|
||||
was 1.54e-7, and no detector or receipt-gate decision changed. All 42 tracked
|
||||
fixtures also kept byte-identical public records.
|
||||
|
||||
Three alternating fresh-process CPU pairs on the same synthetic 512x384 PNG
|
||||
gave PyTorch 7.037/5.235/5.003 seconds and ONNX 6.058/4.956/4.528 seconds under
|
||||
the then-current machine load: ONNX won all three pairs, but its 4.956-second
|
||||
median was only 5% below PyTorch's 5.235 seconds. Median peak RSS fell from
|
||||
3.78 GB to 1.92 GB. These are process-cold, OS-cache-warm local results, not a
|
||||
hosted-container estimate; the memory reduction is the stronger result. ONNX
|
||||
Runtime graph optimization is disabled because rebuilding the already
|
||||
constant-folded 1.2 GB graph increased session startup. CoreML is not selected:
|
||||
its static session took 18.8 seconds to construct and its dynamic-batch probe
|
||||
failed. Dynamic INT8 reduced the graph to 305,462,280 bytes but changed 3 of the
|
||||
42 fixture verdicts in both directions across the public decision boundary, so
|
||||
it remains rejected without retraining and a new operating point. Splitting
|
||||
Model 2 does not address cold start: its detector and provider files total about
|
||||
2 MB and loaded in roughly 3 ms in the same stage profile; CLIP-L Model 1
|
||||
dominates.
|
||||
|
||||
### Pixel forensics
|
||||
|
||||
[`pixel_evidence.py`](../src/remove_ai_watermarks/pixel_evidence.py) measures six
|
||||
@@ -1253,29 +1293,31 @@ The Jimeng pill has an additional decision gate because its visual detector is
|
||||
weaker than the other registered marks. Keep that policy in the registry, not
|
||||
inside unrelated detector engines.
|
||||
|
||||
Everything about a mark is one registry row: its product family, its label regime,
|
||||
the platform sentence `identify` reports for it, and the metadata signals that
|
||||
confirm its vendor. `identify._VISIBLE_MARK_PLATFORM` and the signal mapping in
|
||||
Everything about a mark is one registry row: its product family, manufacturer,
|
||||
label regime, the platform sentence `identify` reports for it, and the metadata
|
||||
signals that confirm its vendor. The manufacturer is distinct from the label
|
||||
regime: Alibaba, ByteDance, Kuaishou, Tencent, and other companies all use TC260,
|
||||
but their marks do not belong to one manufacturer family.
|
||||
`identify._VISIBLE_MARK_PLATFORM` and the signal mapping in
|
||||
`api.visible_provenance` are derived from those rows rather than hand-maintained
|
||||
beside them, so registering a mark is one edit. Two marks carry no platform of
|
||||
their own: the Gemini sparkle has its own higher-confidence path, and the pill
|
||||
alone is too weak to attribute.
|
||||
|
||||
The set of marks that veto the pill is DERIVED from the registry rows: every mark
|
||||
under the same label regime (`tc260`) belonging to a different product. It used to
|
||||
be a hand-written list of keys, and that list drifted -- LiblibAI was registered
|
||||
alongside RunningHub and Baidu, both of which were added to it, and LiblibAI was
|
||||
not, so a confident LiblibAI detection did not suppress the pill the way its two
|
||||
siblings did. Marks outside the TC260 regime (Gemini, Samsung) are deliberately
|
||||
not vetoers: neither can put `jimeng` into `provenance`, so neither can enable the
|
||||
arm it would be vetoing.
|
||||
The set of marks that veto the Jimeng pill is derived from the registry rows:
|
||||
every other product from the same manufacturer. Doubao therefore vetoes the pill
|
||||
because both products are ByteDance, while Qwen, Kling, Yuanbao, RunningHub,
|
||||
Baidu, and LiblibAI do not become siblings merely because they use the same TC260
|
||||
standard. Marks from other manufacturers cannot enable or veto the
|
||||
ByteDance-specific arm.
|
||||
|
||||
A TC260 label relaxes the vendor its `ContentProducer` names, resolved through
|
||||
`KnownMark.tc260_producer_codes`. The label itself is vendor-agnostic, so this used to
|
||||
relax ByteDance's two products on every China-AIGC image -- which both risked a
|
||||
false fill on an image carrying some other vendor's mark and denied that vendor's
|
||||
own mark the relaxed gate its `provenance_ncc_factor` was calibrated for. An
|
||||
absent or unmapped producer still falls back to the ByteDance pair.
|
||||
absent or unmapped producer confirms no particular product and leaves every detector
|
||||
strict.
|
||||
|
||||
`remove_auto_marks` removes every selected mark, not only the strongest one.
|
||||
This matters for images that carry marks in more than one corner.
|
||||
@@ -1344,6 +1386,25 @@ be represented by the shared base:
|
||||
- [`liblib_engine.py`](../src/remove_ai_watermarks/liblib_engine.py)
|
||||
- [`microsoft_engine.py`](../src/remove_ai_watermarks/microsoft_engine.py)
|
||||
|
||||
Qwen is the one image registry row that currently covers two layout families.
|
||||
The shared text engine handles `千问AI生成`; `QwenEngine` also scans a square
|
||||
bottom-right region for Qwen Create's three-lobe symbol. Its 0.65 NCC gate was
|
||||
measured against the cleared provider fixture (0.848) and 132 decoded tracked
|
||||
controls other than that positive (maximum 0.516 on 2026-09-10, zero fires).
|
||||
NCC is not portable across OpenCV builds: issue 108 reported 0.81 on
|
||||
`data/fixtures/provenance/chatgpt-1.png` (sand and denim, no glyph) while this
|
||||
tree's OpenCV 4.11 scores 0.437 on the same file. A fire also requires
|
||||
binarized IoU of the bright residual with the three-lobe silhouette at or
|
||||
above 0.70; the true fixture stays at or above 0.75 down to 512 px, and
|
||||
chatgpt-1 sits at 0.55. `provenance_ncc_factor` remains 1.0, so OpenAI C2PA
|
||||
cannot relax this mark.
|
||||
The removal mask reuses the aligned three-lobe alpha and expands it by 6% of
|
||||
the detected side; a solid square was rejected because classical inpainting
|
||||
left a visible rectangular blur.
|
||||
`scripts/visible_alpha_solve.py qwen_symbol` rebuilds the template from the
|
||||
cleared fixture with a cubic local-background fit and retains only the three
|
||||
interior lobes.
|
||||
|
||||
LiblibAI has two registry entries under one product. `liblib` retains the
|
||||
historical bottom-center wordmark detector. `liblib_pill` covers the compact
|
||||
top-left `AI生成` pill and never fires at strict trust: the shape is generic, so
|
||||
@@ -1988,10 +2049,9 @@ so it is left alone.
|
||||
Measured 2026-08-26 against the anonymous oracle `meta.ai/identification`
|
||||
(verdicts read from the settled page text; corpus and full row-level provenance in
|
||||
`data/contentseal/manifest.csv`). The original measurement used the
|
||||
default resolution-adaptive curve. Current routing can infer the Meta cohort
|
||||
from a retained standalone AI IPTC tag, after checking C2PA vendor evidence
|
||||
first. Stripped files need explicit `--vendor meta` to select the measured
|
||||
Meta floor; absence of C2PA alone does not prevent automatic attribution.
|
||||
default resolution-adaptive curve. The retained standalone AI IPTC tag is a
|
||||
shared standard and no longer infers Meta. Files need explicit `--vendor meta`
|
||||
to select the measured Meta floor from independently known provenance.
|
||||
|
||||
| source (2.56 MP generation) | detected at | clean from |
|
||||
|---|---|---|
|
||||
@@ -2005,13 +2065,9 @@ Full spread: worst first-clean boundary (0.0525, 0.06] on lighthouse, easiest
|
||||
source already clean at 0.015. Following the same derivation as the OpenAI and
|
||||
Microsoft floors (worst clean boundary plus one full observed cross-source
|
||||
spread): 0.06 + (0.0525 - 0.015) = 0.0975, rounded up to **0.1**. Shipped as
|
||||
`QWEN_ZIMAGE_META_STRENGTH` with two routing paths: auto mode routes a file
|
||||
whose only provenance is the standalone AI IPTC tag onto the cohort
|
||||
(`vendor_for_strength` checks C2PA issuers first, so Google/OpenAI/Microsoft
|
||||
evidence always wins, and the tag's other users ship no invisible watermark
|
||||
this profile targets), and `--vendor meta` / `InvisibleOptions.vendor` names the
|
||||
cohort explicitly on stripped files, implying the scrub runs (naming the cohort
|
||||
asserts the watermark is present). sdxl-zimage has no measured Meta rung and an
|
||||
`QWEN_ZIMAGE_META_STRENGTH`; `--vendor meta` / `InvisibleOptions.vendor` names
|
||||
the cohort explicitly, implying the scrub runs (naming the cohort asserts the
|
||||
watermark is present). sdxl-zimage has no measured Meta rung and an
|
||||
explicit meta vendor there falls to the conservative unknown 0.25. The default
|
||||
resolution-adaptive curve (~0.1305 at 2.56 MP) also clears every measured
|
||||
source, so default behavior needed no change. Oracle verdicts carry a
|
||||
@@ -2148,7 +2204,25 @@ Contracts:
|
||||
pixels on save. Callers that know the decode state pass
|
||||
`orientation_applied=False` for raw pixels and `True` for a transposed
|
||||
raster. Dimensions alone cannot distinguish mirrors or 180-degree rotation.
|
||||
The omitted value retains the compatibility shape heuristic. Display tags
|
||||
TIFF and HEIF are the container exceptions on the explicit claim. Every
|
||||
reader in the stack (cv2's libtiff path under `IMREAD_UNCHANGED` too, and
|
||||
Pillow's TIFF plugin, identically) applies the TIFF orientation tag on
|
||||
decode, so `_read_display_tags` ignores the claim for TIFF and decides on
|
||||
the IFD's stored `ImageWidth`/`ImageLength` against the raster, via
|
||||
`_stored_size` -- `Image.size` already reports the upright geometry there,
|
||||
which is what let a turned raster be tagged into a second rotation
|
||||
(issue #106). HEIF blanks the tag instead: pillow-heif's opener rewrites
|
||||
the EXIF orientation to 1 (the real value waits in
|
||||
`info["original_orientation"]`), and libheif turns the raster exactly
|
||||
when the container declares an effective `irot`/`imir` transform
|
||||
(`isobmff.heif_transform_applied` scans the `meta`/`iprp`/`ipco`
|
||||
property store), so `Image.size` matches the raster in both cases and the
|
||||
shape heuristic has nothing to compare. A camera HEIC (stored raster,
|
||||
EXIF-only rotation, issue #105) re-declares its turn on the output; a
|
||||
re-encoded one whose writer baked `irot` decodes upright and is not
|
||||
tagged again. AVIF keeps the generic path: this plugin build neither
|
||||
blanks its tag nor applies the transform there. The omitted value retains
|
||||
the compatibility shape heuristic. Display tags
|
||||
are captured before encoding, including when source and destination match.
|
||||
|
||||
The metadata strip is the other half of that contract: Pillow cannot hold 16-bit
|
||||
|
||||
@@ -258,6 +258,7 @@ silently change the heads.
|
||||
| File | Role |
|
||||
| --- | --- |
|
||||
| `clip-l-ft.pt` | HeadedCLIP state dict (CLIP-L plus unused linear head) |
|
||||
| `clip-l-ft-vision-fp32.onnx` | Static batch-one vision tower and projection for the optional CPU backend |
|
||||
| `probe-weights-clip-l-ft.npz` | Ridge mean, scale, weights, `thr_oi_1pct` |
|
||||
| `detector.pt` | Freeze MLP |
|
||||
| `provider.pt` | Focal heads keyed `openai`, `google`, `bytedance`, `tc260`, `meta_muse_image`, `no_ai` |
|
||||
|
||||
@@ -52,6 +52,7 @@
|
||||
},
|
||||
"files": [
|
||||
"clip-l-ft.pt",
|
||||
"clip-l-ft-vision-fp32.onnx",
|
||||
"probe-weights-clip-l-ft.npz",
|
||||
"detector.pt",
|
||||
"provider.pt"
|
||||
|
||||
+17
-4
@@ -38,7 +38,11 @@ print(result.label, result.detector, result.provider)
|
||||
|
||||
Missing extra raises `RuntimeError` with
|
||||
`pip install 'remove-ai-watermarks[classify]'`. `device` is a library
|
||||
parameter (`None` / `"auto"` / `"cpu"` / `"cuda"`), not a CLI option.
|
||||
parameter (`None` / `"auto"` / `"cpu"` / `"cuda"`), not a CLI option. The
|
||||
explicit `backend="onnx"` path is CPU-only and requires
|
||||
`clip-l-ft-vision-fp32.onnx` in `RAIW_CLASSIFY_WEIGHTS` or the pinned Hub
|
||||
snapshot. PyTorch remains the default. Install the optional runtime with
|
||||
`pip install 'remove-ai-watermarks[classify-onnx]'`.
|
||||
|
||||
Weights are not in git. First call downloads
|
||||
[`wiltodelta/raiw-photo-classify`](https://huggingface.co/wiltodelta/raiw-photo-classify)
|
||||
@@ -47,9 +51,18 @@ at the freeze revision pinned in `classify.py`, or reads
|
||||
`probe-weights-clip-l-ft.npz`, `detector.pt`, and `provider.pt`. The training
|
||||
catalog is not published with the package. Deployments that pre-cache the
|
||||
snapshot for offline resolution must request exactly
|
||||
`classify.WEIGHTS_ALLOW_PATTERNS` (the runtime resolves that same list under
|
||||
`HF_HUB_OFFLINE=1`; a partial pre-cache fails the whole resolution, even for
|
||||
files the runtime would fall back on).
|
||||
`classify.WEIGHTS_ALLOW_PATTERNS`. That exported list covers both backends;
|
||||
the online runtime narrows its download to the selected model plus the shared
|
||||
heads. A partial offline pre-cache fails resolution, even for files the runtime
|
||||
would otherwise fall back on.
|
||||
|
||||
Generate the optional vision-only graph from the exact frozen checkpoint:
|
||||
|
||||
```bash
|
||||
uv run --with onnx python scripts/export_photo_classify_onnx.py \
|
||||
--checkpoint /path/to/clip-l-ft.pt \
|
||||
--out /path/to/clip-l-ft-vision-fp32.onnx
|
||||
```
|
||||
|
||||
## What one call returns
|
||||
|
||||
|
||||
+7
-1
@@ -226,7 +226,9 @@ head abstains to `None`. Otherwise it is `None`, including
|
||||
when 124-d extraction refuses the file.
|
||||
|
||||
`device` is a library parameter: `None` / `"auto"` detect, `"cpu"` or `"cuda"`
|
||||
pin. It is not a CLI option.
|
||||
pin. It is not a CLI option. `backend="onnx"` selects the optional CPU-only
|
||||
FP32 vision runtime and requires `remove-ai-watermarks[classify-onnx]`;
|
||||
`backend="torch"` remains the default.
|
||||
|
||||
Missing extra raises `RuntimeError` with the quoted install command
|
||||
`'remove-ai-watermarks[classify]'`. Guide:
|
||||
@@ -245,6 +247,10 @@ import remove_ai_watermarks as raiw
|
||||
vendors = raiw.visible_provenance("input.png")
|
||||
```
|
||||
|
||||
A known TC260 `ContentProducer` maps to one registered product. A missing or
|
||||
unmapped producer, and the shared standalone IPTC AI tag, return no vendor key;
|
||||
neither standard identifies a manufacturer by itself.
|
||||
|
||||
Get the full provenance report:
|
||||
|
||||
```python
|
||||
|
||||
@@ -11,7 +11,7 @@ A release is complete only after all four published surfaces are verified:
|
||||
|---|---|---|
|
||||
| PyPI | The `remove-ai-watermarks` wheel and source distribution | `publish.yml` |
|
||||
| Homebrew | The `remove-ai-watermarks` formula in `wiltodelta/homebrew-tap` | `distribute.yml` |
|
||||
| Hugging Face Space | The Space demo deployed on the new release through a requirements-pin bump in `wiltodelta/raiw-hf-space` | `raiw-hf-space` sync workflow; the pin bump is manual, `distribute.yml` only re-installs the pinned version |
|
||||
| Hugging Face Space | The Space demo deployed on the new release through a requirements-pin bump in `wiltodelta/raiw-hf-space` | `distribute.yml` dispatches `pin-library.yml` in that repo, waits for the Hub `requirements.txt` pin, then factory-rebuilds; `sync-to-hf.yml` mirrors the pin onto the Space |
|
||||
| ComfyUI Registry | A compatible release of `wiltodelta/ComfyUI-remove-ai-watermarks` with its own node version | `distribute.yml` and the node repository's workflows |
|
||||
|
||||
The GitHub Release is the trigger and release record for this flow. Conda is
|
||||
@@ -62,16 +62,20 @@ PyPI API token from the repository.
|
||||
waits for the matching source distribution to appear on PyPI, then:
|
||||
|
||||
- updates the Homebrew tap formula URL and SHA-256;
|
||||
- triggers a factory rebuild of the Hugging Face Space: this re-installs the
|
||||
version pinned in the Space repo, it does NOT upgrade the demo;
|
||||
- bumps the Hugging Face Space pin and factory-rebuilds the demo on that pin;
|
||||
- synchronizes, tests, versions, and publishes the ComfyUI nodes.
|
||||
|
||||
Upgrading the Hugging Face Space is a separate, manual step: bump
|
||||
`remove-ai-watermarks[visible,heif]` in `wiltodelta/raiw-hf-space` (`pyproject.toml`,
|
||||
`uv lock`, and the re-exported `requirements.txt`), then push. That repository's
|
||||
`sync-to-hf.yml` mirrors the files onto the Space, which rebuilds on the new
|
||||
pin. Its callback smoke tests run in that repository's CI on every push; they
|
||||
are the scenario check for the demo surface.
|
||||
Upgrading the Hugging Face Space is dispatched from `distribute.yml`: it runs
|
||||
`pin-library.yml` in `wiltodelta/raiw-hf-space`, which bumps
|
||||
`remove-ai-watermarks[visible,heif]` in `pyproject.toml`, refreshes `uv.lock`,
|
||||
and pushes. That repository's `sync-to-hf.yml` then mirrors the files onto the
|
||||
Space. `distribute.yml` waits for the Hub `requirements.txt` pin to match,
|
||||
then factory-rebuilds. The `RAIW_HF_SPACE_TOKEN` repository secret is a
|
||||
fine-grained token limited to the Space repository, with Contents write and
|
||||
Actions write. Without the secret the pin bump is skipped and only the
|
||||
factory rebuild runs, which reinstalls whatever version is already pinned.
|
||||
The Space repo's callback smoke tests still run on every push; they are the
|
||||
scenario check for the demo surface.
|
||||
|
||||
The Hugging Face **model** `wiltodelta/raiw-photo-classify` is not part of that
|
||||
release fan-out. It holds the photo-classify freeze weights. The library extra
|
||||
@@ -79,8 +83,9 @@ release fan-out. It holds the photo-classify freeze weights. The library extra
|
||||
`RAIW_CLASSIFY_WEIGHTS`. Update the Hub repo only with
|
||||
`.github/workflows/publish-photo-classify-hf.yml` (`workflow_dispatch`,
|
||||
type `publish`). Full mode downloads the four weight files from the GitHub
|
||||
Release tag `photo-classify-freeze-2026-08-31` (or the tag you pass) and
|
||||
uploads them with the card in `docs/photo-classify-hf/`. Card mode updates
|
||||
Release tag `photo-classify-freeze-2026-08-31` (or the tag you pass), exports
|
||||
the static FP32 vision-only ONNX graph from `clip-l-ft.pt`, and uploads all five
|
||||
files with the card in `docs/photo-classify-hf/`. Card mode updates
|
||||
the README, `operating-point.json`, and the receipt-gate head staged from
|
||||
the package assets; freeze weights are untouched. A library version bump
|
||||
alone does not upload new heads -- bump the pinned Hub revision when a new
|
||||
@@ -160,6 +165,36 @@ surface complete. The Claude community marketplace remains a manual,
|
||||
authenticated publish. Do not treat any listing as complete until the live
|
||||
catalog shows the skill.
|
||||
|
||||
When `metadata.version` in `SKILL.md` changes, finish the catalog follow-up as
|
||||
part of the release that carries that version:
|
||||
|
||||
1. Verify the automatic ClawHub submission and then the live version. Do not
|
||||
publish it a second time by hand.
|
||||
2. If the Claude community directory listing is live, publish the updated
|
||||
plugin through its authenticated submission surface and verify the listing.
|
||||
If the original submission is still under review, leave it pending rather
|
||||
than opening a duplicate.
|
||||
3. If the AI Skills Catalog listing is live, send the new skill version through
|
||||
the update route supplied with its acceptance. The catalog stores compatible
|
||||
packages as well as source metadata, so a GitHub push alone is not a verified
|
||||
update. If no update control was supplied, contact the catalog or resubmit
|
||||
and verify the resulting listing. Do not duplicate a pending initial review.
|
||||
4. If the OpenAI plugin directory listing has been approved, upload the new
|
||||
plugin archive through the Platform portal and verify the published version.
|
||||
Until the initial product-specific review is approved, there is no release
|
||||
action beyond tracking that review.
|
||||
5. SkillsMP, skills.sh, AgenticSkills, and the awesome lists point at GitHub or
|
||||
refresh from it. Do not resubmit a routine version bump. Contact them or open
|
||||
a new PR only when their public description, category, tags, or repository
|
||||
URL must change.
|
||||
|
||||
The automated release workflow cannot perform the authenticated directory
|
||||
steps. A release with a changed skill version is not fully distributed until
|
||||
each applicable manual step is verified or explicitly marked pending because
|
||||
the catalog has not accepted the initial submission. The catalog-specific
|
||||
submission state and links live in
|
||||
[`docs/agent-skill.md`](agent-skill.md#where-the-skill-can-actually-be-listed).
|
||||
|
||||
## Release verification
|
||||
|
||||
Forensic transports are versioned independently from the package. Before publishing
|
||||
@@ -210,6 +245,9 @@ version, or any other status, fails. The full checklist for completeness:
|
||||
rebuild, and one live `identify` and one live `visible` API call succeed
|
||||
against the running Space;
|
||||
- a clean install can run `remove-ai-watermarks --version`.
|
||||
- when the Agent Skill version changed, every applicable manual catalog update
|
||||
above is verified, or its initial submission remains pending without a
|
||||
duplicate submission.
|
||||
|
||||
A clean-install check run immediately after publication can fail with "no
|
||||
version of remove-ai-watermarks==X.Y.Z" even though the simple index already
|
||||
@@ -219,9 +257,18 @@ failure.
|
||||
|
||||
The ComfyUI sync run can fail the same way on its own: it resolves the version
|
||||
from the PyPI JSON API (which updates first) but installs the test dependency
|
||||
with pip against the simple index, whose CDN edges lag by minutes. Rerun the
|
||||
failed `distribute.yml` comfyui job once the simple index lists the release;
|
||||
nothing about the release itself is wrong.
|
||||
with pip against the simple index, whose CDN edges lag by minutes. The
|
||||
`distribute.yml` waiter also has a clock: 0.40.0's node suite finished after
|
||||
the 15-minute poll, so the job now waits 30 minutes and then accepts a
|
||||
registry row that already requires this library version. Rerun the failed
|
||||
`comfyui` job only when the registry still has no such row.
|
||||
|
||||
The automatic `verify-release.yml` run after `distribute.yml` checks the
|
||||
Hugging Face Space pin. When `RAIW_HF_SPACE_TOKEN` is set, that pin lands
|
||||
during distribution. Without the secret, bump the pin by dispatching
|
||||
`pin-library.yml` in `wiltodelta/raiw-hf-space` and then dispatch
|
||||
`verify-release.yml` with the release version. Do not read a missing Space
|
||||
pin as a broken PyPI or Homebrew publish.
|
||||
|
||||
If the release changed CLI routing, extras, exit codes, mark keys, or the
|
||||
intended-use boundary, confirm `skills/remove-ai-watermarks/` still matches.
|
||||
@@ -13,7 +13,7 @@ The `visible` command registers these mark keys:
|
||||
| `gemini` | Google Gemini visible watermark (sparkle) | Usually bottom right | Detection includes a false positive gate. |
|
||||
| `doubao` | `豆包AI生成` | Bottom right | Vendor specific text detector. |
|
||||
| `jimeng` | `★ 即梦AI` | Bottom right | Vendor specific text detector. |
|
||||
| `qwen` | `千问AI生成` | Bottom right | Strict visual gate. |
|
||||
| `qwen` | `千问AI生成` or the three-lobe Qwen symbol | Bottom right | Separate strict templates cover the text and Qwen Create symbol variants. |
|
||||
| `kling` | `可灵AI 3.0` or `KlingAI 3.0` | Bottom right | Separate calibrated silhouettes cover the older CJK and current IMAGE 3.0 Latin variants. |
|
||||
| `yuanbao` | `元宝` over `AI生成` | Bottom right | Standard two-line variant only. |
|
||||
| `samsung` | `✦ Contenuti generati dall'AI` | Bottom left | Calibrated for the Italian text variant. |
|
||||
@@ -197,12 +197,7 @@ This project has no local Content Seal decoder. Meta Model API outputs and
|
||||
Meta CDN copies carry an XMP `iptcExt:DigitalSourceType =
|
||||
trainedAlgorithmicMedia` companion tag, which `identify` reports through the
|
||||
existing Made-with-AI path; that IPTC code is a standard, not a Meta-exclusive
|
||||
signal, so it cannot key a strength cohort the way the C2PA issuer does. Since
|
||||
0.33.0 a standalone AI digital-source tag (no C2PA manifest) additionally emits
|
||||
the additive `content_seal` signal - the strength router's Meta bet as evidence,
|
||||
medium confidence, with the same caveat - so clients select pixel removal from
|
||||
the signal list exactly the way InvisMark is additive over `soft_binding`. It
|
||||
is an attribution, never a decode. The
|
||||
signal, so it neither asserts Content Seal nor selects a strength cohort. The
|
||||
external oracle is `https://meta.ai/identification`: anonymous, no login,
|
||||
accepts image, video, and audio, enforces an unspecified daily identification
|
||||
limit, and answers with model attribution (`Muse Image 1 - Meta`) plus a
|
||||
@@ -215,12 +210,9 @@ resolution-adaptive strength (oracle-verified on 2.56 MP generations); measured
|
||||
strength boundaries are recorded in `data/contentseal/manifest.csv` and
|
||||
[module internals](module-internals.md#meta-content-seal-boundaries-for-qwen-zimage).
|
||||
The derived Meta floor (0.1 by the standard spread method) ships as a measured
|
||||
cohort: auto mode routes a file whose only provenance is the standalone AI IPTC
|
||||
tag to it (the tag is not Meta-exclusive; other tag users ship no invisible
|
||||
watermark this profile targets, and Google/OpenAI/Microsoft C2PA evidence
|
||||
always wins first), and `invisible --vendor meta` (also `all` and `batch`, and
|
||||
`InvisibleOptions.vendor` in the API) names the cohort explicitly on stripped
|
||||
files. An explicit vendor implies the scrub runs: naming the cohort asserts the
|
||||
cohort. `invisible --vendor meta` (also `all` and `batch`, and
|
||||
`InvisibleOptions.vendor` in the API) selects it when the source is independently
|
||||
known. An explicit vendor implies the scrub runs: naming the cohort asserts the
|
||||
pixel watermark is present.
|
||||
The seal survives resizing, JPEG recompression, and metadata stripping; it dies
|
||||
to center crops of a third to a half, matching the Reuters 2026-07-11 finding
|
||||
|
||||
+2
-1
@@ -779,7 +779,8 @@ old OpenAI C2PA remains an origin signal without becoming a SynthID claim.
|
||||
|
||||
- **Kakao** (South Korea): SynthID adopter as of May 2026 (Google announcement)
|
||||
- **NVIDIA Cosmos**: SynthID for video (not still images; different pipeline)
|
||||
- **Meta AI**: does NOT use SynthID; uses IPTC `digitalSourceType` marker instead
|
||||
- **Meta AI**: does NOT use SynthID; Muse uses Content Seal and may retain a shared
|
||||
IPTC `digitalSourceType` companion tag that does not identify Meta by itself
|
||||
|
||||
### 4.4 Version evolution (v1 vs v2 hardening)
|
||||
|
||||
|
||||
@@ -490,6 +490,32 @@ bridge or reset its start PTS must fail this gate.
|
||||
A separate constant-rate clip with the same non-zero start guards the
|
||||
start-offset routing without relying on the VFR branch.
|
||||
|
||||
## Signal discovery cadence
|
||||
|
||||
The project skill [signal-discovery](../.claude/skills/signal-discovery/SKILL.md)
|
||||
orchestrates the corpus audits in this plan. `scripts/corpus_gap_scan.py` is only
|
||||
the metadata-serialization probe inside that workflow. It now records raw
|
||||
markers, input shapes, identify errors, integrity clashes, unattributed results,
|
||||
and blind-marker candidates, but it does not cover visible marks, proprietary
|
||||
invisible marks, detection/removal parity, or historical crossfire.
|
||||
|
||||
Run a fresh-delta discovery review weekly after the retained corpus is
|
||||
refreshed, using `corpus_gap_scan.py --since` with an overlap from the preceding
|
||||
run. Replay the complete historical corpus before releases that change a
|
||||
detection, metadata, container, registry, invisible-routing, or removal seam,
|
||||
after every detector registration or recalibration, and monthly as a backstop.
|
||||
The historical pass is required because newer code can expose an old
|
||||
serialization and a new detector can crossfire on an old neighbouring cohort.
|
||||
A stratified sidecar sample is a useful preflight but does not replace a
|
||||
triggered full replay.
|
||||
|
||||
Removal-parity sweeps are change-triggered rather than weekly: run the matching
|
||||
metadata, visible, video, or invisible audit whenever either its detector or
|
||||
cleaner moves. External provider oracles are claim-triggered, never
|
||||
calendar-triggered. Keep private inputs and reports untracked, and implement an
|
||||
actionable detector or cleaner gap in this library rather than in a downstream
|
||||
consumer.
|
||||
|
||||
## Standing gap
|
||||
|
||||
None of this is in `maintain.sh`, and it should not all be -- the sweeps take hours. But
|
||||
|
||||
@@ -51,7 +51,7 @@ AI generator values while retaining unrelated camera and editor EXIF. The
|
||||
shared `xai_signature_pair` helper is the single source of truth for the
|
||||
pair. On the ISOBMFF path, `blank_ai_exif_tokens` provides the corresponding
|
||||
in-place scrub for supported EXIF values, TC260 AIGC blocks, and the xAI pair.
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). The label says only "this is AI", but its `ContentProducer` names the signing entity — `001` + `1` + an 18-char Unified Social Credit Code + a 5-digit product suffix, normalized by `metadata.uscc_of`, or for a few generators a bare product name. `KnownMark.tc260_producer_codes` maps the codes settled per vendor by `scripts/vendor_cohort_harvest.py` to registry mark keys, so an AIGC image relaxes the detector of the vendor it actually carries rather than a guessed pair; an unmapped or absent producer falls back to ByteDance's two products. A code identifies a legal entity, not necessarily one brand, so a hosting or aggregating platform that signs for several apps is a recall bet rather than a proof.
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). The label says only "this is AI", but its `ContentProducer` names the signing entity — `001` + `1` + an 18-char Unified Social Credit Code + a 5-digit product suffix, normalized by `metadata.uscc_of`, or for a few generators a bare product name. `KnownMark.tc260_producer_codes` maps the codes settled per vendor by `scripts/vendor_cohort_harvest.py` to registry mark keys, so an AIGC image relaxes only the detector of the product it actually carries; an unmapped or absent producer leaves every product strict. The same mapping supplies the manufacturer for provenance-conflict checks. A code identifies a legal entity, not necessarily one brand, so a hosting or aggregating platform that signs for several apps is a recall bet rather than a proof.
|
||||
|
||||
**Doubao** (ByteDance) uses it (verified on a public issue sample; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **four image serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `<TC260:AIGC>{...}</TC260:AIGC>` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on compatible samples) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on compatible samples, `ContentProducer="doubao"`), a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on compatible samples, `ContentProducer="001191440300708461136T1308L"`), **and** a bare `AIGC{...}` blob (the label glued straight to its JSON, no `"AIGC":` key wrapper) embedded in a **JPEG APP segment near the JFIF header** — confirmed on compatible samples. The two raw-JSON forms are scanned in one loop (`'"AIGC"'` then `AIGC{`) that **falls through on a non-TC260 / undecodable hit instead of returning** — a quoted `"AIGC"` can appear later in an XMP packet while the real label is a bare `AIGC{...}` earlier in the file, so an unconditional early return on the quoted form would shadow the bare form (the exact bug behind the 06-10 misses). Native MP4/MOV is a fifth serialization: TC260-PG-20257A stores an `AIGC` key in `moov.udta.meta.keys` and the raw JSON in the matching `ilst` item; Doubao's iOS export additionally writes two QuickTime-form variants (a bare `meta` box as a direct `moov` child, and a keyless `hdlr=mdir` metadata list whose `ilst` data items carry the JSON with no key name) — both verified on retained carriers (2026-08-17). The seeking parser reaches a tail `moov` without reading `mdat`; removal replaces the key with `free` and blanks the validated value at the same length so every box size and stream offset stays fixed. All generic forms are gated on at least one TC260 field (`TC260_AIGC_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. `TC260_AIGC_FIELDS` covers **two schemas**: the producer-side one (`Label` / `ContentProducer` / `ProduceID` / `ContentPropagator` / `PropagateID`, Doubao and most China gens) and the **service-provider** one (`ServiceProvider` / `ServiceUser`, plus generic `Time` / `ContentId` which are NOT gated on) — **Tencent Cloud's** AIGC variant (`ServiceProvider` = `腾讯云`), embedded in **EXIF `ImageDescription`**, verified on compatible samples. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated).
|
||||
|
||||
@@ -213,7 +213,7 @@ not ground truth.
|
||||
|
||||
Two consequences we can exploit: (1) the 5% floor is a **scale prior** -- a compliant CN mark's glyphs are large (>= 51 px on a 1024² image), so a CN silhouette ladder can be anchored at ~5-10% of the short side instead of swept broadly, which should cut false fires; (2) every compliant string shares the tail `AI生成` / `AI合成`, so a shared suffix silhouette plus a per-vendor prefix may beat five independent templates. NOT established: whether 文字高度 means cap height, em box, or rendered bounding box (a ~1.3x spread in template scale). Sources: `https://www.tc260.org.cn/upload/2025-03-15/1742009439794081593.pdf`, parent CAC measure `https://www.cac.gov.cn/2025-03/14/c_1743654684782215.htm` (the CAC text itself specifies no size or corner).
|
||||
|
||||
**Alibaba Cloud Qwen -- two surfaces that differ (API tier VERIFIED (a)).** Model Studio docs state verbatim that the API adds a `Qwen-Image` watermark 在图像右下角 and 默认值为 false -- so API output is **unwatermarked by default**, and when enabled the mark is a LATIN wordmark, not CJK, and not GB-compliant in wording. The consumer app's `千问AI生成` (bottom-right) is (b) secondary only -- no Alibaba primary page states it. So Qwen needs TWO templates, and its absence is never evidence of a clean image. Source: `https://help.aliyun.com/zh/model-studio/qwen-image-api`.
|
||||
**Alibaba Cloud Qwen -- several surfaces differ (API tier VERIFIED (a)).** Model Studio docs state verbatim that the API adds a `Qwen-Image` watermark 在图像右下角 and 默认值为 false -- so API output is **unwatermarked by default**, and when enabled the documented mark is a LATIN wordmark, not CJK, and not GB-compliant in wording. The Chinese consumer app's `千问AI生成` (bottom-right) is (b) secondary only -- no Alibaba primary page states it. A cleared Qwen Create `Qwen-Image 2.0` output from 2026-09-03 instead carries the three-lobe Qwen symbol alone; that symbol and the CJK text are registered under one `qwen` key with separate templates. The documented API Latin wordmark remains unsupported, so absence of either registered shape is never evidence of a clean image. Source: `https://help.aliyun.com/zh/model-studio/qwen-image-api`.
|
||||
|
||||
**星绘 is ByteDance (VERIFIED (a): Baidu Baike + App Store listing, now branded 豆包旗下, team folded into Doubao April 2025).** So `星绘AI生成` is very likely the Doubao house style -- same typeface, same corner, possibly the same top-left `AI生成` pill. Starting from the Doubao `TextMarkConfig` and swapping the two lead glyphs is the cheap path. String/position themselves are (c) inferred.
|
||||
|
||||
@@ -221,7 +221,7 @@ Two consequences we can exploit: (1) the 5% floor is a **scale prior** -- a comp
|
||||
|
||||
**Tencent Yuanbao: RESOLVED 2026-07-25, registered (`yuanbao_engine.py`).** The standard mark is a compact two-line italic `元宝` over `AI生成` block at bottom-right. It switches between light and dark strokes with the scene, so detection uses polarity-independent local contrast rather than a white top-hat. The separate one-line overlay variant remains evidence-limited to one example.
|
||||
|
||||
**Meta Muse Image / Content Seal: current state VERIFIED 2026-08-26 against live artifacts.** Muse Image (launched 2026-07-07, first image model from Meta Superintelligence Labs) puts no visible mark on output. Every output carries Content Seal, a proprietary invisible pixel watermark; research lineage is open (`github.com/facebookresearch/content-seal`, Pixel Seal / VideoSeal / Watermark Anything), but the deployed implementation is proprietary and unpublished. API outputs and Meta CDN copies also carry XMP `iptcExt:DigitalSourceType = trainedAlgorithmicMedia` (a standard IPTC code, not Meta-exclusive). The only reader is the anonymous web oracle `meta.ai/identification`, which returns model attribution plus a per-generation ID and timestamp embedded in the payload. Verified robustness (our corpus, `data/contentseal/`): seal survives CDN WebP transcode, 512 px resize, JPEG q85, and full metadata stripping; it is lost to center crops of 33-50% linear size (Reuters measured 55% detector misses on cropped images, 2026-07-11). Removal is fully supported and oracle-verified on the default profile, with a measured Meta strength floor (0.1) shipped behind auto IPTC routing and an explicit `--vendor meta` override; see [supported signals](supported-signals.md) and [module internals](module-internals.md). Legacy paragraph retained below.
|
||||
**Meta Muse Image / Content Seal: current state VERIFIED 2026-08-26 against live artifacts.** Muse Image (launched 2026-07-07, first image model from Meta Superintelligence Labs) puts no visible mark on output. Every output carries Content Seal, a proprietary invisible pixel watermark; research lineage is open (`github.com/facebookresearch/content-seal`, Pixel Seal / VideoSeal / Watermark Anything), but the deployed implementation is proprietary and unpublished. API outputs and Meta CDN copies also carry XMP `iptcExt:DigitalSourceType = trainedAlgorithmicMedia` (a standard IPTC code, not Meta-exclusive). The only reader is the anonymous web oracle `meta.ai/identification`, which returns model attribution plus a per-generation ID and timestamp embedded in the payload. Verified robustness (our corpus, `data/contentseal/`): seal survives CDN WebP transcode, 512 px resize, JPEG q85, and full metadata stripping; it is lost to center crops of 33-50% linear size (Reuters measured 55% detector misses on cropped images, 2026-07-11). Removal is fully supported and oracle-verified on the default profile, with a measured Meta strength floor (0.1) selected through an explicit `--vendor meta` override; the shared IPTC tag alone neither identifies Meta nor asserts Content Seal. See [supported signals](supported-signals.md) and [module internals](module-internals.md). Legacy paragraph retained below.
|
||||
|
||||
**Meta `Imagined with AI` (string VERIFIED (a) from Meta's own newsroom; POSITION NOT VERIFIED).** Sources conflict on placement. Do not encode a corner without a verified sample. A dedicated 2026-08-27 sample hunt failed to obtain one: the Feb-2024 newsroom images are UI mockups whose photos carry no in-pixel mark; community posts (Threads `C8_rS_MuEId` titled "lower left corner", a Facebook share) corroborate bottom-LEFT verbally but their files defeat pixel verification; press screenshots predate the mark (Dec 2023); `imagine.meta.com` is dead (redirects to meta.ai) and its Wayback captures are broken SPA error pages. The mark's generator no longer exists (Muse output has no visible mark), so no fresh sample can be made. Removal stays on the generic `erase --region` path until a legacy capture surfaces. Source: `https://about.fb.com/news/2024/02/labeling-ai-generated-images-on-facebook-instagram-and-threads/`.
|
||||
|
||||
|
||||
+7
-1
@@ -4,7 +4,7 @@
|
||||
# on it, including the ComfyUI node package. The console script below carries
|
||||
# the same weight, since users have it on PATH.
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.39.0"
|
||||
version = "0.40.1"
|
||||
description = "AI watermark remover for visible, invisible, and provenance marks in images and video"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11,<3.15"
|
||||
@@ -154,6 +154,12 @@ classify = [
|
||||
"tokenizers>=0.22,<0.23",
|
||||
"huggingface-hub>=0.20.0",
|
||||
]
|
||||
# Optional vision-only CPU runtime for the photo classifier. Keep it separate
|
||||
# because PyTorch remains the default and ONNX Runtime is needed only on opt-in.
|
||||
classify-onnx = [
|
||||
"remove-ai-watermarks[classify]",
|
||||
"onnxruntime>=1.24.0",
|
||||
]
|
||||
# Universal region eraser backend -- big-LaMa via onnxruntime (Carve/LaMa-ONNX,
|
||||
# Apache-2.0). CPU, no torch. Model (~200 MB) is downloaded on first use and
|
||||
# cached by huggingface_hub; it is never bundled in this repo. The default cv2
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ explicitly names a tracked canonical result.
|
||||
|
||||
| Script | Purpose |
|
||||
| --- | --- |
|
||||
| `corpus_gap_scan.py` | Compare a local image corpus with the library's `identify` results. |
|
||||
| `corpus_gap_scan.py` | Inventory `identify`, metadata markers, unattributed outcomes, errors, and gap candidates over a full or date-bounded corpus. |
|
||||
| `detection_timing.py` | Record per-method metadata and verdict timings. |
|
||||
| `detection_timing_report.py` | Aggregate timing records by method and segment. |
|
||||
| `fidelity_metrics.py` | Compute objective image-fidelity metrics for paired outputs. |
|
||||
@@ -44,7 +44,7 @@ explicitly names a tracked canonical result.
|
||||
| `synthid_corpus.py` | Ingest and inspect the local SynthID reference corpus. |
|
||||
| `vendor_cohort_harvest.py` | Partition TC260 carriers by producer code. |
|
||||
| `vendor_mark_calibrate.py` | Calibrate a candidate vendor text detector. |
|
||||
| `visible_alpha_solve.py` | Rebuild visible-watermark alpha assets from controlled captures. |
|
||||
| `visible_alpha_solve.py` | Rebuild visible-watermark alpha assets from controlled captures and the cleared Qwen symbol fixture. |
|
||||
| `visible_groundtruth.py` | Consolidate blinded contact-sheet labels into ground truth. |
|
||||
| `visible_positives.py` | List corpus images carrying a registered visible mark. |
|
||||
| `visible_recall_sample.py` | Build an unbiased blinded sample for recall measurement. |
|
||||
|
||||
+119
-56
@@ -3,7 +3,8 @@
|
||||
Two jobs in one pass:
|
||||
|
||||
1. **Report** -- run ``identify`` over every image and write one CSV row per file
|
||||
(verdict, platform, confidence, watermarks, signals, integrity clashes).
|
||||
(verdict, platform, confidence, watermarks, signals, raw metadata markers,
|
||||
candidate classes, integrity clashes, and errors).
|
||||
2. **Gap audit** -- for every ``unknown``-verdict file, scan only its *metadata
|
||||
region* (PNG text/eXIf chunks, JPEG APPn segments before SOS, or the file
|
||||
head for other containers) for known provenance markers. A marker found there
|
||||
@@ -20,6 +21,8 @@ Usage:
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals \\
|
||||
--report .local-eval/detector-report.csv
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals \\
|
||||
--since 2026-09-01 --report .local-eval/detector-report-weekly.csv
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,13 +30,17 @@ from __future__ import annotations
|
||||
import csv
|
||||
import logging
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
from _plain_console import Console, Table
|
||||
|
||||
from remove_ai_watermarks.identify import _metadata_region as identify_metadata_region
|
||||
from remove_ai_watermarks.identify import identify
|
||||
from remove_ai_watermarks.metadata import _png_late_metadata
|
||||
from remove_ai_watermarks.metadata import scan_head
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
@@ -80,35 +87,31 @@ MARKERS: tuple[bytes, ...] = (
|
||||
b"Samsung Galaxy",
|
||||
)
|
||||
|
||||
|
||||
def _metadata_region(path: Path) -> bytes:
|
||||
"""Return only the bytes where provenance metadata can live, never the
|
||||
compressed pixel stream (which produces random short-token collisions)."""
|
||||
try:
|
||||
head = path.read_bytes()
|
||||
except OSError:
|
||||
return b""
|
||||
if head[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
# All ancillary metadata chunks (window=0), via the library's own walker.
|
||||
return _png_late_metadata(path, 0)
|
||||
if head[:2] == b"\xff\xd8": # JPEG: APPn segments up to Start-Of-Scan
|
||||
out = bytearray()
|
||||
p = 2
|
||||
n = len(head)
|
||||
while p + 4 <= n and head[p] == 0xFF:
|
||||
marker = head[p + 1]
|
||||
if marker == 0xDA: # SOS -> compressed scan data follows
|
||||
break
|
||||
seg_len = (head[p + 2] << 8) | head[p + 3]
|
||||
out += head[p + 4 : p + 2 + seg_len]
|
||||
p += 2 + seg_len
|
||||
return bytes(out)
|
||||
return head[:65536] # webp/avif/heif/jxl: metadata sits near the head
|
||||
REPORT_FIELDS: tuple[str, ...] = (
|
||||
"path",
|
||||
"suffix",
|
||||
"lib_version",
|
||||
"is_ai",
|
||||
"platform",
|
||||
"confidence",
|
||||
"watermarks",
|
||||
"signals",
|
||||
"integrity_clashes",
|
||||
"markers",
|
||||
"candidate_classes",
|
||||
"error",
|
||||
)
|
||||
|
||||
|
||||
def _row(rep) -> dict[str, str]: # noqa: ANN001 (ProvenanceReport)
|
||||
def _base_row(path: str, suffix: str, lib_version: str) -> dict[str, str]:
|
||||
row = dict.fromkeys(REPORT_FIELDS, "")
|
||||
row.update(path=path, suffix=suffix, lib_version=lib_version)
|
||||
return row
|
||||
|
||||
|
||||
def _row(rep, *, path: str, suffix: str, lib_version: str) -> dict[str, str]: # noqa: ANN001
|
||||
return {
|
||||
"path": "", # filled by caller (relative)
|
||||
**_base_row(path, suffix, lib_version),
|
||||
"is_ai": str(rep.is_ai_generated),
|
||||
"platform": rep.platform or "",
|
||||
"confidence": rep.confidence,
|
||||
@@ -118,6 +121,55 @@ def _row(rep) -> dict[str, str]: # noqa: ANN001 (ProvenanceReport)
|
||||
}
|
||||
|
||||
|
||||
def _marker_hits(region: bytes) -> list[str]:
|
||||
"""Return marker labels found case-insensitively in one metadata region."""
|
||||
folded = region.lower()
|
||||
return sorted({marker.decode("latin-1", "replace") for marker in MARKERS if marker.lower() in folded})
|
||||
|
||||
|
||||
def _marker_hits_for_path(path: Path) -> list[str]:
|
||||
"""Return bounded metadata hits without turning an unreadable row into a fatal scan."""
|
||||
try:
|
||||
return _marker_hits(identify_metadata_region(scan_head(path)))
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _candidate_classes(rep: Any, hits: list[str]) -> list[str]:
|
||||
"""Classify review-worthy outcomes without treating them as proven bugs."""
|
||||
classes: list[str] = []
|
||||
if hits and not rep.is_ai_generated and not rep.signals:
|
||||
classes.append("blind_marker")
|
||||
if rep.is_ai_generated and not rep.platform:
|
||||
classes.append("unattributed_ai")
|
||||
elif rep.signals and not rep.platform:
|
||||
classes.append("unattributed_signal")
|
||||
if rep.integrity_clashes:
|
||||
classes.append("integrity_clash")
|
||||
return classes
|
||||
|
||||
|
||||
def _day_of(path: Path, corpus: Path) -> date | None:
|
||||
"""Read the leading YYYY-MM-DD corpus segment, if this layout has one."""
|
||||
try:
|
||||
segment = path.relative_to(corpus).parts[0]
|
||||
return date.fromisoformat(segment)
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _files(corpus: Path, since: date | None) -> list[Path]:
|
||||
"""Return the deterministic corpus walk, optionally bounded by its date segment."""
|
||||
if since is None:
|
||||
return sorted(path for path in corpus.rglob("*") if path.is_file())
|
||||
roots = [
|
||||
path
|
||||
for path in corpus.iterdir()
|
||||
if path.is_dir() and (day := _day_of(path, corpus)) is not None and day >= since
|
||||
]
|
||||
return sorted(path for root in roots for path in root.rglob("*") if path.is_file())
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--corpus",
|
||||
@@ -132,82 +184,93 @@ def _row(rep) -> dict[str, str]: # noqa: ANN001 (ProvenanceReport)
|
||||
default=None,
|
||||
help="Write the per-file CSV here (default: <corpus>/../detector_report.csv).",
|
||||
)
|
||||
@click.option(
|
||||
"--since",
|
||||
type=click.DateTime(formats=["%Y-%m-%d"]),
|
||||
default=None,
|
||||
help="Only scan files under YYYY-MM-DD corpus directories on or after this date.",
|
||||
)
|
||||
@click.option("--limit", type=int, default=0, help="Scan at most N files (0 = all).")
|
||||
def main(corpus: Path, report: Path | None, limit: int) -> None:
|
||||
def main(corpus: Path, report: Path | None, since, limit: int) -> None: # noqa: ANN001
|
||||
logging.basicConfig(level=logging.WARNING, format="%(message)s")
|
||||
report = report or corpus.parent / "detector_report.csv"
|
||||
|
||||
files = sorted(p for p in corpus.rglob("*") if p.is_file())
|
||||
since_date = since.date() if since is not None else None
|
||||
files = _files(corpus, since_date)
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
console.print(f"Scanning [bold]{len(files)}[/bold] files under {corpus} ...")
|
||||
|
||||
verdicts: Counter[str] = Counter()
|
||||
platforms: Counter[str] = Counter()
|
||||
gap_tokens: Counter[str] = Counter()
|
||||
gaps: list[tuple[str, list[str]]] = []
|
||||
shapes: Counter[str] = Counter()
|
||||
rows: list[dict[str, str]] = []
|
||||
errors = 0
|
||||
lib_version = version("remove-ai-watermarks")
|
||||
|
||||
with click.progressbar(files, label="identify") as bar:
|
||||
for p in bar:
|
||||
rel = str(p.relative_to(corpus))
|
||||
shapes[p.suffix.lower() or "(none)"] += 1
|
||||
try:
|
||||
rep = identify(p)
|
||||
except Exception as exc:
|
||||
log.warning("identify failed on %s: %s", rel, exc)
|
||||
errors += 1
|
||||
row = _base_row(rel, p.suffix.lower(), lib_version)
|
||||
row["candidate_classes"] = "identify_error"
|
||||
row["error"] = f"{type(exc).__name__}: {exc}"[:300]
|
||||
hits = _marker_hits_for_path(p)
|
||||
row["markers"] = "|".join(hits)
|
||||
rows.append(row)
|
||||
continue
|
||||
row = _row(rep)
|
||||
row["path"] = rel
|
||||
row = _row(rep, path=rel, suffix=p.suffix.lower(), lib_version=lib_version)
|
||||
# identify() has already populated scan_head's bounded cache, so this
|
||||
# reuses the exact metadata region rather than reading the file again.
|
||||
hits = _marker_hits_for_path(p)
|
||||
classes = _candidate_classes(rep, hits)
|
||||
row["markers"] = "|".join(hits)
|
||||
row["candidate_classes"] = "|".join(classes)
|
||||
rows.append(row)
|
||||
if rep.is_ai_generated:
|
||||
verdicts["ai"] += 1
|
||||
platforms[rep.platform or "?"] += 1
|
||||
continue
|
||||
verdicts["unknown"] += 1
|
||||
# A gap candidate is a file identify is *blind* to (no signal at all)
|
||||
# yet whose metadata carries a known marker. A file that produced a
|
||||
# signal but no AI verdict (e.g. an ASUS Gallery C2PA signer, which we
|
||||
# attribute but do not call AI) is handled correctly -- not a gap.
|
||||
if rep.signals:
|
||||
continue
|
||||
region = _metadata_region(p)
|
||||
hits = sorted({m.decode("latin-1", "replace") for m in MARKERS if m in region})
|
||||
if hits:
|
||||
gaps.append((rel, hits))
|
||||
gap_tokens.update(hits)
|
||||
|
||||
report.parent.mkdir(parents=True, exist_ok=True)
|
||||
with report.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(
|
||||
f,
|
||||
fieldnames=["path", "is_ai", "platform", "confidence", "watermarks", "signals", "integrity_clashes"],
|
||||
)
|
||||
writer = csv.DictWriter(f, fieldnames=REPORT_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
console.print(f"\nWrote [bold]{len(rows)}[/bold] rows -> {report}")
|
||||
|
||||
console.print(f"\n[bold]Verdicts:[/bold] AI {verdicts['ai']} | unknown {verdicts['unknown']} | errors {errors}")
|
||||
console.print("[bold]Input shapes:[/bold] " + " | ".join(f"{name} {n}" for name, n in shapes.most_common()))
|
||||
plat = Table(title="AI platforms", show_header=False)
|
||||
for name, n in platforms.most_common():
|
||||
plat.add_row(str(n), name)
|
||||
console.print(plat)
|
||||
|
||||
if gaps:
|
||||
candidates = [row for row in rows if row["candidate_classes"]]
|
||||
if candidates:
|
||||
candidate_counts = Counter(candidate for row in candidates for candidate in row["candidate_classes"].split("|"))
|
||||
gap_tokens = Counter(marker for row in rows for marker in row["markers"].split("|") if marker)
|
||||
console.print(
|
||||
f"\n[bold red]Gap candidates[/bold red]: {len(gaps)} unknown files carry a known "
|
||||
f"marker in their metadata region (potential undetected serialization/generator):"
|
||||
f"\n[bold red]Review candidates[/bold red]: {len(candidates)} file(s) "
|
||||
f"({', '.join(f'{name}={n}' for name, n in candidate_counts.most_common())})"
|
||||
)
|
||||
tok = Table(title="markers seen in unknown files")
|
||||
tok = Table(title="metadata markers seen across the run")
|
||||
tok.add_column("count", justify="right")
|
||||
tok.add_column("marker")
|
||||
for name, n in gap_tokens.most_common():
|
||||
tok.add_row(str(n), name)
|
||||
console.print(tok)
|
||||
for rel, hits in gaps:
|
||||
console.print(f" {rel} -> {', '.join(hits)}")
|
||||
for row in candidates:
|
||||
detail = f"; markers={row['markers'].replace('|', ', ')}" if row["markers"] else ""
|
||||
console.print(f" {row['path']} -> {row['candidate_classes'].replace('|', ', ')}{detail}")
|
||||
else:
|
||||
console.print("\n[green]No gap candidates: every unknown file is metadata-free.[/green]")
|
||||
console.print("\n[green]No review candidates in this run.[/green]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export the frozen CLIP-L vision tower as a static FP32 ONNX graph.
|
||||
|
||||
Run from the repository root with the ONNX exporter installed temporarily:
|
||||
|
||||
uv run --with onnx python scripts/export_photo_classify_onnx.py \
|
||||
--checkpoint /path/to/clip-l-ft.pt \
|
||||
--out /path/to/clip-l-ft-vision-fp32.onnx
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import inspect
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VisionEmbedding(nn.Module):
|
||||
"""Only the vision modules used by the classifier's embedding path."""
|
||||
|
||||
def __init__(self, headed: Any) -> None:
|
||||
super().__init__()
|
||||
self.vision_model = headed.clip.vision_model
|
||||
self.visual_projection = headed.clip.visual_projection
|
||||
|
||||
def forward(self, pixel_values: Any) -> Any:
|
||||
pooled = self.vision_model(pixel_values=pixel_values).pooler_output
|
||||
return nn.functional.normalize(self.visual_projection(pooled), dim=-1)
|
||||
|
||||
|
||||
def export_onnx(checkpoint: Path, output: Path) -> None:
|
||||
"""Write the static batch-one graph consumed by the ONNX runtime backend."""
|
||||
from remove_ai_watermarks._internal.clip_l_ft import SIZE, load_headed_clip
|
||||
|
||||
checkpoint = checkpoint.expanduser().resolve()
|
||||
output = output.expanduser().resolve()
|
||||
if not checkpoint.is_file():
|
||||
raise SystemExit(f"missing checkpoint: {checkpoint}")
|
||||
if output.exists():
|
||||
raise SystemExit(f"refusing to overwrite existing output: {output}")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
model = VisionEmbedding(load_headed_clip(checkpoint, torch.device("cpu"))).eval()
|
||||
export_options: dict[str, Any] = {}
|
||||
if "dynamo" in inspect.signature(torch.onnx.export).parameters:
|
||||
export_options["dynamo"] = False
|
||||
with torch.inference_mode():
|
||||
torch.onnx.export(
|
||||
model,
|
||||
(torch.zeros(1, 3, SIZE, SIZE, dtype=torch.float32),),
|
||||
output,
|
||||
input_names=["pixel_values"],
|
||||
output_names=["embedding"],
|
||||
opset_version=17,
|
||||
do_constant_folding=True,
|
||||
**export_options,
|
||||
)
|
||||
with output.open("rb") as handle:
|
||||
digest = hashlib.file_digest(handle, "sha256").hexdigest()
|
||||
log.info("wrote %s bytes=%d sha256=%s", output, output.stat().st_size, digest)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
export_onnx(args.checkpoint, args.out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -2,8 +2,8 @@
|
||||
"""Publish the photo-classify freeze to Hugging Face wiltodelta/raiw-photo-classify.
|
||||
|
||||
Manual. Not a library release step. Weights are not in git: pass --src, a
|
||||
directory that holds clip-l-ft.pt and the probe, with detector.pt and
|
||||
provider.pt either at the root or in run1/.
|
||||
directory that holds clip-l-ft.pt, clip-l-ft-vision-fp32.onnx, and the probe,
|
||||
with detector.pt and provider.pt either at the root or in run1/.
|
||||
|
||||
uv run python scripts/publish_photo_classify_hf.py --src DIR
|
||||
uv run python scripts/publish_photo_classify_hf.py --card-only
|
||||
@@ -28,10 +28,11 @@ log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-photo-classify"
|
||||
CLIP_FILE = "clip-l-ft.pt"
|
||||
CLIP_ONNX_FILE = "clip-l-ft-vision-fp32.onnx"
|
||||
PROBE_FILE = "probe-weights-clip-l-ft.npz"
|
||||
DETECTOR_FILE = "detector.pt"
|
||||
PROVIDER_FILE = "provider.pt"
|
||||
WEIGHT_FILES = (CLIP_FILE, PROBE_FILE, DETECTOR_FILE, PROVIDER_FILE)
|
||||
WEIGHT_FILES = (CLIP_FILE, CLIP_ONNX_FILE, PROBE_FILE, DETECTOR_FILE, PROVIDER_FILE)
|
||||
GATE_FILE = "receipt-gate-2026-09-02.npz"
|
||||
GATE_STABLE_FILE = "receipt-gate.npz"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -68,7 +69,7 @@ def stage_card(dest: Path) -> None:
|
||||
shutil.copy2(readme, dest / "README.md")
|
||||
shutil.copy2(operating, dest / "operating-point.json")
|
||||
# The gate head versions with the model: card mode carries it even when
|
||||
# the four freeze weight files are not being re-uploaded.
|
||||
# the five freeze weight files are not being re-uploaded.
|
||||
shutil.copy2(GATE_SOURCE, dest / GATE_FILE)
|
||||
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ OUT = REPO / ".local-eval" / "sidecar-regression.jsonl"
|
||||
|
||||
# Map a watermark description to a stable behavior family. Order matters: the first
|
||||
# matching pattern wins, so put the specific tokens above the generic ones.
|
||||
_FAMILIES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
_LEGACY_FAMILIES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("synthid", ("synthid",)),
|
||||
("visible_sparkle", ("sparkle",)),
|
||||
("visible_doubao", ("豆包", "doubao")),
|
||||
@@ -71,27 +71,61 @@ _FAMILIES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("trustmark", ("trustmark",)),
|
||||
("open_invisible", ("open invisible", "dwt", "stable diffusion xl")),
|
||||
("xai_signature", ("xai", "grok signature")),
|
||||
("samsung_genai", ("genaitype", "photoeditor_re_edit")),
|
||||
("soft_binding", ("soft binding", "soft-binding", "invismark")),
|
||||
("c2pa_cloud", ("cloud manifest",)),
|
||||
("hf_job", ("hf-job", "hugging face job")),
|
||||
("exif_generator", ("exif", "software tag", "png text")),
|
||||
)
|
||||
|
||||
|
||||
def _registered_visible_family(description: str) -> str | None:
|
||||
"""Map current mark labels from the registry instead of copying its keys here."""
|
||||
from remove_ai_watermarks.watermark_registry import known_marks
|
||||
|
||||
low = description.lower()
|
||||
matches = [
|
||||
(mark.label.lower() in low, len(mark.label), mark)
|
||||
for mark in known_marks()
|
||||
if mark.key != "gemini" and (mark.label.lower() in low or mark.key.replace("_", " ") in low)
|
||||
]
|
||||
if not matches:
|
||||
return None
|
||||
_, _, mark = max(matches, key=lambda candidate: candidate[:2])
|
||||
return f"visible_{mark.key}"
|
||||
|
||||
|
||||
def family_of(description: str) -> str:
|
||||
low = description.lower()
|
||||
for name, tokens in _FAMILIES:
|
||||
if family := _registered_visible_family(description):
|
||||
return family
|
||||
for name, tokens in _LEGACY_FAMILIES:
|
||||
if any(t in low for t in tokens):
|
||||
return name
|
||||
return "other"
|
||||
|
||||
|
||||
def families(descriptions: list[str]) -> set[str]:
|
||||
return {family_of(d) for d in descriptions or []}
|
||||
return {family for description in descriptions or [] if (family := family_of(description)) != "other"}
|
||||
|
||||
|
||||
def sidecar_families(sidecar: dict) -> set[str]:
|
||||
"""Combine stable names with legacy-only families such as SynthID."""
|
||||
signals = sidecar.get("signals") or []
|
||||
return {str(signal) for signal in signals} | families(sidecar.get("watermarks") or [])
|
||||
|
||||
|
||||
def report_families(report: object) -> set[str]:
|
||||
"""Combine current Signal.name values with watermark-only families."""
|
||||
signal_names = {str(signal.name) for signal in getattr(report, "signals", []) or []}
|
||||
return signal_names | families(list(getattr(report, "watermarks", []) or []))
|
||||
|
||||
|
||||
def compare(sidecar: dict, report: object) -> dict:
|
||||
"""Classify the difference between a recorded verdict and a fresh one."""
|
||||
old_ai, new_ai = sidecar.get("is_ai_generated"), getattr(report, "is_ai_generated", None)
|
||||
old_fam = families(sidecar.get("watermarks") or [])
|
||||
new_fam = families(list(getattr(report, "watermarks", []) or []))
|
||||
old_fam = sidecar_families(sidecar)
|
||||
new_fam = report_families(report)
|
||||
old_plat, new_plat = sidecar.get("platform"), getattr(report, "platform", None)
|
||||
old_conf, new_conf = sidecar.get("confidence"), getattr(report, "confidence", None)
|
||||
|
||||
|
||||
@@ -23,11 +23,18 @@ edges:
|
||||
engine reads ``alpha = max(R,G,B)/255`` directly (no background fit). Its assets are
|
||||
the sparkle-on-black capture cropped to two fixed logo sizes (``gemini_bg_{96,48}.png``).
|
||||
|
||||
**Qwen symbol** is solved from the cleared Qwen Create provider fixture. The symbol
|
||||
occupies a recorded square in a smooth linen region. A cubic surface fitted outside
|
||||
that square estimates the local background; the three interior components of the
|
||||
positive residual are the symbol. Keeping the extraction as code makes the bundled
|
||||
asset reproducible instead of a hand-cropped copy of the fixture.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python scripts/visible_alpha_solve.py doubao
|
||||
uv run python scripts/visible_alpha_solve.py jimeng
|
||||
uv run python scripts/visible_alpha_solve.py gemini
|
||||
uv run python scripts/visible_alpha_solve.py qwen_symbol
|
||||
uv run python scripts/visible_alpha_solve.py all
|
||||
"""
|
||||
|
||||
@@ -115,6 +122,12 @@ _GEMINI_ASSETS: dict[int, Path] = {
|
||||
48: _ROOT / "src" / "remove_ai_watermarks" / "assets" / "gemini_bg_48.png",
|
||||
}
|
||||
|
||||
_QWEN_SYMBOL_CAPTURE = _ROOT / "data" / "fixtures" / "provenance" / "qwen-create-qwen-image-2.png"
|
||||
_QWEN_SYMBOL_ASSET = _ROOT / "src" / "remove_ai_watermarks" / "assets" / "qwen_symbol_alpha.png"
|
||||
# Recorded provider-symbol box (x, y, width, height) in the 2048-square fixture.
|
||||
_QWEN_SYMBOL_BOX = (1925, 1917, 70, 70)
|
||||
_QWEN_SYMBOL_BG_PAD = 45
|
||||
|
||||
|
||||
def _union_bbox(mask: NDArray[np.uint8], err: str) -> tuple[int, int, int, int]:
|
||||
"""Union bbox ``(x0, x1, y0, y1)`` of ``mask``'s connected components with area
|
||||
@@ -246,10 +259,48 @@ def solve_gemini() -> dict[int, NDArray[np.uint8]]:
|
||||
return {size: cv2.resize(crop, (size, size), interpolation=cv2.INTER_AREA) for size in _GEMINI_ASSETS}
|
||||
|
||||
|
||||
def solve_qwen_symbol() -> NDArray[np.uint8]:
|
||||
"""Recover the three-lobe Qwen symbol from its cleared provider fixture."""
|
||||
source = image_io.imread(str(_QWEN_SYMBOL_CAPTURE), cv2.IMREAD_COLOR)
|
||||
if source is None:
|
||||
raise FileNotFoundError(f"missing Qwen symbol capture {_QWEN_SYMBOL_CAPTURE}")
|
||||
x, y, width, height = _QWEN_SYMBOL_BOX
|
||||
pad = _QWEN_SYMBOL_BG_PAD
|
||||
crop = source[y - pad : y + height + pad, x - pad : x + width + pad].astype(np.float32)
|
||||
excluded = np.zeros(crop.shape[:2], dtype=bool)
|
||||
excluded[pad - 8 : pad + height + 8, pad - 8 : pad + width + 8] = True
|
||||
background = _cubic_background(crop, excluded)
|
||||
alpha = np.clip(
|
||||
((crop - background) / np.clip(255.0 - background, 1e-3, None)).mean(axis=2),
|
||||
0.0,
|
||||
1.0,
|
||||
)[pad : pad + height, pad : pad + width]
|
||||
|
||||
body = (alpha > 0.10).astype(np.uint8)
|
||||
count, labels, stats, _centroids = cv2.connectedComponentsWithStats(body, connectivity=8)
|
||||
parts = [
|
||||
index
|
||||
for index in range(1, count)
|
||||
if stats[index, cv2.CC_STAT_AREA] >= 300
|
||||
and stats[index, cv2.CC_STAT_LEFT] > 0
|
||||
and stats[index, cv2.CC_STAT_TOP] > 0
|
||||
and stats[index, cv2.CC_STAT_LEFT] + stats[index, cv2.CC_STAT_WIDTH] < width
|
||||
and stats[index, cv2.CC_STAT_TOP] + stats[index, cv2.CC_STAT_HEIGHT] < height
|
||||
]
|
||||
if len(parts) != 3:
|
||||
raise ValueError(f"expected three Qwen symbol lobes, found {len(parts)}")
|
||||
selected = np.isin(labels, parts)
|
||||
ys, xs = np.where(selected)
|
||||
tight = np.where(selected, alpha, 0.0)[ys.min() : ys.max() + 1, xs.min() : xs.max() + 1]
|
||||
tight[tight < 0.08] = 0.0
|
||||
log.info("qwen-symbol: alpha %dx%d max %.3f", tight.shape[1], tight.shape[0], float(tight.max()))
|
||||
return (tight * 255.0).astype(np.uint8)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("engine", type=click.Choice([*_SPECS, "gemini", "all"]))
|
||||
@click.argument("engine", type=click.Choice([*_SPECS, "gemini", "qwen_symbol", "all"]))
|
||||
def main(engine: str) -> None:
|
||||
"""Rebuild the alpha asset(s) for ENGINE (doubao / jimeng / gemini / all)."""
|
||||
"""Rebuild the alpha asset(s) for ENGINE."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
def _write(path: Path, img: NDArray[np.uint8], label: str) -> None:
|
||||
@@ -265,6 +316,8 @@ def main(engine: str) -> None:
|
||||
if engine in ("gemini", "all"):
|
||||
for size, img in solve_gemini().items():
|
||||
_write(_GEMINI_ASSETS[size], img, f"gemini-{size}")
|
||||
if engine in ("qwen_symbol", "all"):
|
||||
_write(_QWEN_SYMBOL_ASSET, solve_qwen_symbol(), "qwen-symbol")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "remove-ai-watermarks",
|
||||
"displayName": "Remove AI Watermarks",
|
||||
"version": "1.0.5",
|
||||
"version": "1.0.8",
|
||||
"description": "Identify and remove AI provenance marks from images and video the user generated or edited, using the remove-ai-watermarks CLI.",
|
||||
"author": {
|
||||
"name": "wiltodelta",
|
||||
|
||||
@@ -5,7 +5,7 @@ license: Apache-2.0
|
||||
compatibility: Python 3.11-3.14. Invisible image removal needs NVIDIA CUDA. Some video writes need ffmpeg. Installer may be uv, pipx, or pip.
|
||||
metadata:
|
||||
author: wiltodelta
|
||||
version: "1.0.5"
|
||||
version: "1.0.8"
|
||||
homepage: https://raiw.cc
|
||||
repository: https://github.com/wiltodelta/remove-ai-watermarks
|
||||
---
|
||||
@@ -164,6 +164,52 @@ watermark" or "this is clean".
|
||||
verdict, because a pixel watermark such as SynthID cannot be detected here once
|
||||
its metadata proxy is gone. Repeat that caveat; do not summarize it away.
|
||||
|
||||
## Examples
|
||||
|
||||
### Identify without overstating the result
|
||||
|
||||
Input: "Was `photo.png` AI-generated?"
|
||||
|
||||
Action: probe the machine, then run:
|
||||
|
||||
```bash
|
||||
remove-ai-watermarks identify photo.png
|
||||
```
|
||||
|
||||
Output after exit 2 from a command that reached the image:
|
||||
|
||||
> Origin unknown. The command found no local provenance signal. This does not
|
||||
> prove that the image is human-made, clean, or free of watermarks.
|
||||
|
||||
### Remove a user-named visible AI label
|
||||
|
||||
Input: "Remove the Gemini sparkle from my `generated.png`."
|
||||
|
||||
Action: after a successful probe, run:
|
||||
|
||||
```bash
|
||||
remove-ai-watermarks visible generated.png --mark gemini -o generated-clean.png
|
||||
```
|
||||
|
||||
Output after exit 0 and confirming that the file exists:
|
||||
|
||||
> Removed the targeted Gemini mark. Output: `generated-clean.png`.
|
||||
|
||||
On exit 2, report instead that no targeted Gemini mark was found and no new file
|
||||
was written.
|
||||
|
||||
### Route an unavailable invisible removal safely
|
||||
|
||||
Input: "Remove SynthID from my image."
|
||||
|
||||
Action: run the probe first. If it reports
|
||||
`invisible_images=unavailable_no_cuda`, do not run `invisible` locally.
|
||||
|
||||
Output:
|
||||
|
||||
> Invisible image removal needs NVIDIA CUDA, which is unavailable on this
|
||||
> machine. No local removal ran. You can use https://raiw.cc instead.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- Without CUDA, `all` does one of two opposite things, and only the probe knows
|
||||
@@ -186,9 +232,8 @@ its metadata proxy is gone. Repeat that caveat; do not summarize it away.
|
||||
- `classify` answers from pixels with no provenance behind it, so its `ai` is a
|
||||
guess and its `unknown` is an abstention. Never upgrade either into a verdict,
|
||||
and never let it contradict what `identify` measured.
|
||||
- A Meta AI (Muse Image) export can retain standalone IPTC metadata that enables
|
||||
heuristic attribution and automatic routing. The IPTC code is not unique to Meta
|
||||
and does not detect Content Seal in pixels. If metadata was stripped, use
|
||||
- A Meta AI (Muse Image) export can retain standalone IPTC metadata, but the shared
|
||||
code does not identify Meta or detect Content Seal in pixels. Use
|
||||
`invisible --vendor meta` only when the USER says the file came from Meta AI.
|
||||
Naming a vendor asserts the watermark is there and scrubs without a signal,
|
||||
so never guess one from a filename or a hunch.
|
||||
|
||||
@@ -26,7 +26,8 @@ Image mark keys: `gemini`, `doubao`, `jimeng`, `qwen`, `kling`, `yuanbao`,
|
||||
`--mark auto` removes every selected match. `microsoft` is the top-right AI
|
||||
badge, separate from the invisible InvisMark watermark on the same vendor's
|
||||
images. `jimeng_pill` is a weak detector and needs corroboration. The compact
|
||||
`liblib_pill` likewise requires LiblibAI metadata or the bottom-center wordmark.
|
||||
`liblib_pill` likewise requires LiblibAI metadata or the bottom-center wordmark
|
||||
(since CLI 0.39.0).
|
||||
|
||||
```bash
|
||||
remove-ai-watermarks erase image.png --region 1640,1930,400,100 -o clean.png
|
||||
@@ -43,7 +44,9 @@ remove-ai-watermarks classify image.png --json
|
||||
```
|
||||
|
||||
Pixel classifier, needs the `classify` extra. It is NOT provenance: it guesses
|
||||
from pixels alone and `identify` never runs it. Labels are `ai` (both models
|
||||
from pixels alone and `identify` never runs it. The CLI has no ONNX flag; the
|
||||
optional CPU ONNX runtime is library-only (`backend="onnx"`) and needs
|
||||
`classify-onnx` (since CLI 0.40.0). Labels are `ai` (both models
|
||||
agree), `human`, and `unknown` -- `unknown` is an abstention, not a negative.
|
||||
Photographed receipts abstain to `unknown` through the bundled receipt gate
|
||||
(since CLI 0.38.0), so a receipt is not reported as `ai`. Offer `classify`
|
||||
@@ -77,10 +80,9 @@ cohort asserts the watermark IS present, so it also runs the scrub without a
|
||||
local signal, exactly like `--force`. Set it only when the USER says where the
|
||||
file came from. `--vendor` is on `invisible`, `all` and `batch`.
|
||||
|
||||
Meta Muse Image exports can retain standalone IPTC metadata that enables
|
||||
heuristic attribution and automatic routing. That generic code is not unique
|
||||
to Meta and does not detect Content Seal in pixels. After metadata is stripped,
|
||||
`--vendor meta` supplies the route only when the user confirms the source.
|
||||
Meta Muse Image exports can retain standalone IPTC metadata, but that generic
|
||||
code does not identify Meta or detect Content Seal in pixels. `--vendor meta`
|
||||
supplies the route only when the user confirms the source.
|
||||
There is no `--model`, `--steps`, or `--guidance-scale` flag, and no `--device`
|
||||
on the image path: auto-detection already finds the only device these profiles
|
||||
run on.
|
||||
@@ -88,8 +90,8 @@ run on.
|
||||
Pipeline profiles: `qwen-zimage` (the default), `sdxl-zimage`, `chroma-zimage`,
|
||||
`auto`. All are CUDA-only and all install from the same `qwen-zimage` extra.
|
||||
`auto` picks the engine from the file's own provenance -- chroma-zimage for
|
||||
Microsoft, qwen-zimage for OpenAI, Google, Meta and unknown -- so prefer it
|
||||
over naming an engine when the user has no reason to care. `sdxl-zimage` is the
|
||||
Microsoft, qwen-zimage for OpenAI (since CLI 0.39.0), Google, Meta and unknown
|
||||
-- so prefer it over naming an engine when the user has no reason to care. `sdxl-zimage` is the
|
||||
heavier alternative and `chroma-zimage` is the Apache-2.0 global stage.
|
||||
|
||||
```bash
|
||||
|
||||
@@ -66,6 +66,7 @@ ffmpeg is missing, or the user asked about Homebrew, pipx, or pip.
|
||||
| Verified text restoration (`--text-manifest`) | add `text-restoration` |
|
||||
| Open DWT-DCT detect | add `detect` |
|
||||
| Pixel classifier (`classify`) | add `classify` |
|
||||
| CPU ONNX photo classifier, library `backend="onnx"` only (since CLI 0.40.0) | add `classify-onnx` |
|
||||
| Adobe TrustMark on Python 3.11-3.12 | add `trustmark` |
|
||||
| Every production extra on this Python | `all` |
|
||||
|
||||
|
||||
@@ -24,13 +24,15 @@ from typing import Any
|
||||
_GPU_MARKERS = ("GPU ", "gpu:")
|
||||
# Below this, the CLI predates flags and behavior this skill documents. Raise it in
|
||||
# the same change as any reference that names a flag the previous release lacked:
|
||||
# 0.37.0 is here because `classify` arrived with it (`--pipeline auto` needed
|
||||
# 0.36.0; chroma-zimage and the doubao video mark 0.35.0;
|
||||
# the `microsoft` mark 0.34.0; `--vendor` 0.32.0), and a stale floor told an agent
|
||||
# its CLI was current right before it typed a value that build rejects.
|
||||
# tests/test_agent_skill.py pins it against the package version, so it can never
|
||||
# advertise a release that does not exist.
|
||||
MIN_CLI_VERSION = (0, 38, 0)
|
||||
# 0.40.0 is here because the skill names the `classify-onnx` extra; 0.39.0 is
|
||||
# here because the `liblib_pill` mark and `auto` routing OpenAI provenance to
|
||||
# qwen-zimage arrived with it; 0.38.0 added the receipt gate; `classify` needed
|
||||
# 0.37.0 (`--pipeline auto` 0.36.0; chroma-zimage and the doubao video mark
|
||||
# 0.35.0; the `microsoft` mark 0.34.0; `--vendor` 0.32.0), and a stale floor
|
||||
# told an agent its CLI was current right before it typed a value that build
|
||||
# rejects. tests/test_agent_skill.py pins it against the package version and
|
||||
# against every "(since CLI X.Y.Z)" annotation in the skill.
|
||||
MIN_CLI_VERSION = (0, 40, 0)
|
||||
# Markers a build without the pixel stack prints. The first is the current CLI's own
|
||||
# install hint; the rest are what older builds emit before the guard existed, and the
|
||||
# Homebrew formula ships exactly such a build.
|
||||
|
||||
@@ -33,7 +33,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.39.0"
|
||||
__version__ = "0.40.1"
|
||||
|
||||
__all__ = [
|
||||
"BatchItemResult",
|
||||
|
||||
@@ -52,12 +52,19 @@ class HeadedCLIP(nn.Module):
|
||||
|
||||
|
||||
def load_headed_clip(checkpoint: Path, device: torch.device) -> HeadedCLIP:
|
||||
"""Build CLIP-L from config and load the freeze state dict."""
|
||||
"""Build CLIP-L without discarded random weights and load the freeze state dict."""
|
||||
import torch
|
||||
from transformers import CLIPConfig, CLIPModel
|
||||
from transformers.initialization import no_init_weights
|
||||
|
||||
clip = CLIPModel(CLIPConfig.from_pretrained(BACKBONE))
|
||||
model = HeadedCLIP(clip)
|
||||
# The frozen checkpoint replaces every parameter. Normal construction spends
|
||||
# most of a CPU cold start randomly initializing 427M parameters first.
|
||||
# ``no_init_weights`` preserves constructor-created buffers while suppressing
|
||||
# only those redundant parameter fills; unlike a meta-device construction,
|
||||
# it cannot leave non-persistent CLIP buffers unmaterialized.
|
||||
with no_init_weights():
|
||||
clip = CLIPModel(CLIPConfig.from_pretrained(BACKBONE))
|
||||
model = HeadedCLIP(clip)
|
||||
state = torch.load(checkpoint, map_location="cpu", weights_only=True)
|
||||
model.load_state_dict(state, strict=True)
|
||||
return model.to(device).eval()
|
||||
@@ -72,6 +79,18 @@ def load_processor() -> Any:
|
||||
return processor
|
||||
|
||||
|
||||
def load_onnx_vision(model: Path) -> Any:
|
||||
"""Load the portable CPU session for the vision-only FP32 export."""
|
||||
import onnxruntime as ort
|
||||
|
||||
options = ort.SessionOptions()
|
||||
# Runtime graph optimization rebuilds this 1.2 GB static graph on every cold
|
||||
# process. The export is already constant-folded; disabling that repeated pass
|
||||
# reduced local session creation without changing the graph or its outputs.
|
||||
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
|
||||
return ort.InferenceSession(str(model), sess_options=options, providers=["CPUExecutionProvider"])
|
||||
|
||||
|
||||
def embed_image(model: HeadedCLIP, processor: Any, image: Image.Image, device: torch.device) -> NDArray[Any]:
|
||||
"""Return a 768-d L2-normalized CLIP-L-ft vector."""
|
||||
import numpy as np
|
||||
@@ -84,3 +103,14 @@ def embed_image(model: HeadedCLIP, processor: Any, image: Image.Image, device: t
|
||||
with torch.inference_mode():
|
||||
vector = model.embed(pixels).float().cpu().numpy()[0]
|
||||
return np.asarray(vector, dtype=np.float64)
|
||||
|
||||
|
||||
def embed_image_onnx(session: Any, processor: Any, image: Image.Image) -> NDArray[Any]:
|
||||
"""Return the vision-only ONNX export's normalized 768-d vector."""
|
||||
import numpy as np
|
||||
|
||||
boxed = letterbox(image)
|
||||
pixels = processor(images=[boxed], return_tensors="np")["pixel_values"]
|
||||
inputs = {"pixel_values": np.asarray(pixels, dtype=np.float32)}
|
||||
vector = session.run(["embedding"], inputs)[0][0]
|
||||
return np.asarray(vector, dtype=np.float64)
|
||||
@@ -231,15 +231,3 @@ AI_GENERATOR_TOKENS = frozenset(
|
||||
|
||||
_C2PA_ACTION_NAMES = _tokens("created|converted|edited|filtered|cropped|resized|opened|placed|watermarked.unbound")
|
||||
C2PA_ACTIONS = {f"c2pa.{action}".encode(): action for action in _C2PA_ACTION_NAMES}
|
||||
|
||||
|
||||
# TC260 producer identity -> the mark key whose vendor signs with it now lives on the
|
||||
# registry rows (``KnownMark.tc260_producer_codes``, read through
|
||||
# ``watermark_registry.tc260_producer_vendors``). Keeping the codes beside the mark is
|
||||
# what stops a newly registered TC260 vendor from silently falling back to ByteDance.
|
||||
#
|
||||
# What a TC260 label confirms when its producer is absent or unmapped. Historical
|
||||
# behavior, kept as the fallback so an unrecognized producer never regresses to no
|
||||
# relaxation at all: ByteDance's two products are the ones the relaxed band was
|
||||
# calibrated on (see _text_mark_engine._DEFAULT_PROVENANCE_NCC_FACTOR).
|
||||
TC260_FALLBACK_VENDORS: frozenset[str] = frozenset({"doubao", "jimeng"})
|
||||
@@ -344,6 +344,68 @@ def is_isobmff(data: bytes) -> bool:
|
||||
return len(data) >= 8 and data[4:8] == b"ftyp"
|
||||
|
||||
|
||||
# What a HEIF ``meta`` box's first child can be; used to probe whether the box
|
||||
# carries the ISO FullBox header (children start 4 bytes into the payload) or
|
||||
# the QuickTime bare form (children start at 0).
|
||||
_HEIF_META_CHILD_TYPES = frozenset({b"hdlr", b"pitm", b"iinf", b"iprp", b"iloc", b"iref", b"idat"})
|
||||
|
||||
|
||||
def heif_transform_applied(path: str | Path) -> bool:
|
||||
"""Whether a HEIF/AVIF file declares an effective container transform.
|
||||
|
||||
Scans the ``meta`` box's ``iprp``/``ipco`` property store for an ``irot``
|
||||
box with a non-zero angle or any ``imir`` box. libheif applies those
|
||||
transforms on decode, and that is the only signal separating a re-encoded
|
||||
HEIF -- the writer baked the rotation as ``irot`` and the decode comes back
|
||||
upright -- from a camera-style one that stores the raw raster and carries
|
||||
the rotation only in its EXIF payload: pillow-heif blanks that tag on open
|
||||
for both and stashes the real value in ``info["original_orientation"]"
|
||||
(issue #105). Association through ``ipma`` is not re-derived: a stray
|
||||
unassociated ``irot`` reads as "transformed", the direction that drops the
|
||||
tag and keeps today's tagless output rather than risking a double rotation.
|
||||
A malformed or missing structure reads as "no transform".
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
sniff = f.read(8)
|
||||
if len(sniff) < 8 or sniff[4:8] != b"ftyp":
|
||||
return False
|
||||
f.seek(0, 2)
|
||||
file_size = f.tell()
|
||||
for _start, end, box_type, payload_off in iter_file_boxes(f, 0, file_size):
|
||||
if box_type != b"meta":
|
||||
continue
|
||||
# ISO serializes ``meta`` as a FullBox, so children start 4 bytes
|
||||
# into the payload; the QuickTime bare form starts at 0. The form
|
||||
# whose first child has a known type is the real one.
|
||||
children = None
|
||||
for offset in (4, 0):
|
||||
first = _read_box_header(f, payload_off + offset, end)
|
||||
if first is not None and first[1] in _HEIF_META_CHILD_TYPES:
|
||||
children = iter_file_boxes(f, payload_off + offset, end)
|
||||
break
|
||||
if children is None:
|
||||
return False
|
||||
for _s, iprp_end, iprp_type, iprp_payload in children:
|
||||
if iprp_type != b"iprp":
|
||||
continue
|
||||
for _s2, ipco_end, ipco_type, ipco_payload in iter_file_boxes(f, iprp_payload, iprp_end):
|
||||
if ipco_type != b"ipco":
|
||||
continue
|
||||
for _s3, _prop_end, prop_type, prop_payload in iter_file_boxes(f, ipco_payload, ipco_end):
|
||||
if prop_type == b"irot":
|
||||
f.seek(prop_payload)
|
||||
angle = f.read(1)
|
||||
if angle and angle[0] & 0x03:
|
||||
return True
|
||||
elif prop_type == b"imir":
|
||||
return True
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def scan_c2pa_region(path: str | Path, *, max_total: int = 4 * 1024 * 1024, strict: bool = False) -> bytes:
|
||||
"""Concatenated payloads of top-level ``uuid`` / ``jumb`` boxes in an ISOBMFF
|
||||
file, found by seeking past other boxes (``mdat`` etc.) by size.
|
||||
|
||||
@@ -113,10 +113,9 @@ QWEN_ZIMAGE_OPENAI_STRENGTH = 0.07675
|
||||
# measured corpus margin, not a universal InvisMark threshold.
|
||||
QWEN_ZIMAGE_MICROSOFT_STRENGTH = 0.15
|
||||
|
||||
# Meta Muse Image's retained standalone IPTC AI tag selects this floor through
|
||||
# vendor_for_strength. That shared standard tag is a routing heuristic, not a
|
||||
# vendor-unique signature or a Content Seal pixel detection. Stripped outputs
|
||||
# need an explicit --vendor meta override when their origin is known.
|
||||
# Meta Muse Image's retained standalone IPTC AI tag is not a vendor-unique
|
||||
# signature or a Content Seal pixel detection, so this floor requires an explicit
|
||||
# --vendor meta override when the origin is independently known.
|
||||
# Derivation (oracle meta.ai/identification, 2026-08-26/27, corpus in
|
||||
# data/contentseal/): five independent 2.56 MP generations bracketed at
|
||||
# lighthouse (0.0525, 0.06], fox (0.03, 0.0375], night_city (0.03, 0.0375],
|
||||
@@ -310,15 +309,9 @@ def resolve_strength(
|
||||
def vendor_for_strength(image_path: Path) -> Literal["openai", "google", "microsoft", "meta"] | None:
|
||||
"""Select the strength cohort from non-invalid pixel-watermark provenance.
|
||||
|
||||
OpenAI / Google / Microsoft come from their C2PA issuers. Meta is the
|
||||
fallback cohort: Muse Image carries no C2PA at all, and its only readable
|
||||
companion is the IPTC ``trainedAlgorithmicMedia`` XMP tag -- a standard code
|
||||
other platforms also use. Attributing that tag to Meta is a measured bet,
|
||||
not an identification: the other tag users in this project's model (ByteDance
|
||||
products, X) ship no invisible pixel watermark this profile targets, so the
|
||||
worst misroute spends the Meta floor (0.1) where the resolution curve would
|
||||
have spent a similar amount, and Google/OpenAI files never reach this arm
|
||||
because their C2PA matched first."""
|
||||
OpenAI / Google / Microsoft come from product-specific C2PA evidence. Meta has no
|
||||
automatic route because its companion IPTC ``trainedAlgorithmicMedia`` value is a
|
||||
shared standard, not manufacturer evidence; callers can select it explicitly."""
|
||||
try:
|
||||
from remove_ai_watermarks._internal.c2pa import (
|
||||
c2pa_info_has_invalid_credential,
|
||||
@@ -337,25 +330,4 @@ def vendor_for_strength(image_path: Path) -> Literal["openai", "google", "micros
|
||||
return "openai"
|
||||
if not c2pa_info_has_invalid_credential(info) and c2pa_info_has_invismark(info):
|
||||
return "microsoft"
|
||||
if _standalone_iptc_ai_tag(image_path):
|
||||
return "meta"
|
||||
return None
|
||||
|
||||
|
||||
def _standalone_iptc_ai_tag(image_path: Path) -> bool:
|
||||
"""True when the file carries an AI IPTC marker with no C2PA around it.
|
||||
|
||||
Mirrors identify's ``standalone_iptc`` condition (the tag is only
|
||||
trustworthy as platform evidence when no manifest supersedes it) without
|
||||
importing the heavy identify module: the shared chunk-aware
|
||||
:func:`metadata.scan_head` window -- Muse WebP outputs place their XMP
|
||||
packet in a tail chunk up to hundreds of KB past a plain head read, which
|
||||
is exactly what scan_head's extensions exist to catch.
|
||||
"""
|
||||
try:
|
||||
from remove_ai_watermarks.metadata import IPTC_AI_MARKERS, c2pa_marker_in, scan_head
|
||||
|
||||
scan = scan_head(image_path)
|
||||
except Exception:
|
||||
return False
|
||||
return any(marker in scan for marker in IPTC_AI_MARKERS) and not c2pa_marker_in(scan)
|
||||
@@ -182,6 +182,11 @@ class TextMarkDetection:
|
||||
provenance: bool = False
|
||||
|
||||
|
||||
def best_detection(*detections: TextMarkDetection) -> TextMarkDetection:
|
||||
"""Prefer an accepted variant, then retain the strongest rejected score."""
|
||||
return max(detections, key=lambda detection: (detection.detected, detection.confidence))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TextMarkScan:
|
||||
"""The trust-level-BLIND half of text-mark detection, reusable across both levels.
|
||||
|
||||
@@ -61,8 +61,9 @@ def visible_provenance(source: str | Path) -> frozenset[str]:
|
||||
|
||||
Mapping: a Google/Gemini C2PA issuer -> ``"gemini"``; a ``samsung_genai`` marker ->
|
||||
``"samsung"``; a China-AIGC (TC260) label -> the vendor its ``ContentProducer``
|
||||
names (``KnownMark.tc260_producer_codes``), falling back to ByteDance's two products
|
||||
when the producer is absent or unmapped.
|
||||
names (``KnownMark.tc260_producer_codes``). An absent or unmapped producer confirms
|
||||
no particular product: the TC260 standard identifies neither a manufacturer nor a
|
||||
product.
|
||||
|
||||
The TC260 label itself says only "this is AI", so it used to relax Doubao and
|
||||
Jimeng on EVERY China-AIGC image -- including one carrying a Qwen or Kling mark,
|
||||
@@ -87,24 +88,21 @@ def visible_provenance(source: str | Path) -> frozenset[str]:
|
||||
def _tc260_vendors(path: Path) -> frozenset[str]:
|
||||
"""Vendor keys a TC260 label confirms, from its ``ContentProducer`` identity.
|
||||
|
||||
An absent, unreadable or unmapped producer falls back to the historical pair rather
|
||||
than to nothing: the caller has already established that the AIGC signal fired, so
|
||||
the image IS China-AIGC labeled, and dropping to no relaxation would lose the
|
||||
detections the fallback recovers today. The re-read is deliberately isolated -- a
|
||||
failure here must narrow the answer, never discard the rest of the provenance.
|
||||
An absent, unreadable or unmapped producer confirms no particular product. The
|
||||
caller has established only that the AIGC signal fired; a shared standard cannot
|
||||
safely relax one manufacturer's detector. The re-read is deliberately isolated --
|
||||
a failure here must narrow the answer, never discard the rest of the provenance.
|
||||
"""
|
||||
import contextlib
|
||||
|
||||
from remove_ai_watermarks._internal.constants import TC260_FALLBACK_VENDORS
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
from remove_ai_watermarks.metadata import aigc_label, uscc_of
|
||||
from remove_ai_watermarks.watermark_registry import tc260_producer_vendors
|
||||
from remove_ai_watermarks.watermark_registry import tc260_producer_mark
|
||||
|
||||
producer = (aigc_label(path) or {}).get("ContentProducer", "")
|
||||
if producer and (vendor := tc260_producer_vendors().get(uscc_of(producer))):
|
||||
return frozenset({vendor})
|
||||
return TC260_FALLBACK_VENDORS
|
||||
if producer and (mark := tc260_producer_mark(uscc_of(producer))):
|
||||
return frozenset({mark.key})
|
||||
return frozenset()
|
||||
|
||||
|
||||
def _load_visible_input(source: str | Path | NDArray[Any]) -> _VisibleInput:
|
||||
@@ -175,7 +173,7 @@ def remove_visible(
|
||||
"""Remove every detected known visible AI mark through localize then fill.
|
||||
|
||||
The registry currently covers the Gemini visible watermark; Doubao, Jimeng,
|
||||
Qwen, Kling AI, Yuanbao, Samsung, RunningHub, and Baidu text marks; the
|
||||
Qwen text or symbol mark, Kling AI, Yuanbao, Samsung, RunningHub, and Baidu text marks; the
|
||||
LiblibAI wordmark and compact pill; one Microsoft top-right AI-badge variant;
|
||||
and the Jimeng pill. Returns ``(result_bgr, [labels removed])``.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
@@ -19,15 +19,18 @@ from typing import Any, Literal
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CLASSIFY_EXTRA = "'remove-ai-watermarks[classify]'"
|
||||
CLASSIFY_ONNX_EXTRA = "'remove-ai-watermarks[classify-onnx]'"
|
||||
WEIGHTS_ENV = "RAIW_CLASSIFY_WEIGHTS"
|
||||
WEIGHTS_REPO = "wiltodelta/raiw-photo-classify"
|
||||
WEIGHTS_REVISION = "4c2763766dd1c8c212d64e01e1cc3f166243c47d"
|
||||
WEIGHTS_REVISION = "de668f0d8a53412923478cc21382a56890748310"
|
||||
CLIP_FILE = "clip-l-ft.pt"
|
||||
CLIP_ONNX_FILE = "clip-l-ft-vision-fp32.onnx"
|
||||
PROBE_FILE = "probe-weights-clip-l-ft.npz"
|
||||
DETECTOR_FILE = "detector.pt"
|
||||
PROVIDER_FILE = "provider.pt"
|
||||
OPERATING_POINT_FILE = "operating-point.json"
|
||||
_WEIGHT_FILES = (CLIP_FILE, PROBE_FILE, DETECTOR_FILE, PROVIDER_FILE)
|
||||
_COMMON_WEIGHT_FILES = (PROBE_FILE, DETECTOR_FILE, PROVIDER_FILE)
|
||||
_WEIGHT_FILES = (CLIP_FILE, *_COMMON_WEIGHT_FILES)
|
||||
|
||||
# 2026-09-02 receipt-document gate: linear head on the same CLIP-L-ft
|
||||
# vector, trained on CORD-v2 train (800, CC BY 4.0) plus 200 synthetic
|
||||
@@ -55,6 +58,7 @@ RECEIPT_GATE_THRESHOLD = 2.0432573877459697
|
||||
# drifting the next time a release adds one.
|
||||
WEIGHTS_ALLOW_PATTERNS: tuple[str, ...] = (
|
||||
*_WEIGHT_FILES,
|
||||
CLIP_ONNX_FILE,
|
||||
OPERATING_POINT_FILE,
|
||||
RECEIPT_GATE_STABLE_FILE,
|
||||
RECEIPT_GATE_FILE,
|
||||
@@ -87,6 +91,7 @@ DetectorLevel = Literal["definitely", "possibly", "likely_human"]
|
||||
PixelLabel = Literal["ai", "human", "unknown"]
|
||||
PixelDomain = Literal["photo"]
|
||||
PixelProvider = Literal["openai", "google", "muse-image", "bytedance"]
|
||||
ClassifierBackend = Literal["torch", "onnx"]
|
||||
PUBLIC_PROVIDER: dict[str, PixelProvider | None] = {
|
||||
"openai": "openai",
|
||||
"google": "google",
|
||||
@@ -255,7 +260,12 @@ def classify_from_scores(
|
||||
return PixelClassification(label=label, domain="photo", detector=level, provider=provider)
|
||||
|
||||
|
||||
def classify_pixels(path: Path, *, device: str | None = None) -> PixelClassification:
|
||||
def classify_pixels(
|
||||
path: Path,
|
||||
*,
|
||||
device: str | None = None,
|
||||
backend: ClassifierBackend = "torch",
|
||||
) -> PixelClassification:
|
||||
"""Run Model 1 then, on DEFINITELY, the receipt gate and Model 2.
|
||||
|
||||
Never called by ``identify``. A DEFINITELY verdict that the receipt gate
|
||||
@@ -263,7 +273,14 @@ def classify_pixels(path: Path, *, device: str | None = None) -> PixelClassifica
|
||||
"""
|
||||
if not is_available():
|
||||
raise RuntimeError(f"pixel classification requires the classify extra. Install: pip install {CLASSIFY_EXTRA}")
|
||||
runtime = _get_runtime(device)
|
||||
if backend == "onnx":
|
||||
from remove_ai_watermarks.optional_deps import module_available
|
||||
|
||||
if not module_available("onnxruntime"):
|
||||
raise RuntimeError(
|
||||
f"the ONNX photo-classifier backend requires ONNX Runtime. Install: pip install {CLASSIFY_ONNX_EXTRA}"
|
||||
)
|
||||
runtime = _get_runtime(device, backend)
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as image:
|
||||
@@ -304,6 +321,7 @@ def classify_pixels(path: Path, *, device: str | None = None) -> PixelClassifica
|
||||
@dataclass
|
||||
class _Runtime:
|
||||
device: Any
|
||||
backend: ClassifierBackend
|
||||
clip_model: Any
|
||||
processor: Any
|
||||
detector: Any
|
||||
@@ -331,7 +349,26 @@ def _resolve_device(device: str | None) -> Any:
|
||||
raise ValueError(f"unsupported classify device {device!r}")
|
||||
|
||||
|
||||
def _weights_dir() -> Path:
|
||||
def _resolve_backend(backend: ClassifierBackend, device: Any) -> ClassifierBackend:
|
||||
if backend == "onnx" and device.type != "cpu":
|
||||
raise ValueError("the ONNX photo-classifier backend supports CPU only")
|
||||
if backend not in ("torch", "onnx"):
|
||||
raise ValueError(f"unsupported classify backend {backend!r}")
|
||||
return backend
|
||||
|
||||
|
||||
def _backend_allow_patterns(backend: ClassifierBackend) -> tuple[str, ...]:
|
||||
model_file = CLIP_ONNX_FILE if backend == "onnx" else CLIP_FILE
|
||||
return (
|
||||
model_file,
|
||||
*_COMMON_WEIGHT_FILES,
|
||||
OPERATING_POINT_FILE,
|
||||
RECEIPT_GATE_STABLE_FILE,
|
||||
RECEIPT_GATE_FILE,
|
||||
)
|
||||
|
||||
|
||||
def _weights_dir(backend: ClassifierBackend = "torch") -> Path:
|
||||
override = os.environ.get(WEIGHTS_ENV)
|
||||
if override:
|
||||
path = Path(override)
|
||||
@@ -345,7 +382,7 @@ def _weights_dir() -> Path:
|
||||
snapshot_download(
|
||||
repo_id=WEIGHTS_REPO,
|
||||
revision=WEIGHTS_REVISION,
|
||||
allow_patterns=list(WEIGHTS_ALLOW_PATTERNS),
|
||||
allow_patterns=list(_backend_allow_patterns(backend)),
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -373,8 +410,10 @@ def _weight_path(folder: Path, name: str) -> Path:
|
||||
return found
|
||||
|
||||
|
||||
def _require_files(folder: Path) -> None:
|
||||
missing = [name for name in _WEIGHT_FILES if _find_weight(folder, name) is None]
|
||||
def _require_files(folder: Path, *, backend: ClassifierBackend = "torch") -> None:
|
||||
model_file = CLIP_ONNX_FILE if backend == "onnx" else CLIP_FILE
|
||||
required = (model_file, *_COMMON_WEIGHT_FILES)
|
||||
missing = [name for name in required if _find_weight(folder, name) is None]
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
"pixel classification weights are missing "
|
||||
@@ -417,14 +456,14 @@ def _build_provider_head() -> Any:
|
||||
)
|
||||
|
||||
|
||||
def _load_runtime(device: Any) -> _Runtime:
|
||||
def _load_runtime(device: Any, backend: ClassifierBackend = "torch") -> _Runtime:
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from remove_ai_watermarks._internal.clip_l_ft import load_headed_clip, load_processor
|
||||
from remove_ai_watermarks._internal.clip_l_ft import load_headed_clip, load_onnx_vision, load_processor
|
||||
|
||||
folder = _weights_dir()
|
||||
_require_files(folder)
|
||||
folder = _weights_dir(backend)
|
||||
_require_files(folder, backend=backend)
|
||||
probe = np.load(_weight_path(folder, PROBE_FILE))
|
||||
detector = _build_detector()
|
||||
detector.load_state_dict(torch.load(_weight_path(folder, DETECTOR_FILE), map_location="cpu", weights_only=True))
|
||||
@@ -441,9 +480,15 @@ def _load_runtime(device: Any) -> _Runtime:
|
||||
ridge_threshold = float(probe["thr_oi_1pct"])
|
||||
mlp_threshold, provider_margin = _operating_point(folder)
|
||||
log.info("loaded photo-classify freeze from %s", folder)
|
||||
clip_model = (
|
||||
load_onnx_vision(_weight_path(folder, CLIP_ONNX_FILE))
|
||||
if backend == "onnx"
|
||||
else load_headed_clip(_weight_path(folder, CLIP_FILE), device)
|
||||
)
|
||||
return _Runtime(
|
||||
device=device,
|
||||
clip_model=load_headed_clip(_weight_path(folder, CLIP_FILE), device),
|
||||
backend=backend,
|
||||
clip_model=clip_model,
|
||||
processor=load_processor(),
|
||||
detector=detector,
|
||||
provider_heads=heads,
|
||||
@@ -457,18 +502,21 @@ def _load_runtime(device: Any) -> _Runtime:
|
||||
)
|
||||
|
||||
|
||||
def _get_runtime(device: str | None) -> _Runtime:
|
||||
def _get_runtime(device: str | None, backend: ClassifierBackend = "torch") -> _Runtime:
|
||||
global _runtime
|
||||
resolved = _resolve_device(device)
|
||||
resolved_backend = _resolve_backend(backend, resolved)
|
||||
with _runtime_lock:
|
||||
if _runtime is None or _runtime.device != resolved:
|
||||
_runtime = _load_runtime(resolved)
|
||||
if _runtime is None or _runtime.device != resolved or _runtime.backend != resolved_backend:
|
||||
_runtime = _load_runtime(resolved, resolved_backend)
|
||||
return _runtime
|
||||
|
||||
|
||||
def _embed(runtime: _Runtime, image: Any) -> Any:
|
||||
from remove_ai_watermarks._internal.clip_l_ft import embed_image
|
||||
from remove_ai_watermarks._internal.clip_l_ft import embed_image, embed_image_onnx
|
||||
|
||||
if runtime.backend == "onnx":
|
||||
return embed_image_onnx(runtime.clip_model, runtime.processor, image)
|
||||
return embed_image(runtime.clip_model, runtime.processor, image, runtime.device)
|
||||
|
||||
|
||||
|
||||
@@ -136,11 +136,6 @@ _C2PA_INVALID_CAVEAT = (
|
||||
"are retained only as removal hints, not as verified provenance."
|
||||
)
|
||||
_IPTC_ONLY_CAVEAT = "The IPTC 'Made with AI' tag flags AI provenance but does not identify the specific platform."
|
||||
_CONTENT_SEAL_CAVEAT = (
|
||||
"Meta Muse Image outputs carry the invisible Content Seal pixel watermark, which has no "
|
||||
"local decoder; `invisible` removes it (auto when this tag is present, or `--vendor meta` "
|
||||
"on stripped files) and meta.ai/identification verifies it."
|
||||
)
|
||||
_INVISIBLE_WM_CAVEAT = (
|
||||
"The open invisible watermark is fragile: it does not survive JPEG re-encoding "
|
||||
"or resizing, so it confirms origin only on a pristine (un-re-encoded) file."
|
||||
@@ -822,6 +817,16 @@ _AI_VENDOR_TOKENS: tuple[tuple[str, str], ...] = (
|
||||
("dreamina", "ByteDance"),
|
||||
("volcengine", "ByteDance"),
|
||||
("volcano engine", "ByteDance"),
|
||||
("qwen", "Alibaba"),
|
||||
("alibaba", "Alibaba"),
|
||||
("kling", "Kuaishou"),
|
||||
("kuaishou", "Kuaishou"),
|
||||
("yuanbao", "Tencent"),
|
||||
("tencent", "Tencent"),
|
||||
("runninghub", "RunningHub"),
|
||||
("baidu", "Baidu"),
|
||||
("ernie", "Baidu"),
|
||||
("liblib", "LiblibAI"),
|
||||
("canva", "Canva"),
|
||||
("elevenlabs", "ElevenLabs"),
|
||||
("eleven labs", "ElevenLabs"),
|
||||
@@ -860,16 +865,16 @@ def _vendor_of(text: str | None) -> str | None:
|
||||
_C2PA_MANIFEST_SOURCE = "c2pa_manifest"
|
||||
_CLASH_SOURCE: dict[str, str] = {"c2pa": _C2PA_MANIFEST_SOURCE, "synthid": _C2PA_MANIFEST_SOURCE}
|
||||
|
||||
# The generic China TC260 AIGC vendor label -- a COUNTRY-LEVEL regulatory "this is AI"
|
||||
# stamp any Chinese generator applies to its own output, naming no specific vendor.
|
||||
_GENERIC_AIGC_VENDOR = "China AIGC (TC260)"
|
||||
# Vendors that apply the TC260 label to their OWN output. When one is co-attributed with
|
||||
# the generic AIGC label, the label is that vendor's own stamp (not an independent
|
||||
# competing origin), so the clash check attributes the AIGC label to it -- else a legit
|
||||
# ByteDance/Doubao image (C2PA "ByteDance" + its own TC260 label) would false-clash once
|
||||
# ByteDance normalizes via _vendor_of. Chinese generators only (Canva/BFL/ElevenLabs,
|
||||
# also added to _vendor_of, are NOT TC260 appliers).
|
||||
_TC260_VENDORS: frozenset[str] = frozenset({"ByteDance"})
|
||||
|
||||
def _tc260_manufacturer_of(producer: str) -> str | None:
|
||||
"""Resolve a TC260 producer to a normalized manufacturer, if registered."""
|
||||
if not producer:
|
||||
return None
|
||||
from remove_ai_watermarks.metadata import uscc_of
|
||||
from remove_ai_watermarks.watermark_registry import tc260_producer_mark
|
||||
|
||||
mark = tc260_producer_mark(uscc_of(producer))
|
||||
return _vendor_of(mark.manufacturer if mark else None)
|
||||
|
||||
|
||||
def _integrity_clashes(
|
||||
@@ -895,18 +900,6 @@ def _integrity_clashes(
|
||||
# families clash only when they belong to different provenance sources (see
|
||||
# _CLASH_SOURCE) AND name different vendors -- so multiple vendors named within
|
||||
# one C2PA manifest (C2PA issuer + SynthID provenance) do not flag.
|
||||
# The generic TC260 AIGC label is a Chinese regulatory "this is AI" stamp. When a
|
||||
# Chinese TC260-applying vendor (ByteDance) is ALSO attributed, the label is that
|
||||
# vendor's own stamp on its own output, so attribute it to that vendor -- a legit
|
||||
# Doubao image carries BOTH a ByteDance C2PA manifest and its own TC260 label and
|
||||
# must not clash. Against a NON-TC260 vendor (OpenAI, Google, ...) the label stays
|
||||
# generic and still clashes as a laundering tell (a foreign-vendor image carrying a
|
||||
# Chinese TC260 label names two different origins).
|
||||
if ai_vendors.get("aigc") == _GENERIC_AIGC_VENDOR:
|
||||
own = next((v for f, v in ai_vendors.items() if f != "aigc" and v in _TC260_VENDORS), None)
|
||||
if own:
|
||||
ai_vendors = {**ai_vendors, "aigc": own} # copy co-located with the relabel
|
||||
|
||||
source = {fam: _CLASH_SOURCE.get(fam, fam) for fam in ai_vendors}
|
||||
independent_conflict = any(
|
||||
source[a] != source[b] and ai_vendors[a] != ai_vendors[b] for a, b in itertools.combinations(ai_vendors, 2)
|
||||
@@ -960,15 +953,13 @@ def _visible_sparkle(image_path: Path, *, image: NDArray[Any] | None = None) ->
|
||||
|
||||
# Visible text marks (registry keys) -> human-readable platform, mirroring the
|
||||
# Gemini-sparkle phrasing. These are the stripped-metadata visual fallback for
|
||||
# the China-served ByteDance generators (normally also caught by the TC260 AIGC
|
||||
# metadata label); the per-engine detection thresholds live in the registry.
|
||||
# several China-served generators (normally also caught by the TC260 AIGC metadata
|
||||
# label); the per-engine detection thresholds live in the registry.
|
||||
# Text mark -> the platform sentence this report prints when that mark is the strongest
|
||||
# evidence, DERIVED from the registry rows so registering a mark is one edit. It was a
|
||||
# hand-maintained copy, and that class of copy is how LiblibAI ended up registered but
|
||||
# missing from the pill veto. Insertion order is the registry's, which is what fixes the
|
||||
# scan order below. The Gemini sparkle and the capture-less pill carry no platform of
|
||||
# their own (`KnownMark.platform is None`) and are excluded here: the sparkle has its
|
||||
# own higher-confidence `_visible_sparkle` path.
|
||||
# evidence, derived from the registry rows so registering a mark is one edit. Insertion
|
||||
# order is the registry's, which fixes the scan order below. The Gemini sparkle and the
|
||||
# capture-less pill carry no platform of their own (`KnownMark.platform is None`) and
|
||||
# are excluded here: the sparkle has its own higher-confidence `_visible_sparkle` path.
|
||||
#
|
||||
# Safe at module scope: `watermark_registry` is already imported above for
|
||||
# GEMINI_SPARKLE_TRUST_CONF, and it is deliberately cv2-free at import time.
|
||||
@@ -1378,38 +1369,12 @@ def _identify_from_evidence(
|
||||
if standalone_iptc:
|
||||
signals.append(Signal("iptc", "digitalSourceType (Made with AI)", "high"))
|
||||
watermarks.append("IPTC digitalSourceType (Made with AI)")
|
||||
# Muse Image stamps every output with the invisible Content Seal, and this
|
||||
# tag is the only provenance such a file carries - the same measured bet
|
||||
# the strength router makes (vendor_for_strength -> "meta"). Emit the seal
|
||||
# as its own stable signal, the way InvisMark is additive over
|
||||
# soft_binding, so clients select pixel removal from the signal list
|
||||
# instead of parsing caveats. It is an attribution, not a decode: no
|
||||
# public Content Seal decoder exists, hence "medium".
|
||||
signals.append(
|
||||
Signal(
|
||||
"content_seal",
|
||||
"Meta Muse Content Seal pixel watermark (attributed by the standalone AI digital-source tag)",
|
||||
"medium",
|
||||
)
|
||||
)
|
||||
watermarks.append("Invisible Content Seal watermark (Meta Muse attribution)")
|
||||
caveats.append(_IPTC_ONLY_CAVEAT)
|
||||
caveats.append(_CONTENT_SEAL_CAVEAT)
|
||||
if platform is None:
|
||||
# Apple Photos Clean Up (Apple Intelligence object removal) marks
|
||||
# the edit with photoshop:Credit / IPTC "Apple Photos Clean Up"
|
||||
# next to compositeWithTrainedAlgorithmicMedia. It was detected but
|
||||
# previously never attributed.
|
||||
if b"Apple Photos Clean Up" in head:
|
||||
platform = "Apple Photos (Clean Up AI edit)"
|
||||
else:
|
||||
# The platform line follows the same measured bet the seal signal
|
||||
# and the strength router make: Muse Image is the tag writer whose
|
||||
# outputs this profile targets, so a hedged Muse attribution is
|
||||
# more useful than "platform not specified" while the panel below
|
||||
# already prices the Content Seal removal. The hedge stays in the
|
||||
# wording - it names the attribution basis, not a detection.
|
||||
platform = "Meta Muse Image (attributed by the standalone AI digital-source tag)"
|
||||
# Apple Photos Clean Up carries an explicit product credit beside the shared
|
||||
# IPTC value. Without such product-specific evidence the standard names no
|
||||
# platform and must not imply a vendor watermark.
|
||||
if platform is None and b"Apple Photos Clean Up" in head:
|
||||
platform = "Apple Photos (Clean Up AI edit)"
|
||||
|
||||
# ── IPTC 2025.1 AI-disclosure fields (Iptc4xmpExt:AISystemUsed etc.) ─
|
||||
iptc_ai = any(m in head for m in IPTC_AI_FIELD_MARKERS)
|
||||
@@ -1437,8 +1402,9 @@ def _identify_from_evidence(
|
||||
signals.append(Signal("aigc", f"TC260 AIGC label{f' (producer {producer})' if producer else ''}", "high"))
|
||||
watermarks.append("China AIGC label (TC260 standard)")
|
||||
if platform is None:
|
||||
platform = "China AIGC-labeled generator (TC260; e.g. Doubao)"
|
||||
ai_vendor_claims["aigc"] = _GENERIC_AIGC_VENDOR
|
||||
platform = "China AIGC-labeled content (TC260 standard)"
|
||||
if manufacturer := _tc260_manufacturer_of(producer):
|
||||
ai_vendor_claims["aigc"] = manufacturer
|
||||
|
||||
# ── Local diffusion parameters (Stable Diffusion / ComfyUI) ──────
|
||||
local_keys = sorted(k for k in meta if k.lower() in _LOCAL_GEN_KEYS)
|
||||
|
||||
@@ -29,6 +29,7 @@ from remove_ai_watermarks._internal.constants import PNG_SIGNATURE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
from PIL.Image import Image as PILImage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -72,7 +73,10 @@ def _pil_read(path: str | Path, flags: int) -> NDArray[Any] | None:
|
||||
"""Decode via Pillow (HEIC/AVIF and any other Pillow-readable container) into the
|
||||
cv2 layout ``flags`` implies: grayscale, 3-channel BGR, or BGRA when the source has
|
||||
alpha and ``IMREAD_UNCHANGED`` was requested. Returns None if Pillow (with the
|
||||
optional HEIF plugin) still cannot open it. This fallback does not apply EXIF orientation."""
|
||||
optional HEIF plugin) still cannot open it. This fallback asks for no EXIF
|
||||
auto-rotation, but a container reader can apply it regardless: Pillow's TIFF
|
||||
reader always turns the raster upright. That decode state is reconciled where
|
||||
it matters, in :func:`_read_display_tags`."""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
@@ -152,6 +156,8 @@ def _register_heif() -> None:
|
||||
# AI-provenance tags the metadata strip exists to drop.
|
||||
|
||||
_ORIENT_EXIF_TAG = 0x0112 # EXIF ImageIFD.Orientation
|
||||
_TIFF_IMAGE_WIDTH = 256 # TIFF ImageWidth, the stored raster's columns
|
||||
_TIFF_IMAGE_LENGTH = 257 # TIFF ImageLength, the stored raster's rows
|
||||
# Orientations that transpose width and height (the 90-degree rotations). Only these
|
||||
# can be detected by comparing the raster's shape with the source's stored size;
|
||||
# the mirror/180 values (2, 3, 4) keep today's tagless behaviour because no cheap
|
||||
@@ -176,6 +182,23 @@ def _orientation_exif(orient: int | None, *, prefixed: bool) -> bytes | None:
|
||||
return dumped[6:] if dumped.startswith(b"Exif\x00\x00") else dumped
|
||||
|
||||
|
||||
def _stored_size(im: PILImage) -> tuple[int, int]:
|
||||
"""The size the container STORES, which is not always ``Image.size``.
|
||||
|
||||
Pillow's TIFF reader applies the EXIF orientation itself: it hands back a turned
|
||||
raster and reports the upright size. Comparing that against a turned raster says
|
||||
"still in stored orientation" for pixels that are already upright, and the tag
|
||||
rides along into a second rotation. The TIFF IFD keeps the real stored geometry in
|
||||
ImageWidth/ImageLength, and no other Pillow plugin exposes ``tag_v2``.
|
||||
"""
|
||||
tags = getattr(im, "tag_v2", None)
|
||||
if tags is not None:
|
||||
width, height = tags.get(_TIFF_IMAGE_WIDTH), tags.get(_TIFF_IMAGE_LENGTH)
|
||||
if isinstance(width, int) and isinstance(height, int):
|
||||
return width, height
|
||||
return im.size
|
||||
|
||||
|
||||
def _read_display_tags(
|
||||
source: Path, raster_shape: tuple[int, int], *, orientation_applied: bool | None = None
|
||||
) -> tuple[bytes | None, int | None]:
|
||||
@@ -195,6 +218,26 @@ def _read_display_tags(
|
||||
tag and keeps today's behaviour rather than risk a double rotation. ICC travels
|
||||
in every case: it describes colour, not geometry. Failures read as "no tags":
|
||||
the restore is an enhancement, never a reason to fail a write.
|
||||
|
||||
TIFF is the exception on the explicit state: every reader in this module's
|
||||
stack applies the TIFF orientation tag on decode (cv2's libtiff path under
|
||||
``IMREAD_UNCHANGED`` too, and Pillow's TIFF plugin, identically), so a caller's
|
||||
blanket ``False`` -- which encodes cv2's JPEG/PNG/WebP contract -- is not true
|
||||
for TIFF. For TIFF sources the parameter is ignored and the geometry check
|
||||
decides against the IFD's stored size: a raster that no longer has the stored
|
||||
dimensions was turned by the decode and must not be tagged again (issue #106).
|
||||
|
||||
HEIF is the other exception: pillow-heif's opener blanks the EXIF orientation
|
||||
on open (the real value waits in ``info["original_orientation"]``), and
|
||||
libheif turns the raster exactly when the container declares an effective
|
||||
``irot``/``imir`` transform, so the geometry check has nothing to compare --
|
||||
``Image.size`` matches the raster in both cases. For HEIF the parameter is
|
||||
likewise ignored and the container decides (issue #105). AVIF keeps the
|
||||
generic path: this plugin build neither blanks its tag nor applies the
|
||||
transform there.
|
||||
|
||||
The stored size cannot be read straight off ``Image``: Pillow's TIFF reader
|
||||
reports the upright one, so it comes through :func:`_stored_size`.
|
||||
"""
|
||||
_register_heif()
|
||||
try:
|
||||
@@ -205,11 +248,29 @@ def _read_display_tags(
|
||||
with Image.open(source) as im:
|
||||
icc = im.info.get("icc_profile")
|
||||
orient = im.getexif().get(_ORIENT_EXIF_TAG)
|
||||
stored_w, stored_h = im.size
|
||||
fmt = im.format
|
||||
# pillow-heif's opener rewrites the EXIF orientation to 1 and stashes
|
||||
# the real value here, so `orient` is blanked for every HEIF.
|
||||
heif_original = im.info.get("original_orientation")
|
||||
stored_w, stored_h = _stored_size(im)
|
||||
except Exception:
|
||||
return None, None
|
||||
if not (isinstance(icc, bytes) and icc):
|
||||
icc = None
|
||||
if fmt == "TIFF":
|
||||
orientation_applied = None
|
||||
if fmt == "HEIF":
|
||||
# The second container whose decode state a caller cannot claim: libheif
|
||||
# turns the raster on decode exactly when the file declares an effective
|
||||
# irot/imir transform, and pillow-heif has blanked the EXIF tag either
|
||||
# way, so the container decides and `original_orientation` supplies the
|
||||
# value (issue #105). AVIF keeps the generic path: this plugin build
|
||||
# neither blanks its tag nor applies the transform there.
|
||||
from remove_ai_watermarks._internal import isobmff
|
||||
|
||||
orientation_applied = isobmff.heif_transform_applied(source)
|
||||
if orient in (None, 1):
|
||||
orient = heif_original
|
||||
if orientation_applied is True:
|
||||
orient = None
|
||||
elif orientation_applied is False:
|
||||
|
||||
@@ -160,14 +160,9 @@ class KlingEngine(TextMarkEngine):
|
||||
super().__init__(_CONFIG)
|
||||
self._latin = TextMarkEngine(_LATIN_CONFIG)
|
||||
|
||||
@staticmethod
|
||||
def _best(*detections: TextMarkDetection) -> TextMarkDetection:
|
||||
"""Prefer an accepted variant, then retain the strongest rejected score."""
|
||||
return max(detections, key=lambda detection: (detection.detected, detection.confidence))
|
||||
|
||||
def detect(self, image: NDArray[Any], *, provenance: bool = False) -> TextMarkDetection:
|
||||
"""Return the strongest CJK or Latin Kling wordmark verdict."""
|
||||
return self._best(
|
||||
return _text_mark_engine.best_detection(
|
||||
super().detect(image, provenance=provenance),
|
||||
self._latin.detect(image, provenance=provenance),
|
||||
)
|
||||
@@ -176,7 +171,9 @@ class KlingEngine(TextMarkEngine):
|
||||
"""Scan each immutable variant once and combine strict/relaxed verdicts."""
|
||||
cjk_strict, cjk_relaxed = super().detect_both(image)
|
||||
latin_strict, latin_relaxed = self._latin.detect_both(image)
|
||||
return self._best(cjk_strict, latin_strict), self._best(cjk_relaxed, latin_relaxed)
|
||||
return _text_mark_engine.best_detection(cjk_strict, latin_strict), _text_mark_engine.best_detection(
|
||||
cjk_relaxed, latin_relaxed
|
||||
)
|
||||
|
||||
def footprint_mask(
|
||||
self,
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
"""Qwen (Alibaba) visible watermark detector/localizer.
|
||||
|
||||
Qwen stamps its generations with a visible "千问AI生成" text strip in the
|
||||
bottom-right corner -- the explicit AIGC label mandated by China's GB 45438-2025
|
||||
Qwen stamps its generations with either a visible "千问AI生成" text strip or a
|
||||
small three-lobe symbol in the bottom-right corner. The text is the explicit AIGC
|
||||
label mandated by China's GB 45438-2025
|
||||
(the same 6-glyph house style as Doubao's "豆包AI生成": a 2-glyph vendor prefix
|
||||
plus the shared `AI生成` tail), preceded by the vendor's tri-lobe logo (not part
|
||||
of the detection silhouette -- logos vary between releases, the CJK run is what
|
||||
discriminates).
|
||||
plus the shared `AI生成` tail), preceded by the vendor's tri-lobe logo. Qwen
|
||||
Create's global consumer surface uses the symbol alone; it has its own square
|
||||
template and search geometry.
|
||||
|
||||
Detection matches the bundled glyph silhouette against the corner; removal is the
|
||||
shared **localize -> fill** (the glyph-bbox :meth:`footprint_mask` feeds
|
||||
@@ -49,10 +50,14 @@ whose producer USCC 91440101MA9Y9T4H7A names the entity, 2026-07-21; harness
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from remove_ai_watermarks import _text_mark_engine
|
||||
from remove_ai_watermarks._text_mark_engine import TextMarkConfig, TextMarkEngine
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from remove_ai_watermarks import _text_mark_engine, image_io
|
||||
from remove_ai_watermarks._text_mark_engine import TextMarkConfig, TextMarkDetection, TextMarkEngine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
@@ -94,6 +99,28 @@ _ALPHA_HEIGHT_FRAC = 0.0416
|
||||
# its nearest rung).
|
||||
_LADDER = (0.78, 1.27)
|
||||
|
||||
# Qwen Create symbol-only mark, measured on the cleared 2048-square provider
|
||||
# fixture. The mark body is ~2.7% of the short side, inset ~3% from both edges.
|
||||
# The strict 0.65 gate leaves 0.13 NCC margin over the tracked-control maximum
|
||||
# (0.516 across 132 decoded images other than the positive on 2026-09-10).
|
||||
_SYMBOL_ASSET = "qwen_symbol_alpha.png"
|
||||
_SYMBOL_SEARCH_FRAC = 0.11
|
||||
_SYMBOL_MARGIN_FRAC = 0.012
|
||||
_SYMBOL_SIZE_FRAC = 0.03125
|
||||
_SYMBOL_LADDER = (0.75, 0.875, 1.0, 1.125, 1.25)
|
||||
_SYMBOL_NCC_THRESHOLD = 0.65
|
||||
# NCC alone is not portable: issue 108 reported 0.81 on
|
||||
# data/fixtures/provenance/chatgpt-1.png (sand/denim, no glyph) while this
|
||||
# OpenCV build scores 0.437 on the same file. True Qwen Create matches keep
|
||||
# binarized IoU >= 0.75 down to 512 px; that fixture sits at 0.55. Require
|
||||
# both the NCC gate and this shape overlap so a high-NCC blob cannot fire.
|
||||
_SYMBOL_SHAPE_IOU_THRESHOLD = 0.70
|
||||
_SYMBOL_TEMPLATE_MASK_FRAC = 0.25
|
||||
_SYMBOL_RESPONSE_MASK_FRAC = 0.35
|
||||
_SYMBOL_MIN_SHORT_SIDE = 256
|
||||
_SYMBOL_MASK_ALPHA = 4 / 255
|
||||
_SYMBOL_MASK_DILATE_FRAC = 0.06
|
||||
|
||||
_CONFIG = TextMarkConfig(
|
||||
name="Qwen",
|
||||
asset_name="qwen_alpha.png",
|
||||
@@ -131,8 +158,133 @@ def _glyph_silhouette() -> NDArray[Any] | None:
|
||||
return _text_mark_engine.glyph_silhouette(_CONFIG.asset_name)
|
||||
|
||||
|
||||
class QwenSymbolDetection(TextMarkDetection):
|
||||
"""Explicit marker for the symbol variant's distinct footprint policy."""
|
||||
|
||||
|
||||
class QwenEngine(TextMarkEngine):
|
||||
"""Detect/localize the visible Qwen "千问AI生成" watermark (locate -> mask; mask feeds the fill)."""
|
||||
"""Detect and localize Qwen text and symbol visible marks."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(_CONFIG)
|
||||
template = _text_mark_engine.load_alpha_template(_SYMBOL_ASSET)
|
||||
if template is None:
|
||||
raise RuntimeError(f"failed to decode embedded asset: {_SYMBOL_ASSET}")
|
||||
self._symbol_template = template
|
||||
self._symbol_template_scales: dict[int, NDArray[Any]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _symbol_shape_iou(response_crop: NDArray[Any], template: NDArray[Any]) -> float:
|
||||
"""Overlap of the bright residual with the three-lobe silhouette."""
|
||||
if response_crop.size == 0 or template.size == 0:
|
||||
return 0.0
|
||||
template_peak = float(template.max())
|
||||
response_peak = float(response_crop.max())
|
||||
if template_peak <= 0.0 or response_peak <= 0.0:
|
||||
return 0.0
|
||||
template_mask = template > (template_peak * _SYMBOL_TEMPLATE_MASK_FRAC)
|
||||
response_mask = response_crop > (response_peak * _SYMBOL_RESPONSE_MASK_FRAC)
|
||||
union = int(np.logical_or(template_mask, response_mask).sum())
|
||||
if union == 0:
|
||||
return 0.0
|
||||
intersection = int(np.logical_and(template_mask, response_mask).sum())
|
||||
return intersection / union
|
||||
|
||||
def _scaled_symbol_template(self, side: int) -> NDArray[Any]:
|
||||
"""Return a cached square symbol template for a measured ladder size."""
|
||||
template = self._symbol_template_scales.get(side)
|
||||
if template is None:
|
||||
if len(self._symbol_template_scales) >= len(_SYMBOL_LADDER) * 4:
|
||||
self._symbol_template_scales.clear()
|
||||
template = cv2.resize(self._symbol_template, (side, side), interpolation=cv2.INTER_AREA)
|
||||
self._symbol_template_scales[side] = template
|
||||
return template
|
||||
|
||||
def _symbol_scan(self, image: NDArray[Any] | None) -> QwenSymbolDetection:
|
||||
"""Locate the strongest symbol-shaped candidate in the lower-right search box."""
|
||||
det = QwenSymbolDetection()
|
||||
if image is None or image.size == 0:
|
||||
return det
|
||||
height, width = image.shape[:2]
|
||||
base = min(height, width)
|
||||
if base < _SYMBOL_MIN_SHORT_SIDE:
|
||||
return det
|
||||
search_side = max(80, int(base * _SYMBOL_SEARCH_FRAC))
|
||||
margin = int(base * _SYMBOL_MARGIN_FRAC)
|
||||
origin_x = max(0, width - margin - search_side)
|
||||
origin_y = max(0, height - margin - search_side)
|
||||
roi = image_io.to_bgr(image[origin_y : origin_y + search_side, origin_x : origin_x + search_side])
|
||||
luma: NDArray[Any] = roi.mean(axis=2, dtype=np.float32)
|
||||
background: NDArray[Any] = cv2.GaussianBlur(luma, (0, 0), sigmaX=max(4.0, search_side * 0.20))
|
||||
response: NDArray[Any] = np.clip(luma - background, 0.0, None)
|
||||
|
||||
best_score = 0.0
|
||||
best_region = (0, 0, 0, 0)
|
||||
best_iou = 0.0
|
||||
for rung in _SYMBOL_LADDER:
|
||||
side = max(12, int(base * _SYMBOL_SIZE_FRAC * rung))
|
||||
if side >= search_side:
|
||||
continue
|
||||
template = self._scaled_symbol_template(side)
|
||||
scores = cv2.matchTemplate(response, template, cv2.TM_CCOEFF_NORMED)
|
||||
_minimum, score, _min_location, location = cv2.minMaxLoc(scores)
|
||||
ncc = float(score)
|
||||
if ncc <= best_score:
|
||||
continue
|
||||
crop = response[location[1] : location[1] + side, location[0] : location[0] + side]
|
||||
best_score = ncc
|
||||
best_region = (origin_x + location[0], origin_y + location[1], side, side)
|
||||
best_iou = self._symbol_shape_iou(crop, template)
|
||||
det.confidence = best_score
|
||||
det.detected = best_score >= _SYMBOL_NCC_THRESHOLD and best_iou >= _SYMBOL_SHAPE_IOU_THRESHOLD
|
||||
det.region = best_region
|
||||
return det
|
||||
|
||||
def detect(self, image: NDArray[Any], *, provenance: bool = False) -> TextMarkDetection:
|
||||
"""Return the stronger text or symbol variant."""
|
||||
text = super().detect(image, provenance=provenance)
|
||||
symbol = self._symbol_scan(image)
|
||||
symbol.provenance = provenance
|
||||
return _text_mark_engine.best_detection(text, symbol)
|
||||
|
||||
def detect_both(self, image: NDArray[Any] | None) -> tuple[TextMarkDetection, TextMarkDetection]:
|
||||
"""Return both trust verdicts from one text scan and one symbol scan."""
|
||||
strict_text, relaxed_text = super().detect_both(image)
|
||||
strict_symbol = self._symbol_scan(image)
|
||||
relaxed_symbol = replace(strict_symbol, provenance=True)
|
||||
return _text_mark_engine.best_detection(strict_text, strict_symbol), _text_mark_engine.best_detection(
|
||||
relaxed_text, relaxed_symbol
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_symbol_detection(detection: TextMarkDetection | None) -> bool:
|
||||
return isinstance(detection, QwenSymbolDetection) and detection.detected
|
||||
|
||||
def footprint_mask(
|
||||
self,
|
||||
image: NDArray[Any] | None,
|
||||
*,
|
||||
force: bool = False,
|
||||
dilate: int | None = None,
|
||||
detection: TextMarkDetection | None = None,
|
||||
) -> NDArray[Any] | None:
|
||||
"""Mask the detected symbol box or delegate to the text-mark footprint."""
|
||||
if image is None or image.size == 0:
|
||||
return None
|
||||
resolved = detection if detection is not None else self.detect(image)
|
||||
if self._is_symbol_detection(resolved):
|
||||
x, y, width, height = resolved.region
|
||||
alpha = cv2.resize(self._symbol_template, (width, height), interpolation=cv2.INTER_LINEAR)
|
||||
radius = dilate if dilate is not None else max(2, int(max(width, height) * _SYMBOL_MASK_DILATE_FRAC))
|
||||
local = np.pad((alpha > _SYMBOL_MASK_ALPHA).astype(np.uint8) * 255, radius)
|
||||
if radius > 0:
|
||||
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * radius + 1, 2 * radius + 1))
|
||||
local = cv2.dilate(local, kernel)
|
||||
mask = np.zeros(image.shape[:2], dtype=np.uint8)
|
||||
target_x, target_y = x - radius, y - radius
|
||||
x1, y1 = max(0, target_x), max(0, target_y)
|
||||
x2, y2 = min(image.shape[1], target_x + local.shape[1]), min(image.shape[0], target_y + local.shape[0])
|
||||
local_x, local_y = x1 - target_x, y1 - target_y
|
||||
mask[y1:y2, x1:x2] = local[local_y : local_y + y2 - y1, local_x : local_x + x2 - x1]
|
||||
return mask
|
||||
return super().footprint_mask(image, force=force, dilate=dilate, detection=resolved)
|
||||
@@ -21,7 +21,7 @@ Entries:
|
||||
- ``gemini`` -- Google Gemini / Nano Banana visible watermark (sparkle), bottom-right.
|
||||
- ``doubao`` -- ByteDance Doubao "豆包AI生成" text strip, bottom-right.
|
||||
- ``jimeng`` -- ByteDance Jimeng / Dreamina "★ 即梦AI" wordmark, bottom-right.
|
||||
- ``qwen`` -- Alibaba Cloud Qwen "千问AI生成" text strip, bottom-right.
|
||||
- ``qwen`` -- Alibaba Cloud Qwen "千问AI生成" text or three-lobe symbol, bottom-right.
|
||||
- ``kling`` -- Kuaishou Kling AI "可灵AI 3.0" / "KlingAI 3.0" strip, bottom-right.
|
||||
- ``yuanbao`` -- Tencent Yuanbao "元宝 / AI生成" two-line mark, bottom-right.
|
||||
- ``samsung`` -- Samsung Galaxy AI "Contenuti generati dall'AI" strip, bottom-left.
|
||||
@@ -265,9 +265,12 @@ class KnownMark:
|
||||
# REQUIRED, deliberately: a defaulted empty product would let two rows that forgot
|
||||
# the field corroborate each other and silently bypass the trust gate.
|
||||
product: str
|
||||
# The company responsible for the product. This is deliberately separate from the
|
||||
# labeling standard: Alibaba, ByteDance, Kuaishou, Tencent, and other manufacturers
|
||||
# all use TC260, but that does not make their marks one product family.
|
||||
manufacturer: str
|
||||
# Which provenance regime this mark's vendor labels under, or None. "tc260" means
|
||||
# the vendor stamps the China AIGC label, so a confident detection of it names a
|
||||
# DIFFERENT TC260 product than the Jimeng pill -- see _keep_pill.
|
||||
# the vendor stamps the China AIGC label; it does not identify the manufacturer.
|
||||
label_regime: str | None
|
||||
# The sentence `identify` reports when THIS mark is the strongest evidence. None for
|
||||
# a mark that never names a platform on its own: `gemini` has its own higher-
|
||||
@@ -287,8 +290,7 @@ class KnownMark:
|
||||
# Social Credit Codes as normalized by ``metadata.uscc_of``, plus the bare product
|
||||
# names a few generators write instead. Here rather than in a separate table
|
||||
# because a newly registered TC260 mark whose codes were forgotten fails SILENTLY:
|
||||
# it falls back to relaxing ByteDance's pair on an image carrying the new mark,
|
||||
# the exact false positive the producer code exists to prevent.
|
||||
# its own detector remains strict even on an image carrying that producer's label.
|
||||
tc260_producer_codes: tuple[str, ...] = ()
|
||||
# Optional single-pass dual verdict for the arbiter's perception stage (see
|
||||
# `detect_both`). None = fall back to two `_detect` calls.
|
||||
@@ -598,6 +600,7 @@ def _text_mark(
|
||||
location: str,
|
||||
*,
|
||||
platform: str,
|
||||
manufacturer: str,
|
||||
product: str | None = None,
|
||||
label_regime: str | None = "tc260",
|
||||
provenance_signals: tuple[str, ...] = ("aigc",),
|
||||
@@ -607,9 +610,10 @@ def _text_mark(
|
||||
"""Build a text-mark registry row from its shared detector and mask adapters.
|
||||
|
||||
``product`` defaults to the key (one mark, one product); pass it only when two
|
||||
marks share a product. ``label_regime`` and ``provenance_signals`` default to the
|
||||
China-AIGC label because every text mark registered so far except Samsung and
|
||||
Microsoft uses it.
|
||||
marks share a product. ``manufacturer`` is separate because unrelated companies
|
||||
use the same TC260 standard. ``label_regime`` and ``provenance_signals`` default
|
||||
to the China-AIGC label because every text mark registered so far except Samsung
|
||||
and Microsoft uses it.
|
||||
"""
|
||||
return KnownMark(
|
||||
key,
|
||||
@@ -617,6 +621,7 @@ def _text_mark(
|
||||
location,
|
||||
True,
|
||||
product or key,
|
||||
manufacturer,
|
||||
label_regime,
|
||||
platform,
|
||||
_engine_mark_detect(key, label, location),
|
||||
@@ -660,14 +665,14 @@ def _pill_features(image: NDArray[Any]) -> dict[str, float]:
|
||||
|
||||
|
||||
_REGISTRY: tuple[KnownMark, ...] = (
|
||||
# Gemini is a Google C2PA/SynthID product, not a China-AIGC labeler: label_regime
|
||||
# is None so it can never act as a TC260 sibling in _keep_pill.
|
||||
# Gemini is a Google C2PA/SynthID product, not a China-AIGC labeler.
|
||||
KnownMark(
|
||||
"gemini",
|
||||
"Google Gemini visible watermark (sparkle)",
|
||||
"bottom-right",
|
||||
True,
|
||||
"gemini",
|
||||
"google",
|
||||
None,
|
||||
# No platform sentence: the sparkle has its own higher-confidence
|
||||
# `_visible_sparkle` path in identify, which names the platform itself.
|
||||
@@ -682,6 +687,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"Doubao 豆包AI生成 text",
|
||||
"bottom-right",
|
||||
platform="ByteDance Doubao (visible 豆包AI生成 mark detected)",
|
||||
manufacturer="bytedance",
|
||||
tc260_producer_codes=("91110102MACQD9K640", "doubao"),
|
||||
),
|
||||
_text_mark(
|
||||
@@ -689,13 +695,15 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"Jimeng 即梦AI wordmark",
|
||||
"bottom-right",
|
||||
platform="ByteDance Jimeng / Dreamina (visible 即梦AI mark detected)",
|
||||
manufacturer="bytedance",
|
||||
tc260_producer_codes=("9144030008867405X2",),
|
||||
),
|
||||
_text_mark(
|
||||
"qwen",
|
||||
"Qwen 千问AI生成 text",
|
||||
"Qwen 千问AI生成 text or three-lobe symbol",
|
||||
"bottom-right",
|
||||
platform="Alibaba Cloud Qwen (visible 千问AI生成 mark detected)",
|
||||
platform="Alibaba Cloud Qwen (visible text or symbol mark detected)",
|
||||
manufacturer="alibaba",
|
||||
tc260_producer_codes=("91440101MA9Y9T4H7A",),
|
||||
),
|
||||
_text_mark(
|
||||
@@ -703,6 +711,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"Kling AI 可灵AI / KlingAI 3.0 text",
|
||||
"bottom-right",
|
||||
platform="Kuaishou Kling AI (visible 可灵AI / KlingAI 3.0 mark detected)",
|
||||
manufacturer="kuaishou",
|
||||
tc260_producer_codes=("91110108335469089C",),
|
||||
),
|
||||
_text_mark(
|
||||
@@ -710,6 +719,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"Tencent Yuanbao 元宝 / AI生成 mark",
|
||||
"bottom-right",
|
||||
platform="Tencent Yuanbao (visible 元宝 / AI生成 mark detected)",
|
||||
manufacturer="tencent",
|
||||
tc260_producer_codes=("91440300708461136T",),
|
||||
),
|
||||
# Samsung Galaxy AI is a device editing marker (samsung_genai), not a TC260 label.
|
||||
@@ -717,6 +727,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"samsung",
|
||||
"Samsung Galaxy AI text",
|
||||
"bottom-left",
|
||||
manufacturer="samsung",
|
||||
label_regime=None,
|
||||
provenance_signals=("samsung_genai",),
|
||||
platform="Samsung Galaxy AI (visible 'Contenuti generati dall'AI' mark detected)",
|
||||
@@ -726,6 +737,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"RunningHub AI生成 text",
|
||||
"top-left",
|
||||
platform="RunningHub (visible RunningHub AI生成 mark detected)",
|
||||
manufacturer="runninghub",
|
||||
tc260_producer_codes=("91340100MAEB4N8H76", "RunningHub"),
|
||||
),
|
||||
_text_mark(
|
||||
@@ -733,6 +745,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"Baidu 百度 AI生成 text",
|
||||
"bottom-right",
|
||||
platform="Baidu (visible 百度 AI生成 mark detected)",
|
||||
manufacturer="baidu",
|
||||
tc260_producer_codes=("91110000802100433B",),
|
||||
),
|
||||
_text_mark(
|
||||
@@ -740,6 +753,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"LiblibAI wordmark",
|
||||
"bottom-center",
|
||||
platform="LiblibAI (visible LiblibAI mark detected)",
|
||||
manufacturer="liblib",
|
||||
tc260_producer_codes=("91110105MACJ6K1C8A",),
|
||||
),
|
||||
KnownMark(
|
||||
@@ -748,6 +762,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"top-left",
|
||||
True,
|
||||
"liblib",
|
||||
"liblib",
|
||||
"tc260",
|
||||
# The compact pill is generic on its own; the bottom wordmark or LiblibAI
|
||||
# metadata is what makes the relaxed verdict actionable.
|
||||
@@ -765,6 +780,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"microsoft",
|
||||
"Microsoft top-right AI badge",
|
||||
"top-right",
|
||||
manufacturer="microsoft",
|
||||
label_regime=None,
|
||||
provenance_signals=(),
|
||||
platform="Microsoft (visible top-right AI badge detected)",
|
||||
@@ -777,6 +793,7 @@ _REGISTRY: tuple[KnownMark, ...] = (
|
||||
"top-left",
|
||||
True,
|
||||
"jimeng",
|
||||
"bytedance",
|
||||
"tc260",
|
||||
# The capture-less pill is too weak a detector to attribute a platform on its
|
||||
# own; the Jimeng wordmark is what names ByteDance.
|
||||
@@ -868,28 +885,15 @@ def resolve_trust(
|
||||
return "confirmed" if confirmed else "strict"
|
||||
|
||||
|
||||
def tc260_producer_vendors() -> dict[str, str]:
|
||||
"""TC260 ``ContentProducer`` identity -> the mark key whose vendor signs with it.
|
||||
|
||||
Derived from the registry rows, so registering a TC260 mark and its producer codes
|
||||
is one edit. A mark registered without codes falls through to
|
||||
:data:`TC260_FALLBACK_VENDORS`, which relaxes ByteDance's pair -- a silent wrong
|
||||
answer on an image carrying the new mark, which is why the codes belong on the row
|
||||
next to ``label_regime`` rather than in a table someone must remember to update.
|
||||
"""
|
||||
return {code: mark.key for mark in _REGISTRY for code in mark.tc260_producer_codes}
|
||||
def tc260_producer_mark(code: str) -> KnownMark | None:
|
||||
"""Return the registry row for one TC260 ``ContentProducer`` identity."""
|
||||
return next((mark for mark in _REGISTRY if code in mark.tc260_producer_codes), None)
|
||||
|
||||
|
||||
def _pill_suppressors() -> set[str]:
|
||||
"""Marks whose detection vetoes the capture-less pill: same label regime as the
|
||||
pill, different product. Derived so a newly registered TC260 mark cannot be
|
||||
forgotten here -- which is exactly how LiblibAI ended up missing."""
|
||||
"""Other products from the pill's manufacturer whose detection vetoes it."""
|
||||
pill = get_mark("jimeng_pill")
|
||||
return {
|
||||
m.key
|
||||
for m in _REGISTRY
|
||||
if m.label_regime is not None and m.label_regime == pill.label_regime and m.product != pill.product
|
||||
}
|
||||
return {m.key for m in _REGISTRY if m.manufacturer == pill.manufacturer and m.product != pill.product}
|
||||
|
||||
|
||||
def _keep_pill(keys: set[str], *, provenance: frozenset[str], footprint_flat: bool) -> bool:
|
||||
@@ -905,18 +909,14 @@ def _keep_pill(keys: set[str], *, provenance: frozenset[str], footprint_flat: bo
|
||||
top-left footprint is flat enough for an invisible fill (``footprint_flat``),
|
||||
so real flat-scene pills (and harmless flat false fires) are cleaned while the
|
||||
damaging textured false fires are left untouched.
|
||||
A Doubao image is TC260 too but is not Jimeng-basic, so the pill never rides on a
|
||||
Doubao detection; every other TC260 product's mark likewise names a different
|
||||
product and suppresses the pill.
|
||||
A Doubao image is TC260 too but is not Jimeng-basic, so its mark suppresses the
|
||||
pill. Marks from other TC260 manufacturers are unrelated and do not participate in
|
||||
this ByteDance-family decision.
|
||||
No confirmation at all -> never remove (blocks false fires on non-Jimeng content).
|
||||
|
||||
The suppressor set is DERIVED from the registry (same label regime, different
|
||||
product), not hand-listed. The hand-written list had drifted: LiblibAI was
|
||||
registered alongside RunningHub and Baidu but never added to it, so a confident
|
||||
LiblibAI detection did not veto the pill the way its two siblings did. Marks
|
||||
outside the TC260 regime (Gemini, Samsung) are deliberately NOT suppressors --
|
||||
neither can put ``"jimeng"`` into ``provenance``, so neither can enable the arm
|
||||
they would be vetoing."""
|
||||
The suppressor set is derived from the manufacturer and product fields rather than
|
||||
from the shared TC260 standard. Marks from Google, Samsung, or another TC260
|
||||
manufacturer cannot enable or veto this ByteDance-specific arm."""
|
||||
if _pill_suppressors() & keys:
|
||||
return False
|
||||
if "jimeng" in keys:
|
||||
|
||||
@@ -86,6 +86,19 @@ def test_skill_stays_under_the_500_line_budget() -> None:
|
||||
assert lines <= 500, f"SKILL.md is {lines} lines; move detail into references/"
|
||||
|
||||
|
||||
def test_skill_keeps_concrete_input_output_examples() -> None:
|
||||
text = SKILL.read_text(encoding="utf-8")
|
||||
examples = text.split("## Examples\n", maxsplit=1)[1].split("\n## ", maxsplit=1)[0]
|
||||
cases = re.split(r"(?m)^### ", examples)[1:]
|
||||
|
||||
assert len(cases) == 3
|
||||
for case in cases:
|
||||
assert re.search(r'(?m)^Input: "[^"]+"$', case)
|
||||
assert re.search(r"(?m)^Action:", case)
|
||||
assert re.search(r"(?m)^Output(?: [^:\n]+)?:$", case)
|
||||
assert re.search(r"(?m)^> ", case)
|
||||
|
||||
|
||||
def test_probe_min_cli_version_never_exceeds_the_package_version() -> None:
|
||||
import tomllib
|
||||
|
||||
@@ -104,6 +117,35 @@ def test_probe_min_cli_version_never_exceeds_the_package_version() -> None:
|
||||
sys.modules.pop("probe", None)
|
||||
|
||||
|
||||
def test_probe_floor_covers_every_since_annotation() -> None:
|
||||
"""MIN_CLI_VERSION must be at least the newest named "(since CLI X.Y.Z)" value.
|
||||
|
||||
A reference that names a value the previous release lacked has to raise the
|
||||
floor in the same change; this reads the annotations back out of the skill
|
||||
tree so a stale floor fails the suite instead of an agent's CLI.
|
||||
"""
|
||||
|
||||
sys.path.insert(0, str(ROOT / "skills" / "remove-ai-watermarks" / "scripts"))
|
||||
try:
|
||||
import probe
|
||||
|
||||
annotated = []
|
||||
for path in (ROOT / "skills" / "remove-ai-watermarks").rglob("*"):
|
||||
if path.suffix not in {".md", ".py"} or not path.is_file():
|
||||
continue
|
||||
for match in re.finditer(r"\(since CLI (\d+)\.(\d+)\.(\d+)\)", path.read_text(encoding="utf-8")):
|
||||
annotated.append(tuple(int(part) for part in match.groups()))
|
||||
assert annotated, "no (since CLI X.Y.Z) annotations found; the floor guard lost its source"
|
||||
highest = max(annotated)
|
||||
assert highest <= probe.MIN_CLI_VERSION, (
|
||||
f"probe.py MIN_CLI_VERSION {probe.MIN_CLI_VERSION} is below {highest}, the newest "
|
||||
"(since CLI X.Y.Z) annotation in the skill; a build that old rejects a named value"
|
||||
)
|
||||
finally:
|
||||
sys.path.remove(str(ROOT / "skills" / "remove-ai-watermarks" / "scripts"))
|
||||
sys.modules.pop("probe", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"relative",
|
||||
[
|
||||
|
||||
+10
-16
@@ -136,9 +136,9 @@ class TestVisibleProvenance:
|
||||
assert "doubao" in prov
|
||||
assert "jimeng" not in prov
|
||||
|
||||
def test_unmapped_tc260_producer_falls_back_to_the_bytedance_pair(self, monkeypatch, tmp_path):
|
||||
"""An unrecognized producer must not lose the relaxation entirely: the label is
|
||||
still evidence that some China-AIGC vendor made the image."""
|
||||
@pytest.mark.parametrize("producer", ["0011999999999999999999999", ""])
|
||||
def test_tc260_without_a_mapped_producer_does_not_confirm_a_product(self, monkeypatch, tmp_path, producer):
|
||||
"""A shared labeling standard cannot identify a product by itself."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from remove_ai_watermarks import identify, metadata
|
||||
@@ -148,8 +148,8 @@ class TestVisibleProvenance:
|
||||
"identify",
|
||||
lambda *a, **k: SimpleNamespace(platform=None, signals=[SimpleNamespace(name="aigc")]),
|
||||
)
|
||||
monkeypatch.setattr(metadata, "aigc_label", lambda _p: {"ContentProducer": "0011999999999999999999999"})
|
||||
assert raiw.visible_provenance(tmp_path / "x.png") == frozenset({"doubao", "jimeng"})
|
||||
monkeypatch.setattr(metadata, "aigc_label", lambda _p: {"ContentProducer": producer})
|
||||
assert raiw.visible_provenance(tmp_path / "x.png") == frozenset()
|
||||
|
||||
def test_known_tc260_producer_names_a_single_vendor(self, monkeypatch, tmp_path):
|
||||
from types import SimpleNamespace
|
||||
@@ -165,16 +165,10 @@ class TestVisibleProvenance:
|
||||
monkeypatch.setattr(metadata, "aigc_label", lambda _p: {"ContentProducer": "001191440101MA9Y9T4H7A00001"})
|
||||
assert raiw.visible_provenance(tmp_path / "x.png") == frozenset({"qwen"})
|
||||
|
||||
def test_every_mapped_producer_names_a_registered_mark(self):
|
||||
"""The table drives the arbiter's provenance set, so a typo'd key would relax
|
||||
nothing and fail silently. Checked here, not at import: importing the registry
|
||||
from _internal.constants would drag it into every metadata-only path."""
|
||||
from remove_ai_watermarks._internal.constants import TC260_FALLBACK_VENDORS
|
||||
from remove_ai_watermarks.watermark_registry import known_marks, mark_keys, tc260_producer_vendors
|
||||
def test_every_primary_tc260_mark_declares_a_producer_code(self):
|
||||
"""Only companion pills may inherit a primary mark's producer identity."""
|
||||
from remove_ai_watermarks.watermark_registry import known_marks
|
||||
|
||||
keys = set(mark_keys())
|
||||
assert set(tc260_producer_vendors().values()) <= keys
|
||||
assert keys >= TC260_FALLBACK_VENDORS
|
||||
# Primary TC260 marks name their producer. Companion pills inherit the primary
|
||||
# mark's product-level provenance rather than duplicating an external identity.
|
||||
unmapped = {m.key for m in known_marks() if m.label_regime == "tc260" and not m.tc260_producer_codes}
|
||||
@@ -189,7 +183,7 @@ class TestVisibleProvenance:
|
||||
assert raiw.visible_provenance(tmp_path / "missing.png") == frozenset()
|
||||
|
||||
def test_uses_report_signals_for_falsy_metadata_values(self, monkeypatch, tmp_path):
|
||||
"""An empty TC260 object and Samsung genAIType=0 are still present signals.
|
||||
"""An empty TC260 object names no product; Samsung genAIType=0 is present.
|
||||
|
||||
The report has already normalized those values, so the public API must not
|
||||
re-read the file and accidentally discard them by truthiness.
|
||||
@@ -204,7 +198,7 @@ class TestVisibleProvenance:
|
||||
)
|
||||
monkeypatch.setattr(identify, "identify", lambda *args, **kwargs: report)
|
||||
|
||||
assert raiw.visible_provenance(tmp_path / "synthetic.png") == frozenset({"doubao", "jimeng", "samsung"})
|
||||
assert raiw.visible_provenance(tmp_path / "synthetic.png") == frozenset({"samsung"})
|
||||
|
||||
|
||||
class TestRemoveVisibleOutputPath:
|
||||
|
||||
@@ -133,12 +133,3 @@ class TestDetectAndMask:
|
||||
assert xs.max() >= bx + bw - 1
|
||||
assert ys.min() <= by
|
||||
assert ys.max() >= by + bh - 1
|
||||
|
||||
|
||||
class TestPillInteraction:
|
||||
def test_confident_baidu_detection_suppresses_the_jimeng_pill(self):
|
||||
# A Baidu image is TC260 too but is not Jimeng-basic: like Doubao/Qwen/
|
||||
# Kling, a confident Baidu detection must veto the pill (``_keep_pill``).
|
||||
from remove_ai_watermarks.watermark_registry import _keep_pill
|
||||
|
||||
assert not _keep_pill({"baidu"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
+65
-9
@@ -13,6 +13,7 @@ from click.testing import CliRunner
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks.classify import (
|
||||
CLIP_ONNX_FILE,
|
||||
MLP_THRESHOLD,
|
||||
PROVIDER_MARGIN,
|
||||
RECEIPT_GATE_FILE,
|
||||
@@ -54,16 +55,14 @@ def _scores(
|
||||
|
||||
def test_hub_snapshot_is_the_freeze_revision() -> None:
|
||||
assert WEIGHTS_REPO == "wiltodelta/raiw-photo-classify"
|
||||
assert WEIGHTS_REVISION == "4c2763766dd1c8c212d64e01e1cc3f166243c47d"
|
||||
assert WEIGHTS_REVISION == "de668f0d8a53412923478cc21382a56890748310"
|
||||
|
||||
|
||||
def test_runtime_requests_the_exported_allow_patterns(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The exported pre-cache list and the runtime request must stay one list.
|
||||
def test_runtime_requests_only_the_selected_backend_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Online downloads stay narrow while the exported pre-cache covers both.
|
||||
|
||||
An offline deploy pre-caches WEIGHTS_ALLOW_PATTERNS from the Hub; the
|
||||
runtime then resolves the same snapshot offline. If the runtime's request
|
||||
grows a file the export misses, every offline deployment breaks on a
|
||||
four-fifths-present cache, so pin the seam in both directions.
|
||||
runtime requests a subset of that snapshot for its selected backend.
|
||||
"""
|
||||
from remove_ai_watermarks.classify import _WEIGHT_FILES, _weights_dir
|
||||
|
||||
@@ -81,8 +80,65 @@ def test_runtime_requests_the_exported_allow_patterns(monkeypatch: pytest.Monkey
|
||||
assert _weights_dir() == Path("/cached/snapshot")
|
||||
assert captured[0]["repo_id"] == WEIGHTS_REPO
|
||||
assert captured[0]["revision"] == WEIGHTS_REVISION
|
||||
assert captured[0]["allow_patterns"] == list(WEIGHTS_ALLOW_PATTERNS)
|
||||
torch_patterns = captured[0]["allow_patterns"]
|
||||
assert "clip-l-ft.pt" in torch_patterns
|
||||
assert CLIP_ONNX_FILE not in torch_patterns
|
||||
assert set(_WEIGHT_FILES) <= set(WEIGHTS_ALLOW_PATTERNS)
|
||||
assert CLIP_ONNX_FILE in WEIGHTS_ALLOW_PATTERNS
|
||||
|
||||
captured.clear()
|
||||
assert _weights_dir("onnx") == Path("/cached/snapshot")
|
||||
onnx_patterns = captured[0]["allow_patterns"]
|
||||
assert CLIP_ONNX_FILE in onnx_patterns
|
||||
assert "clip-l-ft.pt" not in onnx_patterns
|
||||
assert set(torch_patterns) | set(onnx_patterns) == set(WEIGHTS_ALLOW_PATTERNS)
|
||||
|
||||
|
||||
def test_backend_requires_its_own_model_file(tmp_path: Path) -> None:
|
||||
from remove_ai_watermarks.classify import DETECTOR_FILE, PROBE_FILE, PROVIDER_FILE, _require_files
|
||||
|
||||
for name in (CLIP_ONNX_FILE, PROBE_FILE, DETECTOR_FILE, PROVIDER_FILE):
|
||||
(tmp_path / name).touch()
|
||||
|
||||
_require_files(tmp_path, backend="onnx")
|
||||
with pytest.raises(RuntimeError, match=r"clip-l-ft\.pt"):
|
||||
_require_files(tmp_path, backend="torch")
|
||||
|
||||
|
||||
def test_onnx_backend_is_cpu_only() -> None:
|
||||
import torch
|
||||
|
||||
from remove_ai_watermarks.classify import _resolve_backend
|
||||
|
||||
assert _resolve_backend("onnx", torch.device("cpu")) == "onnx"
|
||||
with pytest.raises(ValueError, match=r"ONNX.*CPU"):
|
||||
_resolve_backend("onnx", torch.device("cuda"))
|
||||
|
||||
|
||||
def test_onnx_backend_names_its_install_extra(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from remove_ai_watermarks import classify as classify_mod
|
||||
from remove_ai_watermarks import optional_deps
|
||||
|
||||
monkeypatch.setattr(classify_mod, "is_available", lambda: True)
|
||||
monkeypatch.setattr(optional_deps, "module_available", lambda *_names: False)
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"remove-ai-watermarks\[classify-onnx\]"):
|
||||
classify_mod.classify_pixels(Path("unused.png"), backend="onnx")
|
||||
|
||||
|
||||
def test_embed_dispatches_to_the_selected_backend(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from remove_ai_watermarks import classify as classify_mod
|
||||
from remove_ai_watermarks._internal import clip_l_ft
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(clip_l_ft, "embed_image", lambda *_args: calls.append("torch") or "torch-vector")
|
||||
monkeypatch.setattr(clip_l_ft, "embed_image_onnx", lambda *_args: calls.append("onnx") or "onnx-vector")
|
||||
runtime = SimpleNamespace(backend="onnx", clip_model=object(), processor=object(), device=object())
|
||||
|
||||
assert classify_mod._embed(runtime, object()) == "onnx-vector"
|
||||
runtime.backend = "torch"
|
||||
assert classify_mod._embed(runtime, object()) == "torch-vector"
|
||||
assert calls == ["onnx", "torch"]
|
||||
|
||||
|
||||
def test_shipped_operating_point_matches_the_runtime_defaults() -> None:
|
||||
@@ -332,7 +388,7 @@ def _patch_definitely_runtime(monkeypatch: pytest.MonkeyPatch, *, forensic_calls
|
||||
weights_dir=None,
|
||||
)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify.is_available", lambda: True)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._get_runtime", lambda device: runtime)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._get_runtime", lambda device, backend="torch": runtime)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._embed", lambda *args, **kwargs: np.full(768, 100.0))
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._mlp_score", lambda *args, **kwargs: 10.0)
|
||||
monkeypatch.setattr(
|
||||
@@ -402,7 +458,7 @@ def test_124d_is_not_extracted_unless_definitely(monkeypatch: pytest.MonkeyPatch
|
||||
weights_dir=None,
|
||||
)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify.is_available", lambda: True)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._get_runtime", lambda device: runtime)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._get_runtime", lambda device, backend="torch": runtime)
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._embed", lambda *args, **kwargs: np.zeros(768))
|
||||
monkeypatch.setattr("remove_ai_watermarks.classify._mlp_score", lambda *args, **kwargs: 0.0)
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""CLIP-L checkpoint loading without redundant parameter initialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_load_headed_clip_skips_random_initialization_and_materializes_checkpoint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
torch = pytest.importorskip("torch")
|
||||
transformers = pytest.importorskip("transformers")
|
||||
|
||||
from remove_ai_watermarks._internal import clip_l_ft
|
||||
|
||||
config = transformers.CLIPConfig(
|
||||
projection_dim=2,
|
||||
text_config={
|
||||
"vocab_size": 8,
|
||||
"bos_token_id": 0,
|
||||
"eos_token_id": 1,
|
||||
"pad_token_id": 1,
|
||||
"hidden_size": 4,
|
||||
"intermediate_size": 8,
|
||||
"num_hidden_layers": 1,
|
||||
"num_attention_heads": 1,
|
||||
"max_position_embeddings": 8,
|
||||
},
|
||||
vision_config={
|
||||
"hidden_size": 4,
|
||||
"intermediate_size": 8,
|
||||
"num_hidden_layers": 1,
|
||||
"num_attention_heads": 1,
|
||||
"image_size": 4,
|
||||
"patch_size": 2,
|
||||
"num_channels": 3,
|
||||
},
|
||||
)
|
||||
expected = clip_l_ft.HeadedCLIP(transformers.CLIPModel(config))
|
||||
with torch.no_grad():
|
||||
for index, parameter in enumerate(expected.parameters(), start=1):
|
||||
parameter.fill_(float(index))
|
||||
checkpoint = tmp_path / "tiny-clip.pt"
|
||||
torch.save(expected.state_dict(), checkpoint)
|
||||
|
||||
def reject_random_initialization(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise AssertionError("checkpoint loading must not initialize discarded random weights")
|
||||
|
||||
monkeypatch.setattr(transformers.CLIPConfig, "from_pretrained", lambda _backbone: config)
|
||||
monkeypatch.setattr(torch.nn.init, "kaiming_uniform_", reject_random_initialization)
|
||||
|
||||
loaded = clip_l_ft.load_headed_clip(checkpoint, torch.device("cpu"))
|
||||
|
||||
assert not any(value.is_meta for value in (*loaded.parameters(), *loaded.buffers()))
|
||||
expected_state = expected.state_dict()
|
||||
loaded_state = loaded.state_dict()
|
||||
for name, expected_value in expected_state.items():
|
||||
assert torch.equal(loaded_state[name], expected_value)
|
||||
for (name, buffer), (expected_name, expected_buffer) in zip(
|
||||
loaded.named_buffers(), expected.named_buffers(), strict=True
|
||||
):
|
||||
assert name == expected_name
|
||||
assert torch.equal(buffer, expected_buffer)
|
||||
|
||||
|
||||
def test_load_onnx_vision_uses_the_portable_cpu_provider(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
from remove_ai_watermarks._internal import clip_l_ft
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class FakeOptions:
|
||||
graph_optimization_level: Any = None
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, path: str, *, sess_options: FakeOptions, providers: list[str]) -> None:
|
||||
captured.update(path=path, optimization=sess_options.graph_optimization_level, providers=providers)
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"onnxruntime",
|
||||
SimpleNamespace(
|
||||
GraphOptimizationLevel=SimpleNamespace(ORT_DISABLE_ALL="disabled"),
|
||||
InferenceSession=FakeSession,
|
||||
SessionOptions=FakeOptions,
|
||||
),
|
||||
)
|
||||
model = tmp_path / "vision.onnx"
|
||||
|
||||
assert isinstance(clip_l_ft.load_onnx_vision(model), FakeSession)
|
||||
assert captured == {
|
||||
"path": str(model),
|
||||
"optimization": "disabled",
|
||||
"providers": ["CPUExecutionProvider"],
|
||||
}
|
||||
|
||||
|
||||
def test_embed_image_onnx_uses_the_freeze_preprocessor() -> None:
|
||||
from remove_ai_watermarks._internal import clip_l_ft
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class FakeProcessor:
|
||||
def __call__(self, *, images: list[Image.Image], return_tensors: str) -> dict[str, np.ndarray]:
|
||||
captured["image_size"] = images[0].size
|
||||
captured["return_tensors"] = return_tensors
|
||||
return {"pixel_values": np.ones((1, 3, 224, 224), dtype=np.float64)}
|
||||
|
||||
class FakeSession:
|
||||
def run(self, outputs: list[str], inputs: dict[str, np.ndarray]) -> list[np.ndarray]:
|
||||
captured["outputs"] = outputs
|
||||
captured["input_dtype"] = inputs["pixel_values"].dtype
|
||||
return [np.array([[3.0, 4.0]], dtype=np.float32)]
|
||||
|
||||
vector = clip_l_ft.embed_image_onnx(FakeSession(), FakeProcessor(), Image.new("RGB", (40, 20)))
|
||||
|
||||
assert captured == {
|
||||
"image_size": (224, 224),
|
||||
"return_tensors": "np",
|
||||
"outputs": ["embedding"],
|
||||
"input_dtype": np.dtype(np.float32),
|
||||
}
|
||||
assert vector.dtype == np.float64
|
||||
assert vector.tolist() == [3.0, 4.0]
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Vision-only photo-classifier export helpers, without the real checkpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _exporter() -> Any:
|
||||
path = Path(__file__).resolve().parents[1] / "scripts" / "export_photo_classify_onnx.py"
|
||||
spec = importlib.util.spec_from_file_location("export_photo_classify_onnx", path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_vision_embedding_drops_the_text_tower_and_normalizes() -> None:
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
class FakeVision(torch.nn.Module):
|
||||
def forward(self, *, pixel_values: Any) -> Any:
|
||||
return SimpleNamespace(pooler_output=pixel_values)
|
||||
|
||||
clip = SimpleNamespace(
|
||||
vision_model=FakeVision(),
|
||||
visual_projection=torch.nn.Identity(),
|
||||
text_model=object(),
|
||||
)
|
||||
model = _exporter().VisionEmbedding(SimpleNamespace(clip=clip))
|
||||
|
||||
output = model(torch.tensor([[3.0, 4.0]]))
|
||||
|
||||
assert set(dict(model.named_children())) == {"vision_model", "visual_projection"}
|
||||
assert torch.equal(output, torch.tensor([[0.6, 0.8]]))
|
||||
+26
-58
@@ -30,6 +30,7 @@ from remove_ai_watermarks.identify import (
|
||||
_attribute_platform,
|
||||
_integrity_clashes,
|
||||
_issuers_in,
|
||||
_tc260_manufacturer_of,
|
||||
_vendor_of,
|
||||
evidence_from_metadata_record,
|
||||
extract_provenance_evidence,
|
||||
@@ -731,6 +732,8 @@ class TestIdentifyRealSamples:
|
||||
assert r.is_ai_generated is True
|
||||
assert r.platform == "Apple Photos (Clean Up AI edit)"
|
||||
assert r.ai_source_kind == "enhanced"
|
||||
assert "content_seal" not in [signal.name for signal in r.signals]
|
||||
assert not any("Meta" in watermark for watermark in r.watermarks)
|
||||
|
||||
def test_standalone_iptc_composite_synthetic_is_enhanced(self, tmp_path: Path):
|
||||
p = tmp_path / "composite.jpg"
|
||||
@@ -743,15 +746,8 @@ class TestIdentifyRealSamples:
|
||||
assert r.is_ai_generated is True
|
||||
assert r.ai_source_kind == "enhanced"
|
||||
|
||||
def test_standalone_ai_tag_attributes_the_content_seal(self, tmp_path: Path):
|
||||
"""A standalone AI digital-source tag emits the seal as its own signal.
|
||||
|
||||
Muse Image outputs carry no C2PA; this tag is their only provenance, and
|
||||
Muse stamps every output with the invisible Content Seal. The signal is
|
||||
the strength router's Meta bet as evidence - an attribution, not a decode
|
||||
(no public decoder exists), so its confidence is medium and the caveat
|
||||
still points at the oracle.
|
||||
"""
|
||||
def test_standalone_ai_tag_does_not_claim_a_vendor_watermark(self, tmp_path: Path):
|
||||
"""The shared IPTC standard proves neither Meta nor Content Seal."""
|
||||
p = tmp_path / "muse-tag.jpg"
|
||||
p.write_bytes(
|
||||
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="trainedAlgorithmicMedia"></x:xmpmeta>\xff\xd9'
|
||||
@@ -761,38 +757,9 @@ class TestIdentifyRealSamples:
|
||||
|
||||
names = [s.name for s in r.signals]
|
||||
assert "iptc" in names
|
||||
assert "content_seal" in names
|
||||
seal = next(s for s in r.signals if s.name == "content_seal")
|
||||
assert seal.confidence == "medium"
|
||||
assert "Invisible Content Seal watermark (Meta Muse attribution)" in r.watermarks
|
||||
assert any("meta.ai/identification" in c for c in r.caveats)
|
||||
|
||||
def test_seal_platform_attribution_follows_the_signal(self, tmp_path: Path):
|
||||
"""The Likely-source line follows the same bet the seal signal makes.
|
||||
|
||||
Apple keeps its own attribution; every other standalone-tag file gets the
|
||||
hedged Muse attribution instead of "platform not specified", so the panel
|
||||
that prices the Content Seal and the source line agree.
|
||||
"""
|
||||
muse = tmp_path / "muse.jpg"
|
||||
muse.write_bytes(
|
||||
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="trainedAlgorithmicMedia"></x:xmpmeta>\xff\xd9'
|
||||
)
|
||||
apple = tmp_path / "apple.jpg"
|
||||
apple.write_bytes(
|
||||
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="compositeWithTrainedAlgorithmicMedia" '
|
||||
b'photoshop:Credit="Apple Photos Clean Up"></x:xmpmeta>\xff\xd9'
|
||||
)
|
||||
|
||||
assert identify(muse, check_visible=False, check_invisible=False).platform == (
|
||||
"Meta Muse Image (attributed by the standalone AI digital-source tag)"
|
||||
)
|
||||
assert identify(apple, check_visible=False, check_invisible=False).platform == "Apple Photos (Clean Up AI edit)"
|
||||
|
||||
def test_c2pa_backed_file_gets_no_content_seal_attribution(self):
|
||||
"""C2PA issuers win first: a manifest-backed file is not Meta-routed."""
|
||||
r = identify(SAMPLES_DIR / "flux-1.png", check_visible=False, check_invisible=False)
|
||||
assert "content_seal" not in [s.name for s in r.signals]
|
||||
assert "content_seal" not in names
|
||||
assert r.platform is None
|
||||
assert not any("Content Seal" in watermark for watermark in r.watermarks)
|
||||
|
||||
def test_flux_bfl_c2pa_png(self):
|
||||
# flux-1.png: real Black Forest Labs FLUX.2 Playground output (signed C2PA).
|
||||
@@ -1541,7 +1508,7 @@ class TestIdentifyInvisibleWatermark:
|
||||
|
||||
|
||||
class TestIdentifyAIGC:
|
||||
"""China TC260 AIGC label is detected and attributed (e.g. Doubao)."""
|
||||
"""China TC260 AIGC labels are detected without guessing the product."""
|
||||
|
||||
def _aigc_png(self, tmp_path: Path) -> Path:
|
||||
from PIL import Image
|
||||
@@ -1644,6 +1611,11 @@ class TestVendorOf:
|
||||
|
||||
|
||||
class TestIntegrityClashesHelper:
|
||||
def test_tc260_manufacturer_comes_from_the_registered_producer(self):
|
||||
assert _tc260_manufacturer_of("001191440101MA9Y9T4H7A00001") == "Alibaba"
|
||||
assert _tc260_manufacturer_of("0011999999999999999999999") is None
|
||||
assert _tc260_manufacturer_of("") is None
|
||||
|
||||
def test_two_ai_vendors_clash(self):
|
||||
clashes = _integrity_clashes({"c2pa": "OpenAI", "exif_generator": "Ideogram"}, None, camera_has_ai_marker=True)
|
||||
assert len(clashes) == 1
|
||||
@@ -1665,19 +1637,11 @@ class TestIntegrityClashesHelper:
|
||||
== []
|
||||
)
|
||||
|
||||
def test_bytedance_c2pa_plus_own_aigc_no_clash(self):
|
||||
# A legit ByteDance/Doubao image carries BOTH a ByteDance C2PA manifest and its
|
||||
# own China TC260 AIGC label. The label is ByteDance's own regulatory stamp, so
|
||||
# it must be attributed to ByteDance and NOT read as a competing origin.
|
||||
assert (
|
||||
_integrity_clashes({"c2pa": "ByteDance", "aigc": "China AIGC (TC260)"}, None, camera_has_ai_marker=True)
|
||||
== []
|
||||
)
|
||||
def test_matching_tc260_manufacturer_no_clash(self):
|
||||
assert _integrity_clashes({"c2pa": "ByteDance", "aigc": "ByteDance"}, None, camera_has_ai_marker=True) == []
|
||||
|
||||
def test_foreign_vendor_plus_aigc_still_clashes(self):
|
||||
# But a NON-Chinese vendor's C2PA next to a China TC260 label names two different
|
||||
# origins -- a laundering tell that must still fire (the generic label stays generic).
|
||||
clashes = _integrity_clashes({"c2pa": "OpenAI", "aigc": "China AIGC (TC260)"}, None, camera_has_ai_marker=True)
|
||||
def test_different_tc260_manufacturer_clashes(self):
|
||||
clashes = _integrity_clashes({"c2pa": "OpenAI", "aigc": "ByteDance"}, None, camera_has_ai_marker=True)
|
||||
assert len(clashes) == 1
|
||||
assert "Conflicting AI-origin" in clashes[0]
|
||||
|
||||
@@ -1754,9 +1718,13 @@ class TestIntegrityClashEndToEnd:
|
||||
|
||||
def test_two_generator_stamps_clash(self, tmp_path: Path):
|
||||
# An OpenAI C2PA manifest (AI source) on an image that ALSO carries a
|
||||
# China TC260 AIGC label = two independent generator stamps naming
|
||||
# different origins -> a laundering tell.
|
||||
path = self._c2pa_jpeg(tmp_path, b"OpenAI ... trainedAlgorithmicMedia ... TC260:AIGC label")
|
||||
# Qwen TC260 AIGC label = two independent generator stamps naming
|
||||
# different manufacturers -> a laundering tell.
|
||||
path = self._c2pa_jpeg(
|
||||
tmp_path,
|
||||
b"OpenAI ... trainedAlgorithmicMedia ... "
|
||||
b'<TC260:AIGC>{"Label":"1","ContentProducer":"91440101MA9Y9T4H7A"}</TC260:AIGC>',
|
||||
)
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.integrity_clashes
|
||||
assert any("Conflicting AI-origin" in c for c in r.integrity_clashes)
|
||||
@@ -1773,7 +1741,7 @@ class TestIntegrityClashEndToEnd:
|
||||
# the provenance is inconsistent (a laundering / spoofing tell).
|
||||
path = self._c2pa_jpeg(
|
||||
tmp_path,
|
||||
b'Pixel Camera ... <TC260:AIGC>{"Label":"1","ContentProducer":"BYTEDANCE001"}</TC260:AIGC>',
|
||||
b'Pixel Camera ... <TC260:AIGC>{"Label":"1","ContentProducer":"91110102MACQD9K640"}</TC260:AIGC>',
|
||||
)
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.platform == "Google Pixel (camera, C2PA capture)"
|
||||
|
||||
@@ -409,3 +409,154 @@ class TestRemoverWriteCarriesDisplayTags:
|
||||
with Image.open(out) as im:
|
||||
assert im.info.get("icc_profile") == icc
|
||||
assert im.getexif().get(0x0112) is None
|
||||
|
||||
|
||||
class TestTiffOrientation:
|
||||
"""Pillow's TIFF reader applies the EXIF orientation itself, and reports the
|
||||
upright size while doing it. A stored-size comparison against ``Image.size``
|
||||
therefore matches for a raster that is already upright, and the tag is carried
|
||||
onto it -- a viewer then turns the picture a second time."""
|
||||
|
||||
ORIENT = 6 # Rotate 90 CW.
|
||||
|
||||
@staticmethod
|
||||
def _tagged(path: Path, orient: int) -> Path:
|
||||
"""A 96x64 stored raster carrying ``orient``, written by the container itself."""
|
||||
import piexif
|
||||
from PIL import Image
|
||||
|
||||
exif = piexif.dump({"0th": {piexif.ImageIFD.Orientation: orient}})
|
||||
Image.new("RGB", (96, 64), "red").save(path, exif=exif)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _out_orientation(path: Path) -> object:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as im:
|
||||
return im.getexif().get(0x0112)
|
||||
|
||||
def test_a_turned_tiff_decode_is_not_tagged_into_a_second_rotation(self, tmp_path: Path) -> None:
|
||||
src = self._tagged(tmp_path / "src.tif", self.ORIENT)
|
||||
raster = image_io.imread(src, cv2.IMREAD_UNCHANGED)
|
||||
assert raster is not None
|
||||
# Pillow already applied the rotation: the raster is upright, 64x96.
|
||||
assert (raster.shape[1], raster.shape[0]) == (64, 96)
|
||||
out = tmp_path / "out.png"
|
||||
assert image_io.imwrite(out, raster, display_tags_from=src) is True
|
||||
assert self._out_orientation(out) is None
|
||||
|
||||
def test_the_visible_api_path_does_not_tag_a_turned_tiff(self, tmp_path: Path) -> None:
|
||||
# The visible write passes orientation_applied=False ("raw decode"), which is
|
||||
# a lie for TIFF: both readers turn the raster. _read_display_tags must ignore
|
||||
# that claim for TIFF and let the IFD geometry decide, or a CLI/library user
|
||||
# gets a turned raster TAGGED for the same turn again (issue #106).
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks import api
|
||||
|
||||
src = self._tagged(tmp_path / "src.tif", self.ORIENT)
|
||||
out = tmp_path / "out.png"
|
||||
_, removed = api.remove_visible(src, out, strip_metadata=False)
|
||||
assert removed == [] # plain red: no mark, the passthrough write is exercised
|
||||
with Image.open(out) as im:
|
||||
assert im.size == (64, 96) # the upright raster, not the stored landscape
|
||||
assert im.getexif().get(0x0112) is None # and no tag asking for another turn
|
||||
|
||||
|
||||
class TestHeifOrientation:
|
||||
"""A camera HEIC stores its raster and carries the rotation in the EXIF
|
||||
payload; a pillow-heif-written one bakes the rotation as an ``irot`` transform
|
||||
that the decode applies. pillow-heif blanks the EXIF tag on open for both and
|
||||
stashes the real value in ``info["original_orientation"]``, and ``Image.size``
|
||||
matches the raster either way, so the container's transform boxes are the only
|
||||
signal telling the two apart (issue #105)."""
|
||||
|
||||
ORIENT = 6 # Rotate 90 CW.
|
||||
|
||||
@staticmethod
|
||||
def _baked(path: Path) -> Path:
|
||||
"""A 96x64 stored raster whose rotation is baked as an irot transform."""
|
||||
import piexif
|
||||
from PIL import Image
|
||||
|
||||
exif = piexif.dump({"0th": {piexif.ImageIFD.Orientation: TestHeifOrientation.ORIENT}})
|
||||
Image.new("RGB", (96, 64), "red").save(path, exif=exif)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _camera_style(path: Path) -> Path:
|
||||
"""The camera counterpart: same EXIF payload, identity transform, so the
|
||||
decode returns the stored raster and only the tag says to turn it."""
|
||||
data = bytearray(TestHeifOrientation._baked(path).read_bytes())
|
||||
meta_end = data.find(b"mdat")
|
||||
irot = data.find(b"irot", 0, meta_end if meta_end != -1 else len(data))
|
||||
assert irot != -1, "fixture lost its irot box"
|
||||
data[irot + 4] = 0 # angle 0 is the identity transform
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def test_malformed_input_reads_as_no_transform(self, tmp_path: Path) -> None:
|
||||
from remove_ai_watermarks._internal import isobmff
|
||||
|
||||
not_bmff = tmp_path / "garbage.bin"
|
||||
not_bmff.write_bytes(b"\x00\x00\x00\x0cJUNKgarbage")
|
||||
assert isobmff.heif_transform_applied(not_bmff) is False
|
||||
empty = tmp_path / "empty.heic"
|
||||
empty.write_bytes(b"")
|
||||
assert isobmff.heif_transform_applied(empty) is False
|
||||
assert isobmff.heif_transform_applied(tmp_path / "missing.heic") is False
|
||||
|
||||
@pytest.mark.skipif(not _heif_writable("HEIF"), reason="no HEIC encoder in this env")
|
||||
def test_the_transform_box_alone_reports_the_decode_state(self, tmp_path: Path) -> None:
|
||||
from remove_ai_watermarks._internal import isobmff
|
||||
|
||||
assert isobmff.heif_transform_applied(self._baked(tmp_path / "baked.heic")) is True
|
||||
assert isobmff.heif_transform_applied(self._camera_style(tmp_path / "cam.heic")) is False
|
||||
|
||||
@pytest.mark.skipif(not _heif_writable("HEIF"), reason="no HEIC encoder in this env")
|
||||
def test_a_camera_heic_carries_its_orientation_tag(self, tmp_path: Path) -> None:
|
||||
# Issue #105: the blanked tag used to be dropped, so the output went out
|
||||
# sideways with nothing on it saying so.
|
||||
from PIL import Image
|
||||
|
||||
src = self._camera_style(tmp_path / "cam.heic")
|
||||
raster = image_io.imread(src, cv2.IMREAD_UNCHANGED)
|
||||
assert raster is not None
|
||||
assert (raster.shape[1], raster.shape[0]) == (96, 64) # the stored raster
|
||||
out = tmp_path / "out.png"
|
||||
assert image_io.imwrite(out, raster, display_tags_from=src) is True
|
||||
with Image.open(out) as im:
|
||||
assert im.size == (96, 64)
|
||||
assert im.getexif().get(0x0112) == self.ORIENT # the turn re-declared
|
||||
|
||||
@pytest.mark.skipif(not _heif_writable("HEIF"), reason="no HEIC encoder in this env")
|
||||
def test_a_baked_heif_is_not_tagged_into_a_second_rotation(self, tmp_path: Path) -> None:
|
||||
from PIL import Image
|
||||
|
||||
src = self._baked(tmp_path / "baked.heic")
|
||||
raster = image_io.imread(src, cv2.IMREAD_UNCHANGED)
|
||||
assert raster is not None
|
||||
assert (raster.shape[1], raster.shape[0]) == (64, 96) # libheif turned it upright
|
||||
out = tmp_path / "out.png"
|
||||
assert image_io.imwrite(out, raster, display_tags_from=src) is True
|
||||
with Image.open(out) as im:
|
||||
assert im.getexif().get(0x0112) is None
|
||||
|
||||
@pytest.mark.skipif(not _heif_writable("HEIF"), reason="no HEIC encoder in this env")
|
||||
def test_the_visible_api_path_carries_a_camera_heic_tag(self, tmp_path: Path) -> None:
|
||||
# The visible write passes orientation_applied=False ("raw decode"), a
|
||||
# claim it cannot make for HEIF: whether libheif turned the raster is a
|
||||
# per-file fact of the container. _read_display_tags must override the
|
||||
# claim the way it does for TIFF (issue #105).
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks import api
|
||||
|
||||
src = self._camera_style(tmp_path / "cam.heic")
|
||||
out = tmp_path / "out.png"
|
||||
_, removed = api.remove_visible(src, out, strip_metadata=False)
|
||||
assert removed == [] # plain red: no mark, the passthrough write is exercised
|
||||
with Image.open(out) as im:
|
||||
assert im.size == (96, 64) # the stored raster...
|
||||
assert im.getexif().get(0x0112) == self.ORIENT # ...with its turn re-declared
|
||||
@@ -90,13 +90,6 @@ class TestConfig:
|
||||
assert "可灵AI" in mark.label
|
||||
assert mark.in_auto
|
||||
|
||||
def test_confident_kling_detection_suppresses_the_jimeng_pill(self):
|
||||
# A Kling image is TC260 too but is not Jimeng-basic: like Doubao and Qwen,
|
||||
# a confident Kling detection must veto the pill (``_keep_pill``).
|
||||
from remove_ai_watermarks.watermark_registry import _keep_pill
|
||||
|
||||
assert not _keep_pill({"kling"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
|
||||
|
||||
class TestDetect:
|
||||
def test_clean_gradient_not_detected(self):
|
||||
|
||||
@@ -162,14 +162,6 @@ class TestDetectAndMask:
|
||||
img = np.full((2400, 1792, 3), 100, np.uint8)
|
||||
assert eng.footprint_mask(img) is None
|
||||
|
||||
def test_confident_liblib_detection_suppresses_the_jimeng_pill(self):
|
||||
# A LiblibAI image is TC260 too but is not Jimeng-basic: like Doubao/Qwen/
|
||||
# Kling/RunningHub/Baidu, a confident LiblibAI detection must veto the pill.
|
||||
# It was the one mark the hand-written veto list in ``_keep_pill`` missed.
|
||||
from remove_ai_watermarks.watermark_registry import _keep_pill
|
||||
|
||||
assert not _keep_pill({"liblib"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
|
||||
def test_force_masks_the_whole_locate_box_on_a_clean_frame(self):
|
||||
"""``force`` takes priority over detection for this mark, unlike the base
|
||||
policy: a --no-detect caller named the mark, so the honest footprint is the
|
||||
|
||||
@@ -76,7 +76,7 @@ def test_trustmark_is_limited_to_its_numpy_compatible_python_range():
|
||||
def test_extras_use_capability_names_without_legacy_aliases():
|
||||
extras = set(metadata("remove-ai-watermarks").get_all("Provides-Extra") or [])
|
||||
|
||||
assert {"pixels", "heif", "visible", "video", "detect", "diffusion", "classify"} <= extras
|
||||
assert {"pixels", "heif", "visible", "video", "detect", "diffusion", "classify", "classify-onnx"} <= extras
|
||||
assert {"gpu", "remove", "detect-pywavelets", "verify"}.isdisjoint(extras)
|
||||
|
||||
|
||||
@@ -85,6 +85,11 @@ def test_classify_extra_owns_the_photo_heads() -> None:
|
||||
assert {"torch", "transformers", "huggingface-hub", "tokenizers"} <= names
|
||||
|
||||
|
||||
def test_classify_onnx_extra_extends_the_default_classifier() -> None:
|
||||
names = _requirement_names("classify-onnx")
|
||||
assert {"torch", "transformers", "huggingface-hub", "tokenizers", "onnxruntime"} <= names
|
||||
|
||||
|
||||
def test_qwen_zimage_extra_owns_every_profile_tokenizer_backend() -> None:
|
||||
assert "sentencepiece" in _requirement_names("qwen-zimage")
|
||||
|
||||
|
||||
@@ -263,16 +263,13 @@ class TestResolveStrength:
|
||||
assert resolve_strength(None, "meta", "qwen-zimage", size=size) == pytest.approx(QWEN_ZIMAGE_META_STRENGTH)
|
||||
assert resolve_strength(None, "meta", "sdxl-zimage") == SDXL_ZIMAGE_UNKNOWN_STRENGTH
|
||||
|
||||
def test_vendor_for_strength_routes_standalone_iptc_to_meta(self, tmp_path):
|
||||
"""Auto mode: a file whose only provenance is the AI IPTC tag routes to the
|
||||
meta cohort (Muse carries no C2PA; the tag is its fallback companion), while
|
||||
a file without the tag stays on the resolution curve and a C2PA issuer still
|
||||
wins over the tag."""
|
||||
def test_vendor_for_strength_does_not_treat_shared_iptc_as_meta(self, tmp_path):
|
||||
"""A standard AI tag does not identify the Meta strength cohort."""
|
||||
from remove_ai_watermarks._internal.watermark_profiles import vendor_for_strength
|
||||
|
||||
tagged = tmp_path / "tagged.webp"
|
||||
tagged.write_bytes(_MUSE_WEBP_WITH_IPTC_TAG)
|
||||
assert vendor_for_strength(tagged) == "meta"
|
||||
assert vendor_for_strength(tagged) is None
|
||||
|
||||
stripped = tmp_path / "stripped.webp"
|
||||
stripped.write_bytes(b"RIFF\x24\x00\x00\x00WEBPVP8 \x18\x00\x00\x00" + b"\x00" * 16)
|
||||
@@ -460,6 +457,7 @@ class TestPlatformPaths:
|
||||
asset_dir = Path(__file__).parent.parent / "src" / "remove_ai_watermarks" / "assets"
|
||||
assert (asset_dir / "gemini_bg_48.png").exists()
|
||||
assert (asset_dir / "gemini_bg_96.png").exists()
|
||||
assert (asset_dir / "qwen_symbol_alpha.png").exists()
|
||||
|
||||
def test_asset_loading_works(self):
|
||||
"""Verify embedded assets load correctly (critical for packaging)."""
|
||||
@@ -469,3 +467,7 @@ class TestPlatformPaths:
|
||||
# If we get here without error, asset loading works
|
||||
assert engine._alpha_small.shape == (48, 48)
|
||||
assert engine._alpha_large.shape == (96, 96)
|
||||
|
||||
from remove_ai_watermarks.qwen_engine import QwenEngine
|
||||
|
||||
assert QwenEngine()._symbol_template.shape == (53, 53)
|
||||
@@ -23,8 +23,19 @@ def test_hub_id_is_the_photo_classify_repo() -> None:
|
||||
assert _publish().HUB_REPO == "wiltodelta/raiw-photo-classify"
|
||||
|
||||
|
||||
def test_publish_freeze_includes_both_clip_runtimes() -> None:
|
||||
publish = _publish()
|
||||
from remove_ai_watermarks.classify import _WEIGHT_FILES, CLIP_ONNX_FILE, WEIGHTS_ALLOW_PATTERNS
|
||||
|
||||
assert publish.CLIP_FILE in publish.WEIGHT_FILES
|
||||
assert publish.CLIP_ONNX_FILE in publish.WEIGHT_FILES
|
||||
assert set(publish.WEIGHT_FILES) == {*_WEIGHT_FILES, CLIP_ONNX_FILE}
|
||||
assert set(publish.WEIGHT_FILES) <= set(WEIGHTS_ALLOW_PATTERNS)
|
||||
|
||||
|
||||
def test_weight_paths_accepts_run1_layout(tmp_path: Path) -> None:
|
||||
(tmp_path / "clip-l-ft.pt").write_bytes(b"clip")
|
||||
(tmp_path / "clip-l-ft-vision-fp32.onnx").write_bytes(b"onnx")
|
||||
(tmp_path / "probe-weights-clip-l-ft.npz").write_bytes(b"probe")
|
||||
run1 = tmp_path / "run1"
|
||||
run1.mkdir()
|
||||
@@ -37,6 +48,7 @@ def test_weight_paths_accepts_run1_layout(tmp_path: Path) -> None:
|
||||
|
||||
def test_weight_paths_requires_every_file(tmp_path: Path) -> None:
|
||||
(tmp_path / "clip-l-ft.pt").write_bytes(b"clip")
|
||||
(tmp_path / "clip-l-ft-vision-fp32.onnx").write_bytes(b"onnx")
|
||||
with pytest.raises(SystemExit, match=r"missing probe-weights-clip-l-ft\.npz"):
|
||||
_publish().weight_paths(tmp_path)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for the Qwen (千问AI生成) visible-watermark engine (localize -> fill).
|
||||
"""Tests for the Qwen text and symbol visible-watermark engine (localize -> fill).
|
||||
|
||||
Every tuned constant in ``qwen_engine`` was measured on the 117-frame vendor
|
||||
cohort (2026-07-21, ``scripts/vendor_mark_calibrate.py``); these tests pin the
|
||||
@@ -8,15 +8,19 @@ geometry (the exact failure the calibration had to fix).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from remove_ai_watermarks import watermark_registry as registry
|
||||
from remove_ai_watermarks._text_mark_engine import TextMarkDetection, best_detection
|
||||
from remove_ai_watermarks.qwen_engine import (
|
||||
_ALPHA_HEIGHT_FRAC,
|
||||
_ALPHA_WIDTH_FRAC,
|
||||
_LADDER,
|
||||
_SYMBOL_SHAPE_IOU_THRESHOLD,
|
||||
QwenEngine,
|
||||
_alpha_template,
|
||||
_glyph_silhouette,
|
||||
@@ -27,6 +31,9 @@ from remove_ai_watermarks.qwen_engine import (
|
||||
# centres one rung on each mode.
|
||||
_BIG_MODE, _SMALL_MODE = 0.203, 0.124
|
||||
_MARGIN = 0.025 # measured right/bottom margin of the real mark
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
_QWEN_CREATE = _ROOT / "data" / "fixtures" / "provenance" / "qwen-create-qwen-image-2.png"
|
||||
_CHATGPT_1 = _ROOT / "data" / "fixtures" / "provenance" / "chatgpt-1.png"
|
||||
|
||||
|
||||
def _compose(w: int, h: int, mode: float = _BIG_MODE, bg: float = 100.0):
|
||||
@@ -87,6 +94,7 @@ class TestConfig:
|
||||
mark = registry.get_mark("qwen")
|
||||
assert mark.location == "bottom-right"
|
||||
assert "千问AI生成" in mark.label
|
||||
assert "symbol" in mark.label
|
||||
assert mark.in_auto
|
||||
|
||||
|
||||
@@ -129,6 +137,42 @@ class TestDetect:
|
||||
assert eng.detect(wm).detected
|
||||
assert not eng.detect(cv2.resize(wm, (150, 112))).detected
|
||||
|
||||
def test_qwen_create_symbol_detected(self):
|
||||
image = cv2.imread(str(_QWEN_CREATE), cv2.IMREAD_COLOR)
|
||||
assert image is not None
|
||||
det = QwenEngine().detect(image)
|
||||
assert det.detected
|
||||
assert det.confidence >= 0.80
|
||||
x, y, width, height = det.region
|
||||
assert x > image.shape[1] * 0.90
|
||||
assert y > image.shape[0] * 0.90
|
||||
assert width == height
|
||||
|
||||
def test_chatgpt_1_fixture_does_not_fire_qwen(self):
|
||||
# Issue 108: 0.40.0 reported visible_qwen 0.81 on this OpenAI C2PA
|
||||
# beach photo and auto-inpainted sand. NCC on the three-lobe template
|
||||
# is OpenCV-sensitive; the shape overlap must still refuse it.
|
||||
from remove_ai_watermarks.identify import identify
|
||||
from remove_ai_watermarks.image_io import imread
|
||||
|
||||
image = imread(_CHATGPT_1)
|
||||
assert image is not None
|
||||
det = QwenEngine().detect(image)
|
||||
assert not det.detected
|
||||
record = identify(_CHATGPT_1, check_visible=True, check_invisible=False)
|
||||
assert all(signal.name != "visible_qwen" for signal in record.signals)
|
||||
|
||||
def test_symbol_shape_iou_rejects_a_solid_blob(self):
|
||||
template = QwenEngine()._scaled_symbol_template(32)
|
||||
blob = np.full_like(template, float(template.max()))
|
||||
assert QwenEngine._symbol_shape_iou(blob, template) < _SYMBOL_SHAPE_IOU_THRESHOLD
|
||||
assert QwenEngine._symbol_shape_iou(template, template) >= _SYMBOL_SHAPE_IOU_THRESHOLD
|
||||
|
||||
def test_unaccepted_symbol_candidate_cannot_hide_detected_text(self):
|
||||
text_detection = TextMarkDetection(detected=True, confidence=0.45)
|
||||
symbol_detection = TextMarkDetection(detected=False, confidence=0.46)
|
||||
assert best_detection(text_detection, symbol_detection) is text_detection
|
||||
|
||||
|
||||
class TestFootprintMaskAndRemoval:
|
||||
@pytest.mark.parametrize("mode", [_BIG_MODE, _SMALL_MODE])
|
||||
@@ -153,3 +197,29 @@ class TestFootprintMaskAndRemoval:
|
||||
def test_clean_frame_produces_no_mask(self):
|
||||
clean = cv2.GaussianBlur(np.full((640, 853, 3), 120, np.uint8), (5, 5), 0)
|
||||
assert QwenEngine().footprint_mask(clean, force=False) is None
|
||||
|
||||
def test_chatgpt_1_auto_remove_does_not_touch_pixels(self):
|
||||
from remove_ai_watermarks.api import remove_visible_detailed
|
||||
from remove_ai_watermarks.image_io import imread
|
||||
|
||||
image = imread(_CHATGPT_1)
|
||||
assert image is not None
|
||||
report = remove_visible_detailed(image, sensitivity="auto")
|
||||
assert report.status == "no_watermark"
|
||||
assert np.array_equal(report.image, image)
|
||||
|
||||
def test_removes_qwen_create_symbol(self):
|
||||
image = cv2.imread(str(_QWEN_CREATE), cv2.IMREAD_COLOR)
|
||||
assert image is not None
|
||||
before = QwenEngine().detect(image)
|
||||
assert before.detected
|
||||
mask = QwenEngine().footprint_mask(image, detection=before)
|
||||
assert mask is not None
|
||||
x, y, width, height = before.region
|
||||
assert np.count_nonzero(mask[y : y + height, x : x + width]) < width * height * 0.90
|
||||
out, region = registry.get_mark("qwen").remove(image, backend="cv2")
|
||||
assert region == before.region
|
||||
assert not QwenEngine().detect(out).detected
|
||||
assert np.array_equal(
|
||||
out[: image.shape[0] // 2, : image.shape[1] // 2], image[: image.shape[0] // 2, : image.shape[1] // 2]
|
||||
)
|
||||
@@ -135,6 +135,48 @@ def test_distributed_version_survives_a_newer_pypi_release(tmp_path: Path) -> No
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix" or shutil.which("bash") is None, reason="Executes an Ubuntu Bash workflow")
|
||||
def test_distributed_version_reads_the_flat_single_artifact_layout(tmp_path: Path) -> None:
|
||||
producer = _step("distribute.yml", "resolve", "Record distributed release")
|
||||
consumer = _step("verify-release.yml", "verify", "Resolve target version")
|
||||
env = {
|
||||
**os.environ,
|
||||
"VERSION": "0.37.0",
|
||||
"GITHUB_RUN_ID": "123",
|
||||
"GITHUB_RUN_ATTEMPT": "2",
|
||||
"GITHUB_OUTPUT": str(tmp_path / "output"),
|
||||
"EVENT_NAME": "workflow_run",
|
||||
"INPUT_VERSION": "",
|
||||
"DISTRIBUTION_RUN_ID": "123",
|
||||
"DISTRIBUTION_RUN_ATTEMPT": "2",
|
||||
}
|
||||
subprocess.run(["bash", "-eu", "-c", producer["run"]], cwd=tmp_path, env=env, check=True)
|
||||
# download-artifact@v8 extracts the only pattern-matched artifact directly
|
||||
# into the target path, with no artifact-name directory.
|
||||
artifact = tmp_path / "distribution-identities"
|
||||
artifact.mkdir()
|
||||
(artifact / "release-identity.json").write_bytes((tmp_path / "release-identity.json").read_bytes())
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "-eu", "-c", consumer["run"]], cwd=tmp_path, env=env, capture_output=True, text=True
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "version=0.37.0" in (tmp_path / "output").read_text()
|
||||
|
||||
|
||||
def test_comfyui_waiter_outlasts_the_node_suite_then_reads_the_registry() -> None:
|
||||
script = _step("distribute.yml", "comfyui", "Sync and publish the ComfyUI nodes")["run"]
|
||||
assert "seq 1 360" in script
|
||||
assert "api.comfy.org/nodes/remove-ai-watermarks/versions" in script
|
||||
|
||||
|
||||
def test_space_pin_is_dispatched_before_the_factory_rebuild() -> None:
|
||||
script = _step("distribute.yml", "hf-space", "Bump the Space pin, then factory-rebuild")["run"]
|
||||
assert "pin-library.yml" in script
|
||||
assert "RAIW_HF_SPACE_TOKEN secret is not set" in script
|
||||
assert "restart_space" in script
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix" or shutil.which("bash") is None, reason="Executes an Ubuntu Bash workflow")
|
||||
def test_clawhub_uses_local_install(tmp_path: Path) -> None:
|
||||
script = _step("distribute.yml", "clawhub", "Publish the agent skill when its version moved")["run"]
|
||||
|
||||
@@ -124,12 +124,3 @@ class TestDetectAndMask:
|
||||
region = wm[10:60, 12:360]
|
||||
shifted[100 : 100 + region.shape[0], 200 : 200 + region.shape[1]] = region
|
||||
assert not eng.detect(shifted).detected
|
||||
|
||||
|
||||
class TestPillInteraction:
|
||||
def test_confident_runninghub_detection_suppresses_the_jimeng_pill(self):
|
||||
# A RunningHub frame names its own product; its detection must veto the
|
||||
# Jimeng pill the same way Doubao/Qwen/Kling do (``_keep_pill``).
|
||||
from remove_ai_watermarks.watermark_registry import _keep_pill
|
||||
|
||||
assert not _keep_pill({"runninghub"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Contracts for the signal-discovery corpus harnesses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import corpus_gap_scan as gap
|
||||
import sidecar_regression as regression
|
||||
|
||||
|
||||
def _report(**overrides):
|
||||
values = {
|
||||
"is_ai_generated": None,
|
||||
"platform": None,
|
||||
"signals": [],
|
||||
"integrity_clashes": [],
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_gap_markers_are_case_insensitive():
|
||||
assert "c2pa" in gap._marker_hits(b"UPPERCASE C2PA MANIFEST")
|
||||
|
||||
|
||||
def test_gap_candidates_cover_more_than_blind_unknowns():
|
||||
signal = SimpleNamespace(name="iptc")
|
||||
assert gap._candidate_classes(_report(signals=[signal]), []) == ["unattributed_signal"]
|
||||
assert gap._candidate_classes(_report(is_ai_generated=True), []) == ["unattributed_ai"]
|
||||
assert gap._candidate_classes(_report(), ["SynthID"]) == ["blind_marker"]
|
||||
|
||||
|
||||
def test_since_filter_uses_corpus_date_directories(tmp_path):
|
||||
old = tmp_path / "2026-09-01" / "old.png"
|
||||
current = tmp_path / "2026-09-10" / "current.png"
|
||||
undated = tmp_path / "loose.png"
|
||||
for path in (old, current, undated):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.touch()
|
||||
assert gap._files(tmp_path, date(2026, 9, 10)) == [current]
|
||||
|
||||
|
||||
def test_sidecar_regression_prefers_stable_signal_names():
|
||||
sidecar = {"signals": ["visible_qwen"], "watermarks": ["wording may change"]}
|
||||
assert regression.sidecar_families(sidecar) == {"visible_qwen"}
|
||||
|
||||
|
||||
def test_current_report_uses_signal_names_not_watermark_prose():
|
||||
report = SimpleNamespace(
|
||||
signals=[SimpleNamespace(name="visible_liblib")],
|
||||
watermarks=["new wording not known to the legacy mapper"],
|
||||
)
|
||||
assert regression.report_families(report) == {"visible_liblib"}
|
||||
|
||||
|
||||
def test_current_report_keeps_watermark_only_synthid_family():
|
||||
report = SimpleNamespace(signals=[], watermarks=["SynthID watermark (Google)"])
|
||||
assert regression.report_families(report) == {"synthid"}
|
||||
|
||||
|
||||
def test_registry_drives_current_visible_families():
|
||||
from remove_ai_watermarks.watermark_registry import known_marks
|
||||
|
||||
for mark in known_marks():
|
||||
expected = "visible_sparkle" if mark.key == "gemini" else f"visible_{mark.key}"
|
||||
assert regression.family_of(mark.label) == expected
|
||||
|
||||
|
||||
def test_registry_keys_cover_legacy_visible_wording():
|
||||
assert regression.family_of("Qwen 千问 AI label") == "visible_qwen"
|
||||
assert regression.family_of("Microsoft AI badge") == "visible_microsoft"
|
||||
@@ -231,6 +231,7 @@ class TestPostRemovalValidation:
|
||||
location="top-left",
|
||||
in_auto=True,
|
||||
product="test",
|
||||
manufacturer="test",
|
||||
label_regime=None,
|
||||
platform="Test",
|
||||
_detect=detect_after,
|
||||
@@ -524,26 +525,16 @@ class TestArbiter:
|
||||
assert "doubao" in keys
|
||||
assert "jimeng_pill" not in keys
|
||||
|
||||
def test_pill_dropped_on_qwen(self):
|
||||
# A Qwen frame is TC260 too but is not Jimeng-basic either: a confident
|
||||
# bottom-right 千问AI生成 detection suppresses the pill exactly like Doubao's.
|
||||
@pytest.mark.parametrize("key", ["qwen", "yuanbao"])
|
||||
def test_other_manufacturers_do_not_join_the_bytedance_pill_family(self, key):
|
||||
# These marks and Jimeng share TC260, but not a manufacturer. The shared
|
||||
# standard must not create a product relationship.
|
||||
cands = [
|
||||
self._c("qwen", strict=True, relaxed=True),
|
||||
self._c(key, strict=True, relaxed=True),
|
||||
self._c("jimeng_pill", strict=True, relaxed=True, flat=True),
|
||||
]
|
||||
keys = self._keys(cands, reg.Context(provenance=frozenset({"jimeng"})))
|
||||
assert "qwen" in keys
|
||||
assert "jimeng_pill" not in keys
|
||||
|
||||
def test_pill_dropped_on_yuanbao(self):
|
||||
# The standard Yuanbao mark identifies a different TC260 product, so a
|
||||
# coincident top-left pill match must not be treated as Jimeng-basic.
|
||||
cands = [
|
||||
self._c("yuanbao", strict=True, relaxed=True),
|
||||
self._c("jimeng_pill", strict=True, relaxed=True, flat=True),
|
||||
]
|
||||
keys = self._keys(cands, reg.Context(provenance=frozenset({"jimeng"})))
|
||||
assert keys == {"yuanbao"}
|
||||
assert keys == {key, "jimeng_pill"}
|
||||
|
||||
def test_pill_metadata_arm_gated_on_flatness(self):
|
||||
ctx = reg.Context(provenance=frozenset({"jimeng"}))
|
||||
@@ -665,10 +656,9 @@ class TestSinglePassPerception:
|
||||
class TestMarkKnowledgeIsOnTheRow:
|
||||
"""Registering a mark is ONE edit: the row carries everything about it.
|
||||
|
||||
Product family, label regime, the platform sentence and the metadata signals that
|
||||
confirm the vendor all used to live in separate hand-maintained tables across
|
||||
``watermark_registry``, ``identify`` and ``api``. That is how LiblibAI ended up
|
||||
registered but absent from the pill veto.
|
||||
Product family, manufacturer, label regime, the platform sentence and the metadata
|
||||
signals that confirm the vendor all used to live in separate hand-maintained tables
|
||||
across ``watermark_registry``, ``identify`` and ``api``.
|
||||
"""
|
||||
|
||||
def test_identify_platform_table_is_derived_from_the_rows(self):
|
||||
@@ -697,6 +687,17 @@ class TestMarkKnowledgeIsOnTheRow:
|
||||
if mark.label_regime == "tc260" and mark.key != "jimeng_pill":
|
||||
assert "aigc" in mark.provenance_signals, mark.key
|
||||
|
||||
def test_tc260_marks_keep_distinct_manufacturers(self):
|
||||
manufacturers = {mark.key: mark.manufacturer for mark in reg.known_marks() if mark.label_regime == "tc260"}
|
||||
assert manufacturers["doubao"] == manufacturers["jimeng"] == "bytedance"
|
||||
assert manufacturers["qwen"] == "alibaba"
|
||||
assert manufacturers["kling"] == "kuaishou"
|
||||
assert manufacturers["yuanbao"] == "tencent"
|
||||
|
||||
def test_tc260_producer_lookup_returns_the_registry_row(self):
|
||||
assert reg.tc260_producer_mark("91440101MA9Y9T4H7A") is reg.get_mark("qwen")
|
||||
assert reg.tc260_producer_mark("unknown") is None
|
||||
|
||||
def test_platform_token_marks_are_the_c2pa_attributed_ones(self):
|
||||
# Gemini (Google C2PA) and Microsoft (issuer "Microsoft") are the marks whose
|
||||
# vendor a C2PA platform string can confirm; every other mark reaches its
|
||||
@@ -706,32 +707,19 @@ class TestMarkKnowledgeIsOnTheRow:
|
||||
|
||||
|
||||
class TestPillSuppressors:
|
||||
"""The pill veto is derived from the registry, not hand-listed.
|
||||
"""The pill veto is derived from the Jimeng manufacturer's registry rows."""
|
||||
|
||||
The hand-written list drifted: LiblibAI was registered in the same commit as
|
||||
RunningHub and Baidu, both of which were added to the veto, and it was not. A
|
||||
derived set cannot be forgotten by the next registration.
|
||||
"""
|
||||
|
||||
def test_every_other_tc260_product_suppresses_the_pill(self):
|
||||
def test_other_bytedance_product_suppresses_the_pill(self):
|
||||
pill = reg.get_mark("jimeng_pill")
|
||||
expected = {
|
||||
m.key
|
||||
for m in reg.known_marks()
|
||||
if m.label_regime == "tc260" and m.product != reg.get_mark("jimeng_pill").product
|
||||
m.key for m in reg.known_marks() if m.manufacturer == pill.manufacturer and m.product != pill.product
|
||||
}
|
||||
assert reg._pill_suppressors() == expected
|
||||
assert "liblib" in expected
|
||||
assert expected == {"doubao"}
|
||||
|
||||
def test_pill_dropped_on_liblib(self):
|
||||
assert not reg._keep_pill({"liblib"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
|
||||
def test_pill_dropped_on_liblib_even_with_the_jimeng_wordmark(self):
|
||||
"""The veto precedes the wordmark arm, so a co-firing LiblibAI wins.
|
||||
|
||||
This is the broader half of the change: it needs neither TC260 provenance nor
|
||||
a flat footprint, so it is reachable on more inputs than the metadata arm.
|
||||
"""
|
||||
assert not reg._keep_pill({"liblib", "jimeng"}, provenance=frozenset(), footprint_flat=1.0)
|
||||
def test_other_manufacturers_do_not_suppress_the_pill(self):
|
||||
assert reg._keep_pill({"qwen"}, provenance=frozenset({"jimeng"}), footprint_flat=1.0)
|
||||
assert reg._keep_pill({"liblib", "jimeng"}, provenance=frozenset(), footprint_flat=1.0)
|
||||
|
||||
def test_pill_survives_gemini_and_samsung(self):
|
||||
"""Neither is a TC260 labeler, and neither can put "jimeng" into provenance,
|
||||
|
||||
@@ -1458,7 +1458,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "lightning"
|
||||
version = "2.6.5"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
@@ -1471,9 +1471,9 @@ dependencies = [
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c9/1d/83be8536bec71a0173e762a9a1fd92a24a5ad0d0f74c59550c3c4e6c103b/lightning-2.6.5.tar.gz", hash = "sha256:16a30310ed69afde3748491feb5d13508908effd70390d2bfc203dc0812a4b4a", size = 659201, upload-time = "2026-05-27T14:33:41.806Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/39/b8/fbf1469d917c3ed3b22e00ab3a852fb9a5f85b9dc7f7060eedc3449098e7/lightning-2.6.6.tar.gz", hash = "sha256:1c6329e241f8a85f29fedead89c1ee138b36caa08de60c260c6a7eaee81c8295", size = 659902, upload-time = "2026-09-10T09:41:03.009Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/c5/fca7144236b6fa3279d0fb3172b32576c5ad8b84a63b9432ad6592d24847/lightning-2.6.5-py3-none-any.whl", hash = "sha256:3702fb7ef4ab51a8c3d4a140f4674514fe72973a9673dfa05e07a078e2767389", size = 848611, upload-time = "2026-05-27T14:33:39.714Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/43/7da7fe42f254ce57777d1a86eece3f86d1b3329fc568d48e2bc16c8b7ebc/lightning-2.6.6-py3-none-any.whl", hash = "sha256:2bcbd6ee840071cd076c7dc9953d7ac1040df92f1c5fa76583aae644f037ab85", size = 849265, upload-time = "2026-09-10T09:41:00.882Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3176,7 +3176,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pytorch-lightning"
|
||||
version = "2.6.5"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
@@ -3188,9 +3188,9 @@ dependencies = [
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/52/2c/8e73a3929b4c4bd600cafd38a97aaf7242a8cf518fb9f33d27c274ec898f/pytorch_lightning-2.6.5.tar.gz", hash = "sha256:1c32cefa76a1a9c4c5250338272d961d1e48b180e68396849efe128538ddb28e", size = 661673, upload-time = "2026-05-27T14:33:41.961Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/97/e4/5025cb42311b4f8cc6d1b12cd81399ca433159f8e3831ab229c5a6c5177e/pytorch_lightning-2.6.6.tar.gz", hash = "sha256:52c52b20522dd3d3feb3bb3aae2dbe3f291e8e836d4111a74cfd72e9df018834", size = 662507, upload-time = "2026-09-10T09:41:16.684Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/4d/5740c27110b83634d8491c3b5facf0111b3e554c3164f4fb953be9bddaf6/pytorch_lightning-2.6.5-py3-none-any.whl", hash = "sha256:62d9c8549b2278fedc3364f0a5607a56c6063d18635008f8cf3fae8d802b0d76", size = 852407, upload-time = "2026-05-27T14:33:39.856Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/cf/ba8e4cc1b982ff6dc6f9fdef0e4074605438b82640f6270888a10e7e42c3/pytorch_lightning-2.6.6-py3-none-any.whl", hash = "sha256:71f95c7b22f25c4f91cc659e1734bcdc6c69c8b66543f2ed2d78dbe29ce2f167", size = 853045, upload-time = "2026-09-10T09:41:14.923Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3449,7 +3449,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.39.0"
|
||||
version = "0.40.1"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "c2pa-python" },
|
||||
@@ -3492,6 +3492,17 @@ classify = [
|
||||
{ name = "torch" },
|
||||
{ name = "transformers" },
|
||||
]
|
||||
classify-onnx = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "onnxruntime" },
|
||||
{ name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "opencv-python-headless", version = "4.14.0.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "tokenizers" },
|
||||
{ name = "torch" },
|
||||
{ name = "transformers" },
|
||||
]
|
||||
detect = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
@@ -3630,6 +3641,7 @@ requires-dist = [
|
||||
{ name = "invisible-watermark", marker = "extra == 'dev'", specifier = ">=0.2.0" },
|
||||
{ name = "numpy", marker = "python_full_version >= '3.13' and extra == 'pixels'", specifier = ">=2" },
|
||||
{ name = "numpy", marker = "python_full_version < '3.13' and extra == 'pixels'", specifier = ">=1.24.0" },
|
||||
{ name = "onnxruntime", marker = "extra == 'classify-onnx'", specifier = ">=1.24.0" },
|
||||
{ name = "onnxruntime", marker = "extra == 'lama'", specifier = ">=1.24.0" },
|
||||
{ name = "onnxruntime", marker = "extra == 'migan'", specifier = ">=1.24.0" },
|
||||
{ name = "openai", marker = "extra == 'dev'", specifier = ">=2.52.0" },
|
||||
@@ -3647,6 +3659,7 @@ requires-dist = [
|
||||
{ name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.0" },
|
||||
{ name = "pywavelets", marker = "extra == 'detect'", specifier = ">=1.1.1" },
|
||||
{ name = "remove-ai-watermarks", extras = ["classify"], marker = "extra == 'classify-onnx'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["detect"], marker = "extra == 'dev'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["diffusion"], marker = "extra == 'qwen-zimage'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'classify'" },
|
||||
@@ -3671,7 +3684,7 @@ requires-dist = [
|
||||
{ name = "transformers", marker = "extra == 'diffusion'", specifier = ">=5,<6" },
|
||||
{ name = "trustmark", marker = "python_full_version < '3.13' and extra == 'trustmark'", specifier = ">=0.8.0" },
|
||||
]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "lama", "migan", "dev", "all"]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "classify-onnx", "lama", "migan", "dev", "all"]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
|
||||
Reference in new issue
Block a user