mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 21:40:47 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7021b900b | ||
|
|
3f55df6f99 | ||
|
|
4795149d34 | ||
|
|
cf30b4f95a | ||
|
|
8ce8bf14da | ||
|
|
3fb94b220d | ||
|
|
d2c5aa66fc | ||
|
|
380820fa7e | ||
|
|
12e9a87865 |
No files matched your search
@@ -129,6 +129,10 @@ Do not classify an entire module as untestable because its main path downloads a
|
||||
- the photo-classify gate in `test_classify.py`: DEFINITELY is ridge AND MLP,
|
||||
POSSIBLY does not emit a provider, and `identify` does not import
|
||||
`remove_ai_watermarks.classify`.
|
||||
- the source-export classifier in `test_source_classify.py`: the Hub revision
|
||||
and artifact hash are immutable, the pickle-free schema and exact GELU are
|
||||
validated, threshold mutation changes the real decision, and `identify`
|
||||
never consumes the result as watermark evidence.
|
||||
- the `InvisibleOptions` defaults, in `test_api.py`. When one signature promises to
|
||||
mirror another, compare them field by field rather than pinning the values you happen
|
||||
to know about, so the next field added on one side and not the other fails at the
|
||||
|
||||
@@ -7,8 +7,8 @@ name: Distribute on release
|
||||
# - HF Space: factory-rebuild the version already pinned in the Space repo.
|
||||
# - ComfyUI: sync, test, and publish the node against the exact library release.
|
||||
# - ClawHub: publish the agent skill when its own version moved.
|
||||
# All wait for the freshly published sdist to appear on PyPI first, since the
|
||||
# Release event fires in parallel with publish.yml's upload.
|
||||
# All wait for the freshly published distribution to appear in both PyPI APIs
|
||||
# first, since the Release event fires in parallel with publish.yml's upload.
|
||||
|
||||
on:
|
||||
release:
|
||||
@@ -22,6 +22,9 @@ on:
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
# Thirty attempts may spend up to 20 seconds each in the JSON request,
|
||||
# simple-index request, and inter-attempt delay. Leave bounded headroom.
|
||||
timeout-minutes: 25
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
url: ${{ steps.sdist.outputs.url }}
|
||||
@@ -63,26 +66,35 @@ jobs:
|
||||
path: release-identity.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Wait for the sdist on PyPI, capture url + sha256
|
||||
- name: Wait for the PyPI JSON and simple indexes, capture sdist metadata
|
||||
id: sdist
|
||||
run: |
|
||||
set -euo pipefail
|
||||
v="${{ steps.v.outputs.version }}"
|
||||
for i in $(seq 1 30); do
|
||||
if rel=$(curl -fsSL "https://pypi.org/pypi/remove-ai-watermarks/$v/json" 2>/dev/null); then
|
||||
url=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
sha=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
if rel=$(curl --connect-timeout 10 --max-time 20 -fsSL "https://pypi.org/pypi/remove-ai-watermarks/$v/json" 2>/dev/null); then
|
||||
read -r url sha <<< "$(echo "$rel" | python3 -c "import sys,json;u=next((u for u in json.load(sys.stdin)['urls'] if u['packagetype']=='sdist'),{});print(u.get('url',''),u.get('digests',{}).get('sha256',''))")"
|
||||
if [ -n "$url" ] && [ -n "$sha" ]; then
|
||||
echo "url=$url" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "Found sdist for $v"
|
||||
exit 0
|
||||
# PyPI's JSON API can show a freshly uploaded file before the
|
||||
# simple index used by pip has caught up. Downstream release
|
||||
# jobs install by package version, so waiting on JSON alone
|
||||
# creates a release-time race for them.
|
||||
simple=$(curl --connect-timeout 10 --max-time 20 -fsSL https://pypi.org/simple/remove-ai-watermarks/ 2>/dev/null || true)
|
||||
artifact=${url##*/}
|
||||
if [[ "$simple" == *"$artifact"* ]]; then
|
||||
echo "url=$url" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "PyPI JSON and simple indexes list $v"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo "sdist for $v not on PyPI yet (attempt $i), waiting..."
|
||||
sleep 20
|
||||
if [ "$i" -lt 30 ]; then
|
||||
echo "PyPI indexes do not both list $v yet (attempt $i), waiting..."
|
||||
sleep 20
|
||||
fi
|
||||
done
|
||||
echo "::error::sdist for $v did not appear on PyPI in time"
|
||||
echo "::error::PyPI JSON and simple indexes did not both list $v in time"
|
||||
exit 1
|
||||
|
||||
homebrew:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Publish source classifier to Hugging Face
|
||||
|
||||
# Manual only. The derivative model weights live in a GitHub model-freeze
|
||||
# release; the private training corpus and caches never enter this repository.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirm:
|
||||
description: "Type publish"
|
||||
required: true
|
||||
weights_tag:
|
||||
description: "GitHub Release tag holding source-pipeline-mlp.npz"
|
||||
required: true
|
||||
default: source-classify-freeze-2026-09-16-r2
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: ${{ github.event.inputs.confirm == 'publish' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Download derivative model weights
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ github.event.inputs.weights_tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p weights
|
||||
gh release download "$TAG" \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pattern source-pipeline-mlp.npz \
|
||||
--dir weights
|
||||
test -s weights/source-pipeline-mlp.npz
|
||||
|
||||
- name: Verify the frozen artifact
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r .artifact_sha256 docs/source-classify-hf/metrics.json)"
|
||||
test "${#expected}" -eq 64
|
||||
echo "$expected weights/source-pipeline-mlp.npz" | sha256sum --check
|
||||
|
||||
- name: Publish to Hugging Face
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${HF_TOKEN:-}" ]; then
|
||||
echo "HF_TOKEN secret is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --no-project --with huggingface-hub \
|
||||
python scripts/publish_source_classify_hf.py \
|
||||
--src weights \
|
||||
--message "Source-classify freeze (GitHub ${SHA:0:7})"
|
||||
@@ -43,7 +43,7 @@ The published Agent Skill is [`skills/remove-ai-watermarks/`](skills/remove-ai-w
|
||||
|
||||
[`docs/module-internals.md`](docs/module-internals.md) is the canonical per-module map, including design decisions, thresholds, calibration history, incident records, and regression guards. Read the relevant section before changing a subsystem.
|
||||
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); it is not `identify`. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
|
||||
## Data safety
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ specialized Python API can inspect and disrupt the validated local
|
||||
| --- | --- |
|
||||
| Metadata inspection and stripping | `remove-ai-watermarks` |
|
||||
| Photograph AI-versus-camera classification | `remove-ai-watermarks[classify]` |
|
||||
| OpenAI/Google/unknown source-export classification | `remove-ai-watermarks[source-classify]` |
|
||||
| Visible detection and removal | `remove-ai-watermarks[visible]` |
|
||||
| Visible video processing | `remove-ai-watermarks[video]` |
|
||||
| Video SynthID removal | `remove-ai-watermarks[video,diffusion]` |
|
||||
@@ -68,7 +69,7 @@ specialized Python API can inspect and disrupt the validated local
|
||||
| Every production feature available on the active Python | `remove-ai-watermarks[all]` |
|
||||
|
||||
Lower-level and specialized extras include `pixels`, `heif`, `trustmark`,
|
||||
`migan`, `lama`, `diffusion`, and `classify-onnx`. The
|
||||
`migan`, `lama`, `diffusion`, `classify-onnx`, and `source-classify`. The
|
||||
[installation guide](docs/installation.md#feature-extras) documents their exact
|
||||
dependency composition, Python compatibility, and model requirements.
|
||||
|
||||
@@ -96,6 +97,19 @@ remove-ai-watermarks classify image.png
|
||||
|
||||
Guide: [photo pixel classification](docs/photo-classify.md).
|
||||
|
||||
For a lightweight, abstaining OpenAI/Google/unknown source-export signal after
|
||||
metadata removal, use the separate Python API. It is not a SynthID detector:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("image.png")
|
||||
print(result.label, result.reason)
|
||||
```
|
||||
|
||||
Install `remove-ai-watermarks[source-classify]`. Guide:
|
||||
[source-pipeline classification](docs/source-classify.md).
|
||||
|
||||
Signed provenance is the supported route for SynthID and `identify` reads it.
|
||||
There is no local SynthID pixel detector in the package. Research on a
|
||||
periodic lattice expert is in `scripts/synthid_runtime/` and
|
||||
|
||||
+8
-7
@@ -529,9 +529,9 @@ remove-ai-watermarks invisible image.png -o clean.png \
|
||||
--pipeline qwen-zimage --text-manifest verified-lines.json --force
|
||||
```
|
||||
|
||||
Use an explicit `qwen-zimage` or `chroma-zimage` profile with this option. For
|
||||
Google provenance, `auto` resolves to SDXL, whose VAE path has not been calibrated
|
||||
for verified-text restoration and is therefore rejected.
|
||||
The option works with `qwen-zimage`, `sdxl-zimage`, `chroma-zimage`, and `auto`.
|
||||
Each concrete profile reconstructs the donor through its own already loaded VAE;
|
||||
for Google provenance, `auto` therefore keeps the normal SDXL route.
|
||||
|
||||
``verified: true`` may also be set by an automated operator that attests
|
||||
machine-verified geometry: stability-gated detector boxes inside sane caps. Such
|
||||
@@ -556,10 +556,11 @@ container changes remain valid while a resized or edited source fails closed. Th
|
||||
experimental helper
|
||||
`remove_ai_watermarks._internal.text_restoration.source_pixel_sha256` computes it.
|
||||
|
||||
This mode is supported by `qwen-zimage`, `chroma-zimage`, and `auto` at native
|
||||
geometry with `humanize=0`, `unsharp=0`, and adaptive polish disabled. The legacy
|
||||
`sdxl-zimage` profile does not expose a verified-text VAE donor. `all` also accepts the flag,
|
||||
but its manifest must match the pixels entering the invisible stage; if visible-mark
|
||||
This mode is supported by every profile at native geometry with `humanize=0` and
|
||||
`unsharp=0`. An unset adaptive-polish option is forced off so verified text remains
|
||||
the final pixel-writing stage; an explicit request to enable it is rejected. `all`
|
||||
also accepts the flag, but its
|
||||
manifest must match the pixels entering the invisible stage; if visible-mark
|
||||
removal changes those pixels, the hash check rejects the run. One oracle verdict does
|
||||
not certify another manifest, seed, model/runtime version, or output hash.
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ to run the tool. Use the maintainer references only when changing the code.
|
||||
| [Supported signals](supported-signals.md) | You need to know which visible marks, metadata formats, and invisible signals are covered. |
|
||||
| [Known limitations](known-limitations.md) | You need the quality, device, format, or verification boundaries. |
|
||||
| [Photo pixel classification](photo-classify.md) | You want `classify` / `classify_pixels`: AI versus camera, optional provider. |
|
||||
| [Source-pipeline classification](source-classify.md) | You want an abstaining OpenAI/Google/unknown export-pipeline signal after metadata removal. |
|
||||
| [Photo classifier training](photo-classify-training.md) | You need the retrain pack, public sources, and what is not published. |
|
||||
| [Scope, safety, and legal notes](legal-and-safety.md) | You need the intended use and legal context. |
|
||||
|
||||
@@ -42,6 +43,7 @@ The current behavior is defined by the code, tests, README, and user guides.
|
||||
- [AI-generated image classifiers](ai-generated-image-classifiers.md) (photo AI-versus-camera freeze consumed by `classify_pixels`, not `identify`)
|
||||
- [Photo classifier Hugging Face card](photo-classify-hf/README.md) (Hub publication text of the 2026-08-31 freeze)
|
||||
- [SynthID source classifiers](synthid-classifiers.md) (metadata-free OpenAI/Gemini source finding and provider-lineage experiments)
|
||||
- [Source-pipeline classifier model card](source-classify-hf/README.md) (published derivative weights, aggregate evaluation, and intended-use boundary)
|
||||
- [SynthID local detector research](synthid-detector-research.md)
|
||||
- [SynthID mark removal research](synthid-removal-research.md)
|
||||
- [SynthID identity research](synthid-robust-identity-research.md)
|
||||
|
||||
@@ -129,6 +129,7 @@ application actually uses:
|
||||
| `trustmark` | Adobe TrustMark detection on Python 3.11-3.12 | trustmark | Yes |
|
||||
| `classify` | Metadata-free photo AI-versus-camera classifier plus gated provider | `pixels`, Torch, Transformers | Yes |
|
||||
| `classify-onnx` | Optional CPU-only ONNX vision runtime for `classify` | `classify`, ONNX Runtime | Yes |
|
||||
| `source-classify` | Abstaining OpenAI/Google/unknown source-export classifier, not SynthID detection | `pixels`, Hugging Face Hub | Model download, no Torch |
|
||||
| `diffusion` | Torch and Diffusers runtime; video SynthID regeneration | `pixels`, Torch, Diffusers | Yes |
|
||||
| `migan` | MI-GAN ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
| `lama` | big-LaMa ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
@@ -156,6 +157,7 @@ flowchart LR
|
||||
trustmark
|
||||
classify --> pixels
|
||||
onnx["classify-onnx"] --> classify
|
||||
source["source-classify"] --> pixels
|
||||
```
|
||||
|
||||
`heif`, `trustmark`, and `text-draft` are independent branches. Combine them
|
||||
|
||||
@@ -20,6 +20,14 @@ The command is never started by `identify`. A no-signal provenance result stays
|
||||
unknown. This is not a clean verdict and it does not run cleanup. Full guide:
|
||||
[photo pixel classification](photo-classify.md).
|
||||
|
||||
`classify_source` is a different, lightweight OpenAI/Google/unknown
|
||||
source-export classifier. It is useful only while the pixels still resemble an
|
||||
original export. A 95% resize reduced measured Google recall to 1%, and a JPEG
|
||||
quality-75 round trip made every public evaluation row abstain. Its provider
|
||||
label is not evidence that SynthID is present, and `unknown` is not evidence
|
||||
that SynthID is absent. It is also never started by `identify`. Full guide:
|
||||
[source-pipeline classification](source-classify.md).
|
||||
|
||||
## Visible removal
|
||||
|
||||
### Fill quality depends on the background
|
||||
|
||||
@@ -1116,6 +1116,30 @@ both backends explicitly (`tests/test_classify.py` pins the seam). User guide:
|
||||
[photo-classify.md](photo-classify.md). Hub card:
|
||||
[photo-classify-hf/README.md](photo-classify-hf/README.md).
|
||||
|
||||
### Source-pipeline classifier
|
||||
|
||||
[`source_classify.py`](../src/remove_ai_watermarks/source_classify.py) is a
|
||||
separate, lightweight OpenAI/Google/unknown source-export classifier. It fuses
|
||||
the existing 124-dimensional patch descriptor with the 768-dimensional
|
||||
phase-free spectrum in
|
||||
[`_internal/source_spectral.py`](../src/remove_ai_watermarks/_internal/source_spectral.py),
|
||||
then evaluates the frozen 892 -> 128 -> 64 -> 3 GELU MLP in NumPy. Provider
|
||||
labels clear independent calibrated margins; every other row is `unknown`.
|
||||
The NumPy decision treats values within `1e-5` of the stored margin as the
|
||||
inclusive training threshold, absorbing float32 BLAS differences; this matches
|
||||
the frozen training-runtime decisions on all 27,811 cached internal and
|
||||
external feature rows.
|
||||
|
||||
The runtime pins both the immutable Hub revision and the NPZ SHA-256, loads
|
||||
with `allow_pickle=False`, and rejects any field, shape, class, activation, or
|
||||
non-finite-value drift. `RAIW_SOURCE_CLASSIFY_WEIGHTS` is the offline override.
|
||||
The `source-classify` extra adds `pixels` and Hugging Face Hub but no Torch or
|
||||
Transformers. `identify` never imports or runs this path. Its output names a
|
||||
complete source/export pipeline and is not SynthID detection. Unit contracts
|
||||
and the threshold mutation guard live in `tests/test_source_classify.py`;
|
||||
evaluation and attack limits live in
|
||||
[`synthid-classifiers.md`](synthid-classifiers.md).
|
||||
|
||||
The CLIP loader suppresses discarded random parameter initialization with
|
||||
Transformers' `no_init_weights` context before applying the complete frozen
|
||||
state dict. Normal `CLIPModel(config)` construction initialized 427 million
|
||||
@@ -1839,8 +1863,8 @@ verification, and geometry-only operators do not need to invent text or script f
|
||||
|
||||
When enabled, the one profile selected before model loading reconstructs the source
|
||||
once through its already loaded VAE, runs the ordinary global and face stages, and
|
||||
calls the shared restoration compositor. Qwen and Chroma implement this donor hook;
|
||||
SDXL does not. The optional Qwen-only fidelity anchor first blends 15% of the VAE
|
||||
calls the shared restoration compositor. Qwen, SDXL, and Chroma implement this donor
|
||||
hook. The optional Qwen-only fidelity anchor first blends 15% of the VAE
|
||||
reconstruction into the clean result; it is off by default because the blend returned
|
||||
detector-visible OpenAI SynthID in the measured poster fixtures. The compositor derives
|
||||
binary source and candidate silhouettes, groups nearby lines, uses LaMa for the initial
|
||||
@@ -1854,8 +1878,9 @@ extend 8% above and 25% below. The anchored-component gate covers clipped leadin
|
||||
flourishes, trailing punctuation, icons, and descenders without walking into
|
||||
disconnected decoration or background texture.
|
||||
|
||||
The stage is deliberately narrower than the engine: it rejects `sdxl-zimage`, tiles,
|
||||
resolution caps, humanize, unsharp, and adaptive polish. Those combinations change
|
||||
The stage is deliberately narrower than the engine: it rejects tiles, resolution
|
||||
caps, humanize, unsharp, and explicitly enabled adaptive polish. An unset polish
|
||||
value resolves to off in this mode, including for SDXL, because those combinations change
|
||||
geometry or final pixels after the verified layer and have no measured oracle result.
|
||||
It remains opt-in because annotations are manual and provider verdicts apply only to
|
||||
the exact tested output hashes, not to the mechanism in general.
|
||||
@@ -1879,6 +1904,22 @@ the `chroma_text_restoration_study.py` harness (kept outside this repository)
|
||||
under the gitignored
|
||||
`out/text-restoration-engine-study/` directory.
|
||||
|
||||
The 2026-09-15 rerun added the SDXL donor implementation and exercised all three
|
||||
profiles on the same three manifests. SDXL improved text-box MAE from 11.319 to
|
||||
7.970 and from 10.704 to 7.657 on the two OpenAI posters, while PSNR rose from
|
||||
25.83 to 29.24 dB and from 26.17 to 29.04 dB. On the tracked Google CJK carrier
|
||||
(`4affd7f27767a445db6abf741355743ba8d95108ad922c9fff045feed8492236`, already
|
||||
recorded as SynthID-positive), SDXL at the calibrated 0.50 strength improved
|
||||
text-box MAE from 10.738 to 8.564 and PSNR from 23.93 to 24.00 dB. The exact
|
||||
restored output
|
||||
`035e4919e1d84129a347006c98f19eb4431eb5772788f6d2097b94a2c6baec84`
|
||||
was submitted to the Google Gemini oracle after provenance stripping preserved
|
||||
its decoded pixels; the sanitized upload hash was
|
||||
`3e4f06274a7b735f7d06863fd54dc8146f60aeb9cc8315f86294180deb660b82`.
|
||||
The oracle returned `not_detected` on 2026-09-16. This
|
||||
certifies that one carrier, manifest, seed, runtime, and output hash; it does not
|
||||
turn the compositor into a provider-wide guarantee.
|
||||
|
||||
A matched stage-isolation check on the 18-face Gemini portrait grid confirms the
|
||||
division of responsibility. The visible-cleaned, metadata-stripped control and the
|
||||
Z-Image face-only output were both SynthID-positive; Qwen global-only and the full
|
||||
@@ -1986,8 +2027,9 @@ is supported by this pass, so `auto` keeps the measured provenance-cohort table.
|
||||
|
||||
[`_internal/sdxl_zimage_pipeline.py`](../src/remove_ai_watermarks/_internal/sdxl_zimage_pipeline.py)
|
||||
runs the same two-stage recipe on an SDXL global pass. `SdxlZImagePipeline`
|
||||
subclasses `TwoStageZImagePipeline` and implements only the global stage
|
||||
(`_load_global`, `_run_global`), so the face stage is inherited rather than copied
|
||||
subclasses `TwoStageZImagePipeline` and implements the global stage
|
||||
(`_load_global`, `_run_global`) plus the deterministic verified-text VAE donor, so
|
||||
the face stage is inherited rather than copied
|
||||
and cannot drift between the profiles; a test asserts the shared methods are the
|
||||
same objects.
|
||||
|
||||
@@ -2254,6 +2296,17 @@ resolution case -- every verdict so far comes from one fixture and one seed.
|
||||
Both ends of the shipped curve now have oracle verdicts, and the shipped curve clears
|
||||
everything it has been tested at:
|
||||
|
||||
A source-fresh GPT Image 2.5 check on 2026-09-15 exercised the complete shipped
|
||||
profile rather than only a strength sweep. One 1024 x 1024 API generation from
|
||||
Flare and one from Sunburst both validated as current OpenAI C2PA, routed to the
|
||||
flat OpenAI strength, and remained OpenAI SynthID-positive after pixel-identical
|
||||
metadata removal. The complete `qwen-zimage` profile at 0.07675, seed 0, cleared
|
||||
both in the official OpenAI Content Provenance API. The Flare output measured
|
||||
37.48 dB PSNR / 0.9329 SSIM and the Sunburst output 37.01 dB / 0.9179 against
|
||||
their sources. Those fidelity numbers describe two neutral studio-photo
|
||||
carriers, not a content-wide quality estimate, and the source-positive/output-negative
|
||||
oracle pairs do not replace the wider OpenAI calibration that set the floor.
|
||||
|
||||
| oracle | fixture size | detected at | clean from |
|
||||
|---|---|---|---|
|
||||
| openai.com/verify | 1.57 MP | 0.06 | 0.08 |
|
||||
|
||||
+23
-6
@@ -6,12 +6,29 @@ and pipeline modules are intended for maintainers and specialized workflows.
|
||||
Dependency groups are identical for the CLI and Python API. The default install
|
||||
covers metadata extraction, normalization, verdict logic, and stripping.
|
||||
Array/pixel APIs use `pixels`; visible removal uses `visible`; DWT-DCT detection
|
||||
uses `detect`; pixel photo classification uses `classify`; invisible image removal uses `qwen-zimage` and an NVIDIA GPU; and
|
||||
uses `detect`; pixel photo classification uses `classify`; lightweight source-pipeline classification uses `source-classify`; invisible image removal uses `qwen-zimage` and an NVIDIA GPU; and
|
||||
visible video processing uses `video`. The video-pixel SynthID-removal profile is
|
||||
a separate VAE path that still runs on CPU and combines `video` and `diffusion`. Add `heif`
|
||||
independently when path-based pixel APIs must decode HEIC, HEIF, or AVIF. See
|
||||
the complete [feature-extra matrix](installation.md#feature-extras).
|
||||
|
||||
## Classify a source/export pipeline
|
||||
|
||||
Install `remove-ai-watermarks[source-classify]`, then call the explicit API:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("metadata-stripped.png")
|
||||
print(result.label) # "openai" | "google" | "unknown"
|
||||
print(result.reason) # "classified" | "abstained" | "feature_unavailable"
|
||||
```
|
||||
|
||||
This signal classifies a complete image-generation and export pipeline. It is
|
||||
not a SynthID detector and does not establish watermark presence or absence.
|
||||
See [source-pipeline classification](source-classify.md) for the result fields,
|
||||
model pin, evaluation boundary, and offline weights override.
|
||||
|
||||
## Remove visible marks
|
||||
|
||||
Install `remove-ai-watermarks[visible]` before using the visible-removal API.
|
||||
@@ -780,11 +797,11 @@ engine.remove_watermark(
|
||||
|
||||
Install `remove-ai-watermarks[text-restoration]`. The manifest schema and safety
|
||||
constraints are documented in the CLI guide. The engine verifies its decoded RGB
|
||||
hash before loading the diffusion models. Qwen and Chroma reconstruct the donor with
|
||||
the VAE already loaded for the selected profile; no second generative profile runs.
|
||||
The engine rejects SDXL, including `auto` when Google provenance selects it,
|
||||
downscaling, and postprocessing combinations that were not evaluated. Tiling is
|
||||
also rejected because the combined
|
||||
hash before loading the diffusion models. Qwen, SDXL, and Chroma reconstruct the
|
||||
donor with the VAE already loaded for the selected profile; no second generative
|
||||
profile runs. The engine turns an unset `adaptive_polish` off in verified-text mode
|
||||
and rejects explicit postprocessing or downscaling combinations that were not
|
||||
evaluated. Tiling is also rejected because the combined
|
||||
tiled-restoration path has no provider-oracle calibration. `InvisibleOptions` exposes
|
||||
the same field for `remove_all`; after a visible-stage edit, the manifest must be built
|
||||
against the staged pixels rather than the pristine source.
|
||||
|
||||
@@ -92,7 +92,17 @@ alone does not upload new heads -- bump the pinned Hub revision when a new
|
||||
head must reach installed clients. Pin a Hub revision in production so a
|
||||
later freeze cannot silently change installed clients.
|
||||
|
||||
The workflow can also be started manually with an optional version input.
|
||||
The separate `wiltodelta/raiw-source-classify` model is published with
|
||||
`.github/workflows/publish-source-classify-hf.yml`. The workflow downloads the
|
||||
derivative `source-pipeline-mlp.npz` artifact from the declared GitHub model
|
||||
freeze, verifies its SHA-256, and uploads only that file plus the tracked card
|
||||
and aggregate metrics under `docs/source-classify-hf/`. The private corpus,
|
||||
catalog, paths, hashes, embeddings, and feature caches are outside this flow.
|
||||
|
||||
The workflow is manual and requires the model-freeze release tag plus an
|
||||
explicit `publish` confirmation. Keep model-freeze GitHub releases marked as
|
||||
pre-releases so a model artifact cannot replace the latest package release in
|
||||
GitHub's release UI.
|
||||
|
||||
If a distribution job fails because a repository or Hugging Face credential is
|
||||
invalid, rotate the corresponding GitHub secret and rerun the failed job. A
|
||||
@@ -255,13 +265,15 @@ lists it: uv serves its cached pre-release view of the index. Re-run with a
|
||||
fresh cache (`UV_CACHE_DIR=$(mktemp -d)`) before treating it as a release
|
||||
failure.
|
||||
|
||||
The ComfyUI sync run can fail the same way on its own: it resolves the version
|
||||
from the PyPI JSON API (which updates first) but installs the test dependency
|
||||
with pip against the simple index, whose CDN edges lag by minutes. The
|
||||
`distribute.yml` waiter also has a clock: 0.40.0's node suite finished after
|
||||
the 15-minute poll, so the job now waits 30 minutes and then accepts a
|
||||
registry row that already requires this library version. Rerun the failed
|
||||
`comfyui` job only when the registry still has no such row.
|
||||
The ComfyUI sync resolves the version from the PyPI JSON API but installs its
|
||||
test dependency through pip's simple index, whose CDN edges can lag by
|
||||
minutes. `distribute.yml` gates every downstream job on both APIs listing the
|
||||
release, so it cannot dispatch that race again. The ComfyUI waiter itself has
|
||||
a clock: 0.40.0's node suite finished after the 15-minute poll, so it now
|
||||
waits 30 minutes and then accepts a registry row that already requires this
|
||||
library version. For an older distribution that started before the simple
|
||||
index caught up, rerun its failed `comfyui` job only when the registry still
|
||||
has no such row.
|
||||
|
||||
The automatic `verify-release.yml` run after `distribute.yml` checks the
|
||||
Hugging Face Space pin. When `RAIW_HF_SPACE_TOKEN` is set, that pin lands
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
library_name: numpy
|
||||
pipeline_tag: image-classification
|
||||
tags:
|
||||
- image-forensics
|
||||
- source-attribution
|
||||
- open-set-classification
|
||||
---
|
||||
|
||||
# RAIW source-pipeline classifier
|
||||
|
||||
This is a compact, abstaining pixel-forensics classifier for three labels:
|
||||
`openai`, `google`, and `unknown`. It is intended for original-export-looking
|
||||
images whose metadata may have been removed without changing the pixels.
|
||||
|
||||
This model is **not a SynthID detector**. It does not decode or verify a
|
||||
watermark payload. Its labels describe complete generation and export
|
||||
pipelines, so renderer, geometry, encoding, post-processing, and watermark
|
||||
state may be confounded. `unknown` does not mean that SynthID is absent, and a
|
||||
provider label does not prove that SynthID is present.
|
||||
|
||||
## Model
|
||||
|
||||
- Input: a 124-dimensional residual/spectral forensic feature vector plus a
|
||||
768-dimensional phase-free spectral vector.
|
||||
- Head: 892 -> 128 -> 64 -> 3 MLP with GELU activations.
|
||||
- Output: `openai`, `google`, or abstain as `unknown` using independent
|
||||
provider margins.
|
||||
- Artifact: `source-pipeline-mlp.npz`
|
||||
- SHA-256: `6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a`
|
||||
|
||||
The artifact contains only learned parameters, normalization constants,
|
||||
labels, and thresholds. It does not contain training images, paths, image
|
||||
hashes, embeddings, or a training catalog.
|
||||
|
||||
## Evaluation
|
||||
|
||||
The operating point was selected on an internal calibration split. The locked
|
||||
internal test and the public `bigdatamark/synthid-research` dataset were not
|
||||
used for architecture, checkpoint, or threshold selection.
|
||||
|
||||
| Evaluation | Class | Recall | Precision |
|
||||
|---|---|---:|---:|
|
||||
| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% |
|
||||
| Locked internal test (5,369 images) | Google | 57.6% | 87.9% |
|
||||
| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% |
|
||||
| Public external originals (900 images) | OpenAI | 68.7% | 100.0% |
|
||||
| Public external originals (900 images) | Google | 81.3% | 98.8% |
|
||||
| Public external originals (900 images) | Unknown/Flux | 99.0% | 66.4% |
|
||||
|
||||
On the public external set, a 95% resize reduced Google recall to 1.0%, and a
|
||||
JPEG quality-75 round trip caused the model to abstain on every image. These
|
||||
are expected abstentions, not evidence that a watermark was removed.
|
||||
|
||||
## Intended use
|
||||
|
||||
Use this model as one probabilistic signal in a broader forensic report or to
|
||||
suggest an appropriate processing path. Do not use it to claim watermark
|
||||
presence, authorship, policy compliance, fraud, or provenance. Applications
|
||||
should expose the abstention and the model's narrow source-pipeline claim.
|
||||
|
||||
Feature extraction and inference are versioned in the
|
||||
`remove-ai-watermarks` Python project rather than in this model repository.
|
||||
Use its explicit `classify_source` API; `identify` does not run this model.
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"claim": "abstaining OpenAI/Google source-pipeline classification; not SynthID detection",
|
||||
"artifact_sha256": "6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a",
|
||||
"classes": ["openai", "google", "unknown"],
|
||||
"locked_internal_test": {
|
||||
"n": 5369,
|
||||
"confusion_actual_by_predicted": [[230, 6, 145], [0, 217, 160], [41, 24, 4546]],
|
||||
"recall": {"openai": 0.6036745407, "google": 0.5755968170, "unknown": 0.9859032748},
|
||||
"precision": {"openai": 0.8487084871, "google": 0.8785425101, "unknown": 0.9371263657}
|
||||
},
|
||||
"public_external_originals": {
|
||||
"dataset": "bigdatamark/synthid-research",
|
||||
"n": 900,
|
||||
"confusion_actual_by_predicted": [[206, 0, 94], [0, 244, 56], [0, 3, 297]],
|
||||
"recall": {"openai": 0.6866666667, "google": 0.8133333333, "unknown": 0.99},
|
||||
"precision": {"openai": 1.0, "google": 0.9878542510, "unknown": 0.6644295302}
|
||||
},
|
||||
"selection": {
|
||||
"minimum_calibration_precision": {"openai": 0.99, "google": 0.95},
|
||||
"maximum_calibration_unknown_false_positive_rate": 0.01
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# Source-pipeline classification
|
||||
|
||||
`classify_source` is an opt-in, metadata-free pixel classifier for images that
|
||||
still resemble an original provider export. It returns `openai`, `google`, or
|
||||
an explicit `unknown` abstention.
|
||||
|
||||
This is not a SynthID detector. It does not decode or verify a watermark
|
||||
payload, and it must not be used to claim that SynthID is present or absent.
|
||||
The model describes the complete source/export pipeline, where renderer,
|
||||
geometry, encoding, post-processing, and watermark state can be confounded.
|
||||
|
||||
Install the lightweight runtime:
|
||||
|
||||
```bash
|
||||
uv add "remove-ai-watermarks[source-classify]"
|
||||
```
|
||||
|
||||
Call the Python API explicitly:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("metadata-stripped.png")
|
||||
print(result.label) # "openai" | "google" | "unknown"
|
||||
print(result.reason) # "classified" | "abstained" | "feature_unavailable"
|
||||
print(result.margins) # provider margins before thresholding
|
||||
print(result.thresholds) # frozen provider thresholds
|
||||
```
|
||||
|
||||
`feature_unavailable` is returned for images below the required 256x256
|
||||
geometry or for a degenerate feature vector. `abstained` means neither provider
|
||||
cleared its independent calibrated margin. Both are honest `unknown` results.
|
||||
|
||||
The first call downloads the exact pinned revision of
|
||||
[`wiltodelta/raiw-source-classify`](https://huggingface.co/wiltodelta/raiw-source-classify).
|
||||
Set `RAIW_SOURCE_CLASSIFY_WEIGHTS` to the NPZ itself or a directory containing
|
||||
`source-pipeline-mlp.npz` for an offline deployment.
|
||||
|
||||
The model is useful as a routing suggestion for a processing workflow or as one
|
||||
probabilistic signal in a broader forensic report. Do not use it as evidence of
|
||||
authorship, fraud, policy compliance, provenance, or watermark presence. The
|
||||
published model card records the evaluation and robustness limits.
|
||||
@@ -121,6 +121,10 @@ entries enter the watermark inventory, while a fingerprint remains a
|
||||
re-linkability warning, never selects pixel regeneration, and does not suppress
|
||||
independent SynthID evidence.
|
||||
|
||||
The opt-in `classify_source` API is not an `identify` signal. It can suggest an
|
||||
OpenAI or Google source/export pipeline after metadata removal, but it neither
|
||||
detects nor decodes a watermark and never changes the provenance verdict.
|
||||
|
||||
## File and container formats
|
||||
|
||||
Pixel based image commands discover these extensions:
|
||||
@@ -171,7 +175,11 @@ and `scripts/synthid_runtime/`.
|
||||
The tool recognizes presence from supported provenance: Google AI C2PA
|
||||
under Google's all-media watermark policy, and current OpenAI C2PA carrying an
|
||||
explicit `c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action
|
||||
does not assert SynthID.
|
||||
does not assert SynthID. GPT Image 2.5 Flare and Sunburst were verified on
|
||||
2026-09-15: both API outputs carried valid OpenAI C2PA with
|
||||
`c2pa.watermarked.unbound` and exercised the existing model-independent OpenAI
|
||||
route. Automatic routing selected `qwen-zimage` with the OpenAI strength cohort,
|
||||
and both outputs cleared the official OpenAI SynthID check.
|
||||
|
||||
Provider verifiers and provenance APIs are reserved for development-time
|
||||
calibration. They are not exposed through the installed CLI, the top-level
|
||||
|
||||
@@ -62,6 +62,91 @@ support treating augmentation and architecture shift as first-class tests,
|
||||
rather than reading an untransformed source-classification score as watermark
|
||||
evidence.
|
||||
|
||||
### Abstaining fused source classifier, 2026-09-15
|
||||
|
||||
The follow-up experiment combined the existing 124-d forensic descriptor with
|
||||
the 16x16 phase-free spectral grid in a class-balanced ridge head. Its three
|
||||
outputs are `openai`, `google`, and `unknown`; a provider result is emitted only
|
||||
when it beats both the other provider and `unknown` by a margin selected on a
|
||||
separate calibration split. The public implementation is
|
||||
[`scripts/source_pipeline_classifier.py`](../scripts/source_pipeline_classifier.py).
|
||||
It includes training, strict pickle-free model loading, and JSON inference, but
|
||||
no fitted weights: the experimental training freeze contains inputs that are
|
||||
not redistributable. This keeps the code reproducible without publishing a
|
||||
derivative artifact whose data provenance does not permit redistribution.
|
||||
|
||||
Penalty 100 and margin 0.6392407698660646 were selected using only the training
|
||||
and calibration partitions. The 2,669-image locked internal test and Mark's
|
||||
900-image public set were not used for feature, penalty, or threshold selection.
|
||||
Feature-extraction failures count as `unknown` in every result below.
|
||||
|
||||
| Evaluation | Accuracy | OpenAI recall | Google recall | Unknown recall |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Locked internal, 2,669 images | 84.2% | 60.9% | 48.8% | 95.8% |
|
||||
| Mark original exports, 900 images | 90.1% | 76.3% | 94.3% | 99.7% |
|
||||
| Mark crop 2 px | 89.8% | 76.7% | 94.3% | 98.3% |
|
||||
| Mark resize 95% | 59.7% | 81.7% | 0.0% | 97.3% |
|
||||
| Mark JPEG quality 75 | 33.3% | 0.0% | 0.0% | 100.0% |
|
||||
| Mark blur radius 0.7 | 68.3% | 42.3% | 63.0% | 99.7% |
|
||||
|
||||
The untouched-export result is useful source-attribution evidence, but the
|
||||
transforms decide the claim. Resizing eliminated the Google branch, JPEG made
|
||||
the classifier abstain on every row, and blur materially reduced both provider
|
||||
recalls. The locked unknown false-attribution rate was 4.2%, versus 0.9% on the
|
||||
calibration split. This is therefore an inspectable research baseline, not a
|
||||
runtime detector or a product watermark verdict.
|
||||
|
||||
Training requires hash-disjoint `train` and `calibration` directories, each
|
||||
with `openai`, `google`, and `unknown` subdirectories:
|
||||
|
||||
```bash
|
||||
uv run python scripts/source_pipeline_classifier.py train \
|
||||
/path/to/train /path/to/calibration --model-out .local-eval/source-pipeline.npz
|
||||
|
||||
uv run python scripts/source_pipeline_classifier.py predict \
|
||||
.local-eval/source-pipeline.npz image.png
|
||||
```
|
||||
|
||||
A stronger public release needs a publication-cleared multi-era training set
|
||||
and a causal marked/unmarked contrast. Mark's same-prompt triples remain an
|
||||
external source/export test, not evidence that the fitted features read the
|
||||
SynthID payload.
|
||||
|
||||
### Full-corpus source-pipeline model, 2026-09-16
|
||||
|
||||
The next experiment trained a compact nonlinear head on the complete available
|
||||
private corpus, including target-provider exports, photographs, and foreign
|
||||
generator negatives. The corpus itself is not published. The released
|
||||
`source-pipeline-mlp.npz` is a derivative artifact containing only learned
|
||||
parameters, normalization constants, labels, and abstention thresholds; it
|
||||
contains no images, paths, image hashes, embeddings, or training catalog.
|
||||
|
||||
The model joins the 124-dimensional forensic descriptor with the
|
||||
768-dimensional phase-free spectral descriptor and applies a
|
||||
892 -> 128 -> 64 -> 3 GELU MLP. Checkpoint and thresholds were selected on a
|
||||
separate calibration partition. The 5,369-image hash-disjoint locked test and
|
||||
Mark's public 900-image originals were not used for architecture, checkpoint,
|
||||
or threshold selection.
|
||||
|
||||
| Evaluation | Class | Recall | Precision |
|
||||
| --- | --- | ---: | ---: |
|
||||
| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% |
|
||||
| Locked internal test (5,369 images) | Google | 57.6% | 87.9% |
|
||||
| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% |
|
||||
| Mark public originals (900 images) | OpenAI | 68.7% | 100.0% |
|
||||
| Mark public originals (900 images) | Google | 81.3% | 98.8% |
|
||||
| Mark public originals (900 images) | Unknown/FLUX | 99.0% | 66.4% |
|
||||
|
||||
A 95% resize reduced Google recall on the public set to 1%, and a JPEG
|
||||
quality-75 round trip made every row abstain. This behavior matches the narrow
|
||||
claim: the model classifies original-looking source/export pipelines after
|
||||
metadata removal. It neither decodes nor detects SynthID, and its `unknown`
|
||||
label does not mean that SynthID is absent. The model card and aggregate
|
||||
metrics are tracked under
|
||||
[`docs/source-classify-hf/`](source-classify-hf/README.md). Runtime integration
|
||||
is provided by the explicit `classify_source` library API and never feeds the
|
||||
`identify` watermark verdict.
|
||||
|
||||
Krawetz's Gemini-chat TPR critique is a verifier-quality claim, not a
|
||||
feature we can ship. [Lead Stories, 2026-07](https://leadstories.com/analysis/2026/07/google-gemini-synthid-detector-confuses-results-within-same-chat.html)
|
||||
documented Gemini repeating the first file's SynthID verdict inside a
|
||||
|
||||
@@ -775,6 +775,20 @@ SynthID. Current OpenAI manifests distinguish the watermarked output with the
|
||||
signed `c2pa.watermarked.unbound` action. The detector requires that action, so
|
||||
old OpenAI C2PA remains an origin signal without becoming a SynthID claim.
|
||||
|
||||
**GPT Image 2.5 compatibility, verified 2026-09-15.** One fresh low-quality
|
||||
1024 x 1024 API generation from each released model, `gpt-image-2.5-flare` and
|
||||
`gpt-image-2.5-sunburst`, carried a valid bound OpenAI C2PA claim generated by
|
||||
`OpenAI Media Service API`, with `trainedAlgorithmicMedia` and the action set
|
||||
`created, converted, watermarked.unbound`. `identify` reported high-confidence
|
||||
OpenAI provenance plus SynthID, `has_invisible_target` returned true, and the
|
||||
automatic strength cohort resolved to OpenAI for both files. After C2PA was
|
||||
removed without changing decoded pixels, the official OpenAI Content Provenance
|
||||
API still returned SynthID `detected` for both sources. The complete shipped
|
||||
`qwen-zimage` profile at strength 0.07675 and seed 0 then produced two outputs
|
||||
for which the same metadata-independent API returned SynthID `not_detected`.
|
||||
This is one controlled source per model and proves the current routing and
|
||||
operating point on those two sources; it is not a provider-wide removal rate.
|
||||
|
||||
### 4.3 Other vendors
|
||||
|
||||
- **Kakao** (South Korea): SynthID adopter as of May 2026 (Google announcement)
|
||||
|
||||
+9
-2
@@ -4,7 +4,7 @@
|
||||
# on it, including the ComfyUI node package. The console script below carries
|
||||
# the same weight, since users have it on PATH.
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.4"
|
||||
version = "0.41.0"
|
||||
description = "AI watermark remover for visible, invisible, and provenance marks in images and video"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11,<3.15"
|
||||
@@ -160,6 +160,13 @@ classify-onnx = [
|
||||
"remove-ai-watermarks[classify]",
|
||||
"onnxruntime>=1.24.0",
|
||||
]
|
||||
# Compact OpenAI/Google/unknown source-export classifier. This is intentionally
|
||||
# separate from the heavier photo AI-versus-camera classifier and is not a
|
||||
# SynthID detector. Weights download on first use.
|
||||
source-classify = [
|
||||
"remove-ai-watermarks[pixels]",
|
||||
"huggingface-hub>=0.20.0",
|
||||
]
|
||||
# Universal region eraser backend -- big-LaMa via onnxruntime (Carve/LaMa-ONNX,
|
||||
# Apache-2.0). CPU, no torch. Model (~200 MB) is downloaded on first use and
|
||||
# cached by huggingface_hub; it is never bundled in this repo. The default cv2
|
||||
@@ -205,7 +212,7 @@ dev = [
|
||||
]
|
||||
# ``qwen-zimage`` already pulls ``diffusion``; naming both would suggest diffusion is
|
||||
# independently sufficient for a removal, which it is not.
|
||||
all = ["remove-ai-watermarks[video,heif,detect,trustmark,text-restoration,migan,classify]"]
|
||||
all = ["remove-ai-watermarks[video,heif,detect,trustmark,text-restoration,migan,classify,source-classify]"]
|
||||
|
||||
[project.scripts]
|
||||
remove-ai-watermarks = "remove_ai_watermarks.cli:main"
|
||||
|
||||
@@ -51,9 +51,11 @@ explicitly names a tracked canonical result.
|
||||
| `visible_sheets.py` | Build blinded contact sheets for relaxation candidates. |
|
||||
| `watermark_benchmark_cohort.py` | Build the deterministic synthetic DWT-DCT and TrustMark cohort consumed by `watermark_benchmark.py`. |
|
||||
| `spectral_pipeline_probe.py` | Measure a prompt-grouped, phase-free spectral source-pipeline hypothesis without presenting it as SynthID detection. |
|
||||
| `source_pipeline_classifier.py` | Train or run an abstaining OpenAI/Google/unknown source-export classifier; research-only and not SynthID detection. |
|
||||
| `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. |
|
||||
| `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. |
|
||||
| `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. |
|
||||
| `publish_source_classify_hf.py` | Upload the derivative source-pipeline model and its public card to Hugging Face `wiltodelta/raiw-source-classify`. Manual; the Action `publish-source-classify-hf.yml` is the write-token path. |
|
||||
| `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. |
|
||||
|
||||
## Research and diagnostic prototypes
|
||||
@@ -86,5 +88,7 @@ running the sync command without `--check` is the explicit snapshot update.
|
||||
|
||||
`_plain_console.py` provides plain-text fallbacks for Rich output, and
|
||||
`_text_eval.py` contains normalization helpers shared by text-evaluation scripts.
|
||||
`hf_publish.py` owns the common authenticated, allowlisted Hugging Face model
|
||||
repository upload used by the two model publishers.
|
||||
`engine_selection_manifest.py` owns the content-matrix schema and validation
|
||||
shared by its integrity check and the real-image benchmark builder.
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Shared Hugging Face model-repository publication helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def publish_model_folder(
|
||||
stage: Path,
|
||||
*,
|
||||
repo_id: str,
|
||||
token: str,
|
||||
message: str,
|
||||
allow_patterns: Iterable[str],
|
||||
) -> str:
|
||||
"""Upload an explicit file allowlist and return the Hub revision."""
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(repo_id, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
commit = HfApi(token=token).upload_folder(
|
||||
folder_path=str(stage),
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
allow_patterns=list(allow_patterns),
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
@@ -24,6 +24,8 @@ import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-photo-classify"
|
||||
@@ -92,18 +94,13 @@ def stage_weights(dest: Path, src: Path) -> None:
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(HUB_REPO, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
api = HfApi(token=token)
|
||||
commit = api.upload_folder(
|
||||
folder_path=str(stage),
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=["README.md", "operating-point.json", *WEIGHT_FILES, GATE_FILE, GATE_STABLE_FILE],
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish the source-pipeline model to Hugging Face.
|
||||
|
||||
The derivative weight file is downloaded by the GitHub Action from a model
|
||||
freeze release. The private corpus, feature caches, and training catalog never
|
||||
enter this repository or the Hub staging directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-source-classify"
|
||||
MODEL_FILE = "source-pipeline-mlp.npz"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
CARD_DIR = REPO_ROOT / "docs" / "source-classify-hf"
|
||||
PUBLIC_FILES = ("README.md", "metrics.json", MODEL_FILE)
|
||||
|
||||
|
||||
def stage_release(dest: Path, src: Path) -> None:
|
||||
"""Stage the two tracked documents and one derivative model artifact."""
|
||||
sources = {
|
||||
"README.md": CARD_DIR / "README.md",
|
||||
"metrics.json": CARD_DIR / "metrics.json",
|
||||
MODEL_FILE: src.expanduser().resolve() / MODEL_FILE,
|
||||
}
|
||||
missing = [str(path) for path in sources.values() if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit(f"missing publication files: {', '.join(missing)}")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for name, path in sources.items():
|
||||
shutil.copy2(path, dest / name)
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
"""Upload the exact staged allowlist and return the Hub revision."""
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=PUBLIC_FILES,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--src", type=Path, required=True)
|
||||
parser.add_argument("--message", default="Publish source-pipeline classifier")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
token = os.environ.get("HF_TOKEN", "").strip()
|
||||
if not token:
|
||||
raise SystemExit("HF_TOKEN is not set (write role required)")
|
||||
with tempfile.TemporaryDirectory(prefix="raiw-source-hf-") as tmp:
|
||||
stage = Path(tmp)
|
||||
stage_release(stage, args.src)
|
||||
revision = publish(stage, token=token, message=args.message)
|
||||
log.info("published %s revision=%s", HUB_REPO, revision or "(unknown)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,371 @@
|
||||
"""Train or run an abstaining source/export-pipeline classifier.
|
||||
|
||||
This research tool combines the library's 124-d forensic descriptor with the
|
||||
phase-free spectral descriptor from ``spectral_pipeline_probe.py``. Its labels
|
||||
describe complete generation and export pipelines. It does not decode SynthID
|
||||
or establish that a watermark is present.
|
||||
|
||||
Expected training layout::
|
||||
|
||||
train/openai/*.png
|
||||
train/google/*.png
|
||||
train/unknown/*.png
|
||||
calibration/openai/*.png
|
||||
calibration/google/*.png
|
||||
calibration/unknown/*.png
|
||||
|
||||
Examples::
|
||||
|
||||
uv run python scripts/source_pipeline_classifier.py train \
|
||||
train calibration --model-out model.npz
|
||||
uv run python scripts/source_pipeline_classifier.py predict \
|
||||
model.npz image.png
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import click
|
||||
import numpy as np
|
||||
import spectral_pipeline_probe as spectral
|
||||
from numpy.typing import NDArray
|
||||
from PIL import Image
|
||||
from watermark_benchmark import sha256_file
|
||||
|
||||
from remove_ai_watermarks._internal.forensic_124d import FEATURE_WIDTH, PATCH, image_features
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CLASSES = ("openai", "google", "unknown")
|
||||
UNKNOWN_INDEX = CLASSES.index("unknown")
|
||||
DEFAULT_GRID = 16
|
||||
DEFAULT_PENALTY = 100.0
|
||||
DEFAULT_UNKNOWN_FPR = 0.01
|
||||
CLAIM = "source/export pipeline classification; not SynthID detection"
|
||||
|
||||
FloatArray = NDArray[np.float64]
|
||||
IntArray = NDArray[np.int64]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RidgeModel:
|
||||
"""One standardized three-class ridge head with an abstention margin."""
|
||||
|
||||
mean: FloatArray
|
||||
scale: FloatArray
|
||||
weights: FloatArray
|
||||
margin: float
|
||||
grid: int
|
||||
penalty: float
|
||||
|
||||
|
||||
def discover_labeled_paths(root: Path) -> tuple[list[Path], IntArray, list[str]]:
|
||||
"""Find images under the three fixed label directories."""
|
||||
paths: list[Path] = []
|
||||
labels: list[int] = []
|
||||
hashes: list[str] = []
|
||||
for class_index, class_name in enumerate(CLASSES):
|
||||
directory = root / class_name
|
||||
if not directory.is_dir():
|
||||
raise ValueError(f"class directory does not exist: {directory}")
|
||||
class_paths = sorted(
|
||||
path for path in directory.rglob("*") if path.is_file() and path.suffix.lower() in spectral.IMAGE_SUFFIXES
|
||||
)
|
||||
if not class_paths:
|
||||
raise ValueError(f"class directory has no supported images: {directory}")
|
||||
for path in class_paths:
|
||||
paths.append(path)
|
||||
labels.append(class_index)
|
||||
hashes.append(sha256_file(path))
|
||||
if len(hashes) != len(set(hashes)):
|
||||
raise ValueError(f"duplicate image bytes found inside {root}")
|
||||
return paths, np.asarray(labels, dtype=np.int64), hashes
|
||||
|
||||
|
||||
def _feature_from_path(path: Path, geometry: spectral.SpectralGeometry) -> FloatArray | None:
|
||||
"""Return the fused representation for one path and prepared geometry."""
|
||||
with Image.open(path) as source:
|
||||
image = source.convert("RGB")
|
||||
forensic = image_features(np.asarray(image, dtype=np.uint8))
|
||||
if forensic is None:
|
||||
return None
|
||||
phase_free = spectral.feature_from_image(image, geometry, "original")
|
||||
return np.concatenate(
|
||||
(
|
||||
np.asarray(forensic, dtype=np.float64),
|
||||
np.asarray(phase_free, dtype=np.float64),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def feature_from_path(path: Path, grid: int) -> FloatArray | None:
|
||||
"""Return the fused forensic and phase-free spectral representation."""
|
||||
return _feature_from_path(path, spectral.make_geometry(size=PATCH, grid=grid))
|
||||
|
||||
|
||||
def extract_features(paths: list[Path], labels: IntArray, grid: int) -> tuple[FloatArray, IntArray, list[Path]]:
|
||||
"""Extract valid rows and report files whose forensic feature abstains."""
|
||||
geometry = spectral.make_geometry(size=PATCH, grid=grid)
|
||||
features: list[FloatArray] = []
|
||||
kept_labels: list[int] = []
|
||||
skipped: list[Path] = []
|
||||
for index, (path, label) in enumerate(zip(paths, labels, strict=True), start=1):
|
||||
try:
|
||||
feature = _feature_from_path(path, geometry)
|
||||
except (OSError, ValueError) as error:
|
||||
log.warning("Could not extract %s: %s", path, error)
|
||||
feature = None
|
||||
if feature is None:
|
||||
skipped.append(path)
|
||||
continue
|
||||
features.append(feature)
|
||||
kept_labels.append(int(label))
|
||||
if index % 100 == 0:
|
||||
log.info("Extracted %s/%s images", index, len(paths))
|
||||
if not features:
|
||||
raise ValueError("no image produced a valid feature vector")
|
||||
return np.asarray(features), np.asarray(kept_labels, dtype=np.int64), skipped
|
||||
|
||||
|
||||
def fit_ridge(
|
||||
features: FloatArray,
|
||||
labels: IntArray,
|
||||
penalty: float,
|
||||
grid: int = DEFAULT_GRID,
|
||||
) -> RidgeModel:
|
||||
"""Fit a class-balanced standardized ridge head before calibration."""
|
||||
if features.ndim != 2 or len(features) != len(labels):
|
||||
raise ValueError("features and labels must describe the same rows")
|
||||
counts = np.bincount(labels, minlength=len(CLASSES)).astype(np.float64)
|
||||
if np.any(counts == 0):
|
||||
raise ValueError("training data must contain all three classes")
|
||||
if penalty <= 0:
|
||||
raise ValueError("penalty must be positive")
|
||||
mean = features.mean(axis=0)
|
||||
raw_scale = features.std(axis=0)
|
||||
active = raw_scale > 1e-10
|
||||
scale = np.where(active, raw_scale, 1.0)
|
||||
normalized = (features[:, active] - mean[active]) / scale[active]
|
||||
sample_weight = len(labels) / (len(CLASSES) * counts[labels])
|
||||
root_weight = np.sqrt(sample_weight)[:, None]
|
||||
weighted = normalized * root_weight
|
||||
targets = np.eye(len(CLASSES), dtype=np.float64)[labels] * root_weight
|
||||
gram = weighted.T @ weighted + penalty * np.eye(weighted.shape[1])
|
||||
active_weights = cast("FloatArray", np.linalg.solve(gram, weighted.T @ targets))
|
||||
weights = np.zeros((features.shape[1], len(CLASSES)), dtype=np.float64)
|
||||
weights[active] = active_weights
|
||||
return RidgeModel(mean, scale, weights, 0.0, grid, penalty)
|
||||
|
||||
|
||||
def scores(features: FloatArray, model: RidgeModel) -> FloatArray:
|
||||
"""Return the uncalibrated ridge scores."""
|
||||
return ((features - model.mean) / model.scale) @ model.weights
|
||||
|
||||
|
||||
def decision_margins(raw_scores: FloatArray) -> FloatArray:
|
||||
"""Return each row's best-provider advantage over every alternative."""
|
||||
target_scores = raw_scores[:, :UNKNOWN_INDEX]
|
||||
best = np.argmax(target_scores, axis=1)
|
||||
other = 1 - best
|
||||
rows = np.arange(len(raw_scores))
|
||||
reference = np.maximum(raw_scores[rows, other], raw_scores[:, UNKNOWN_INDEX])
|
||||
return target_scores[rows, best] - reference
|
||||
|
||||
|
||||
def calibrate_margin(
|
||||
calibration_scores: FloatArray,
|
||||
calibration_labels: IntArray,
|
||||
unknown_fpr: float,
|
||||
) -> float:
|
||||
"""Choose the smallest empirical cut meeting the unknown false-positive target."""
|
||||
if not 0 <= unknown_fpr < 1:
|
||||
raise ValueError("unknown_fpr must be in [0, 1)")
|
||||
unknown_scores = calibration_scores[calibration_labels == UNKNOWN_INDEX]
|
||||
if not len(unknown_scores):
|
||||
raise ValueError("calibration data must contain unknown rows")
|
||||
return max(
|
||||
0.0,
|
||||
float(
|
||||
np.quantile(
|
||||
decision_margins(unknown_scores),
|
||||
1.0 - unknown_fpr,
|
||||
method="higher",
|
||||
)
|
||||
+ 1e-12
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def predict_scores(raw_scores: FloatArray, margin: float) -> IntArray:
|
||||
"""Return provider labels only when the calibrated margin is met."""
|
||||
best = np.argmax(raw_scores[:, :UNKNOWN_INDEX], axis=1)
|
||||
accepted = decision_margins(raw_scores) >= margin
|
||||
return np.where(accepted, best, UNKNOWN_INDEX).astype(np.int64)
|
||||
|
||||
|
||||
def save_model(path: Path, model: RidgeModel) -> None:
|
||||
"""Write a pickle-free model artifact."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
np.savez_compressed(
|
||||
path,
|
||||
classes=np.asarray(CLASSES),
|
||||
mean=model.mean,
|
||||
scale=model.scale,
|
||||
weights=model.weights,
|
||||
margin=np.asarray(model.margin),
|
||||
spectral_grid=np.asarray(model.grid),
|
||||
penalty=np.asarray(model.penalty),
|
||||
)
|
||||
|
||||
|
||||
def load_model(path: Path) -> RidgeModel:
|
||||
"""Load and validate a pickle-free model artifact."""
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
required = {
|
||||
"classes",
|
||||
"mean",
|
||||
"scale",
|
||||
"weights",
|
||||
"margin",
|
||||
"spectral_grid",
|
||||
"penalty",
|
||||
}
|
||||
if set(payload.files) != required:
|
||||
raise ValueError(f"model fields must be exactly {sorted(required)}")
|
||||
classes = tuple(str(value) for value in payload["classes"])
|
||||
mean = np.asarray(payload["mean"], dtype=np.float64)
|
||||
scale = np.asarray(payload["scale"], dtype=np.float64)
|
||||
weights = np.asarray(payload["weights"], dtype=np.float64)
|
||||
margin = float(payload["margin"])
|
||||
grid = int(payload["spectral_grid"])
|
||||
penalty = float(payload["penalty"])
|
||||
expected_features = FEATURE_WIDTH + 3 * grid * grid
|
||||
if classes != CLASSES:
|
||||
raise ValueError(f"model classes must be {CLASSES}")
|
||||
if mean.shape != (expected_features,) or scale.shape != mean.shape:
|
||||
raise ValueError("model normalization vectors have the wrong shape")
|
||||
if weights.shape != (expected_features, len(CLASSES)):
|
||||
raise ValueError("model weights have the wrong shape")
|
||||
if not np.all(np.isfinite(mean)) or not np.all(np.isfinite(scale)):
|
||||
raise ValueError("model normalization vectors must be finite")
|
||||
if not np.all(np.isfinite(weights)) or np.any(scale <= 0):
|
||||
raise ValueError("model weights must be finite and scales positive")
|
||||
if not np.isfinite(margin) or margin < 0:
|
||||
raise ValueError("model margin must be finite and non-negative")
|
||||
if grid < 2 or PATCH % grid or not np.isfinite(penalty) or penalty <= 0:
|
||||
raise ValueError("model calibration fields are invalid")
|
||||
return RidgeModel(mean, scale, weights, margin, grid, penalty)
|
||||
|
||||
|
||||
@click.group()
|
||||
def main() -> None:
|
||||
"""Train or run the research-only source-pipeline classifier."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
|
||||
|
||||
@main.command("train")
|
||||
@click.argument("train_root", type=click.Path(exists=True, file_okay=False, path_type=Path))
|
||||
@click.argument("calibration_root", type=click.Path(exists=True, file_okay=False, path_type=Path))
|
||||
@click.option("--model-out", type=click.Path(dir_okay=False, path_type=Path), required=True)
|
||||
@click.option("--grid", type=click.IntRange(min=2), default=DEFAULT_GRID, show_default=True)
|
||||
@click.option("--penalty", type=click.FloatRange(min=0.0, min_open=True), default=DEFAULT_PENALTY, show_default=True)
|
||||
@click.option(
|
||||
"--unknown-fpr",
|
||||
type=click.FloatRange(min=0.0, max=1.0, max_open=True),
|
||||
default=DEFAULT_UNKNOWN_FPR,
|
||||
show_default=True,
|
||||
)
|
||||
def train_command(
|
||||
train_root: Path,
|
||||
calibration_root: Path,
|
||||
model_out: Path,
|
||||
grid: int,
|
||||
penalty: float,
|
||||
unknown_fpr: float,
|
||||
) -> None:
|
||||
"""Fit on TRAIN_ROOT and calibrate abstention on CALIBRATION_ROOT."""
|
||||
if PATCH % grid:
|
||||
raise click.UsageError(f"--grid must divide {PATCH}")
|
||||
try:
|
||||
train_paths, train_labels, train_hashes = discover_labeled_paths(train_root)
|
||||
calibration_paths, calibration_labels, calibration_hashes = discover_labeled_paths(calibration_root)
|
||||
overlap = set(train_hashes) & set(calibration_hashes)
|
||||
if overlap:
|
||||
raise ValueError(f"train/calibration byte overlap: {len(overlap)} images")
|
||||
train_features, kept_train_labels, train_skipped = extract_features(train_paths, train_labels, grid)
|
||||
calibration_features, kept_calibration_labels, calibration_skipped = extract_features(
|
||||
calibration_paths, calibration_labels, grid
|
||||
)
|
||||
provisional = fit_ridge(train_features, kept_train_labels, penalty, grid)
|
||||
margin = calibrate_margin(
|
||||
scores(calibration_features, provisional),
|
||||
kept_calibration_labels,
|
||||
unknown_fpr,
|
||||
)
|
||||
model = replace(provisional, margin=margin)
|
||||
save_model(model_out, model)
|
||||
except ValueError as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
click.echo(
|
||||
json.dumps(
|
||||
{
|
||||
"model": str(model_out),
|
||||
"claim": CLAIM,
|
||||
"train_rows": len(train_features),
|
||||
"calibration_rows": len(calibration_features),
|
||||
"feature_abstentions": len(train_skipped) + len(calibration_skipped),
|
||||
"margin": margin,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@main.command("predict")
|
||||
@click.argument("model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
def predict_command(model_path: Path, images: tuple[Path, ...]) -> None:
|
||||
"""Classify original-export IMAGES with MODEL_PATH."""
|
||||
try:
|
||||
model = load_model(model_path)
|
||||
except (OSError, ValueError) as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
results: list[dict[str, object]] = []
|
||||
geometry = spectral.make_geometry(size=PATCH, grid=model.grid)
|
||||
for path in images:
|
||||
try:
|
||||
feature = _feature_from_path(path, geometry)
|
||||
except (OSError, ValueError) as error:
|
||||
raise click.ClickException(f"could not extract {path}: {error}") from error
|
||||
if feature is None:
|
||||
results.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"label": "unknown",
|
||||
"reason": "124-d feature unavailable",
|
||||
"claim": CLAIM,
|
||||
}
|
||||
)
|
||||
continue
|
||||
raw_scores = scores(feature[None, :], model)
|
||||
prediction = int(predict_scores(raw_scores, model.margin)[0])
|
||||
results.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"label": CLASSES[prediction],
|
||||
"decision_margin": float(decision_margins(raw_scores)[0]),
|
||||
"required_margin": model.margin,
|
||||
"scores": {name: float(raw_scores[0, index]) for index, name in enumerate(CLASSES)},
|
||||
"claim": CLAIM,
|
||||
}
|
||||
)
|
||||
click.echo(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -30,6 +30,14 @@ import numpy as np
|
||||
from numpy.typing import NDArray
|
||||
from PIL import Image, ImageFilter
|
||||
|
||||
from remove_ai_watermarks._internal import source_spectral as _spectral
|
||||
|
||||
SpectralGeometry = _spectral.SpectralGeometry
|
||||
make_geometry = _spectral.make_geometry
|
||||
phase_free_log_spectrum = _spectral.phase_free_log_spectrum
|
||||
spectral_feature = _spectral.spectral_feature
|
||||
_source_feature_from_image = _spectral.feature_from_image
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg", ".png", ".webp"})
|
||||
@@ -40,17 +48,6 @@ IntArray = NDArray[np.int64]
|
||||
BoolArray = NDArray[np.bool_]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpectralGeometry:
|
||||
"""Precomputed coordinates for one phase-free spectral representation."""
|
||||
|
||||
size: int
|
||||
grid: int
|
||||
radius: NDArray[np.int32]
|
||||
radius_counts: IntArray
|
||||
passband: BoolArray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FoldModel:
|
||||
"""One standardized dual-ridge model and its held-out rows."""
|
||||
@@ -62,51 +59,6 @@ class FoldModel:
|
||||
weights: FloatArray
|
||||
|
||||
|
||||
def make_geometry(size: int = 256, grid: int = 32) -> SpectralGeometry:
|
||||
"""Build the radial detrending and pooling geometry."""
|
||||
if size < 16 or grid < 2 or size % grid:
|
||||
raise ValueError("size must be at least 16 and divisible by grid")
|
||||
yy, xx = np.mgrid[:size, :size]
|
||||
radius = np.rint(np.hypot(yy - size // 2, xx - size // 2)).astype(np.int32)
|
||||
radius_counts = np.bincount(radius.ravel())
|
||||
passband = (radius >= 4) & (radius <= size * 0.48)
|
||||
return SpectralGeometry(size, grid, radius, radius_counts, passband)
|
||||
|
||||
|
||||
def phase_free_log_spectrum(channel: FloatArray) -> FloatArray:
|
||||
"""Return centered log FFT magnitude, deliberately discarding phase."""
|
||||
if channel.ndim != 2:
|
||||
raise ValueError("channel must be a two-dimensional array")
|
||||
centered = np.asarray(channel, dtype=np.float64) - float(np.mean(channel))
|
||||
return np.log1p(np.abs(np.fft.fftshift(np.fft.fft2(centered))))
|
||||
|
||||
|
||||
def spectral_feature(pixels: FloatArray, geometry: SpectralGeometry) -> FloatArray:
|
||||
"""Extract pooled radial-residual spectra from luma and opponent color."""
|
||||
if pixels.shape != (geometry.size, geometry.size, 3):
|
||||
raise ValueError(f"pixels must have shape {(geometry.size, geometry.size, 3)}, got {pixels.shape}")
|
||||
red, green, blue = np.moveaxis(np.asarray(pixels, dtype=np.float64), 2, 0)
|
||||
channels = (
|
||||
0.299 * red + 0.587 * green + 0.114 * blue,
|
||||
red - green,
|
||||
blue - (red + green) / 2.0,
|
||||
)
|
||||
block = geometry.size // geometry.grid
|
||||
features: list[FloatArray] = []
|
||||
for channel in channels:
|
||||
spectrum = phase_free_log_spectrum(channel)
|
||||
radial_sum = np.bincount(geometry.radius.ravel(), weights=spectrum.ravel())
|
||||
radial_mean = radial_sum / geometry.radius_counts
|
||||
residual = np.where(
|
||||
geometry.passband,
|
||||
spectrum - radial_mean[geometry.radius],
|
||||
0.0,
|
||||
)
|
||||
pooled = residual.reshape(geometry.grid, block, geometry.grid, block).mean(axis=(1, 3))
|
||||
features.append(pooled.ravel())
|
||||
return np.concatenate(features)
|
||||
|
||||
|
||||
def _apply_attack(image: Image.Image, attack: str) -> Image.Image:
|
||||
"""Apply one declared robustness transform before canonical resizing."""
|
||||
if attack == "original":
|
||||
@@ -134,15 +86,13 @@ def _apply_attack(image: Image.Image, attack: str) -> Image.Image:
|
||||
def feature_from_path(path: Path, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
"""Decode one path, apply ATTACK, and return its phase-free feature."""
|
||||
with Image.open(path) as source:
|
||||
return _feature_from_image(source.convert("RGB"), geometry, attack)
|
||||
return feature_from_image(source.convert("RGB"), geometry, attack)
|
||||
|
||||
|
||||
def _feature_from_image(image: Image.Image, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
def feature_from_image(image: Image.Image, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
"""Extract one attacked feature from an already decoded RGB image."""
|
||||
attacked = _apply_attack(image, attack)
|
||||
canonical = attacked.resize((geometry.size, geometry.size), Image.Resampling.LANCZOS)
|
||||
pixels = np.asarray(canonical, dtype=np.float64) / 255.0
|
||||
return spectral_feature(pixels, geometry)
|
||||
return _source_feature_from_image(attacked, geometry)
|
||||
|
||||
|
||||
def discover_grouped_paths(
|
||||
@@ -253,7 +203,7 @@ def _extract_all_attacks(
|
||||
with Image.open(path) as source:
|
||||
image = source.convert("RGB")
|
||||
for attack in attacks:
|
||||
rows[attack].append(_feature_from_image(image, geometry, attack))
|
||||
rows[attack].append(feature_from_image(image, geometry, attack))
|
||||
return {attack: np.asarray(features) for attack, features in rows.items()}
|
||||
|
||||
|
||||
|
||||
@@ -94,6 +94,10 @@ chroma-zimage for Microsoft, qwen-zimage for OpenAI (since CLI 0.39.0), Meta and
|
||||
-- so prefer it over naming an engine when the user has no reason to care. `sdxl-zimage` is the
|
||||
heavier alternative and `chroma-zimage` is the Apache-2.0 global stage.
|
||||
|
||||
Verified-text manifests work with every profile (since CLI 0.40.5). Each profile
|
||||
uses its own already loaded VAE donor, so `auto` keeps the normal SDXL route for a
|
||||
Google image instead of switching engines.
|
||||
|
||||
```bash
|
||||
remove-ai-watermarks all image.png -o clean.png
|
||||
remove-ai-watermarks batch ./images --mode visible --output-dir ./clean
|
||||
|
||||
@@ -32,7 +32,7 @@ _GPU_MARKERS = ("GPU ", "gpu:")
|
||||
# told an agent its CLI was current right before it typed a value that build
|
||||
# rejects. tests/test_agent_skill.py pins it against the package version and
|
||||
# against every "(since CLI X.Y.Z)" annotation in the skill.
|
||||
MIN_CLI_VERSION = (0, 40, 0)
|
||||
MIN_CLI_VERSION = (0, 40, 5)
|
||||
# Markers a build without the pixel stack prints. The first is the current CLI's own
|
||||
# install hint; the rest are what older builds emit before the guard existed, and the
|
||||
# Homebrew formula ships exactly such a build.
|
||||
|
||||
@@ -14,6 +14,7 @@ High-level API (lazy, so ``import remove_ai_watermarks`` stays cheap)::
|
||||
raiw.remove_video_metadata("in.mp4", "out.mp4") # verified metadata strip
|
||||
raiw.remove_video_invisible("in.mp4", "out.mp4") # calibrated video-pixel regeneration
|
||||
raiw.remove_video_visible("in.mp4", "out.mp4") # stable visible video-mark removal
|
||||
raiw.classify_source("in.png") # source pipeline, not SynthID detection
|
||||
|
||||
For a provenance verdict use the ``identify`` submodule::
|
||||
|
||||
@@ -33,7 +34,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.40.4"
|
||||
__version__ = "0.41.0"
|
||||
|
||||
__all__ = [
|
||||
"BatchItemResult",
|
||||
@@ -41,8 +42,10 @@ __all__ = [
|
||||
"InvisibleOptions",
|
||||
"MetadataStripIncomplete",
|
||||
"RemoveAllResult",
|
||||
"SourceClassification",
|
||||
"VisibleRemovalResult",
|
||||
"__version__",
|
||||
"classify_source",
|
||||
"identify_video",
|
||||
"inspect_video_metadata",
|
||||
"remove_all",
|
||||
@@ -71,6 +74,7 @@ if TYPE_CHECKING:
|
||||
remove_visible_detailed,
|
||||
visible_provenance,
|
||||
)
|
||||
from remove_ai_watermarks.source_classify import SourceClassification, classify_source
|
||||
from remove_ai_watermarks.video import (
|
||||
identify_video,
|
||||
inspect_video_metadata,
|
||||
@@ -101,6 +105,10 @@ def __getattr__(name: str) -> object:
|
||||
from remove_ai_watermarks import api
|
||||
|
||||
return getattr(api, name)
|
||||
if name in ("SourceClassification", "classify_source"):
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
return getattr(source_classify, name)
|
||||
if name in (
|
||||
"identify_video",
|
||||
"inspect_video_metadata",
|
||||
|
||||
@@ -36,7 +36,7 @@ from remove_ai_watermarks._internal.two_stage_pipeline import (
|
||||
_prompt_cache_path,
|
||||
_store_prompt_payload,
|
||||
_target_size,
|
||||
edge_pad_to_grid,
|
||||
diffusers_vae_roundtrip,
|
||||
requested_steps,
|
||||
)
|
||||
|
||||
@@ -182,28 +182,12 @@ class ChromaZImagePipeline(TwoStageZImagePipeline):
|
||||
|
||||
def _vae_roundtrip(self, image: Image.Image) -> Image.Image:
|
||||
"""Reconstruct source pixels through the already loaded Chroma VAE."""
|
||||
import torch
|
||||
|
||||
pipe = self._load_global()
|
||||
source_width, source_height = image.size
|
||||
padded = edge_pad_to_grid(image, _LATENT_GRID)
|
||||
|
||||
tensor = pipe.image_processor.preprocess(
|
||||
padded,
|
||||
height=padded.height,
|
||||
width=padded.width,
|
||||
).to(device=self.device, dtype=self.torch_dtype)
|
||||
with torch.inference_mode():
|
||||
encoded = pipe.vae.encode(tensor)
|
||||
if hasattr(encoded, "latent_dist"):
|
||||
# A restoration donor must be deterministic. Diffusers samples this
|
||||
# distribution for img2img noise initialization; its mode is the
|
||||
# faithful VAE reconstruction needed here.
|
||||
latents = encoded.latent_dist.mode()
|
||||
elif hasattr(encoded, "latents"):
|
||||
latents = encoded.latents
|
||||
else:
|
||||
raise AttributeError("Chroma VAE encoder output contains no latents")
|
||||
decoded = pipe.vae.decode(latents, return_dict=False)[0]
|
||||
reconstructed = pipe.image_processor.postprocess(decoded, output_type="pil")[0]
|
||||
return reconstructed.crop((0, 0, source_width, source_height)).convert("RGB")
|
||||
return diffusers_vae_roundtrip(
|
||||
pipe,
|
||||
image,
|
||||
grid=_LATENT_GRID,
|
||||
device=self.device,
|
||||
dtype=self.torch_dtype,
|
||||
profile_name=self.profile_name,
|
||||
)
|
||||
@@ -28,6 +28,7 @@ from remove_ai_watermarks._internal.two_stage_pipeline import (
|
||||
TwoStageZImagePipeline,
|
||||
_target_size,
|
||||
build_canny_control_image,
|
||||
diffusers_vae_roundtrip,
|
||||
requested_steps,
|
||||
)
|
||||
from remove_ai_watermarks._internal.watermark_profiles import (
|
||||
@@ -120,3 +121,15 @@ class SdxlZImagePipeline(TwoStageZImagePipeline):
|
||||
if result.size != image.size:
|
||||
result = result.resize(image.size, Image.Resampling.LANCZOS)
|
||||
return result.convert("RGB")
|
||||
|
||||
def _vae_roundtrip(self, image: Image.Image) -> Image.Image:
|
||||
"""Reconstruct source pixels through the already loaded SDXL VAE."""
|
||||
pipe = self._load_global()
|
||||
return diffusers_vae_roundtrip(
|
||||
pipe,
|
||||
image,
|
||||
grid=_LATENT_GRID,
|
||||
device=self.device,
|
||||
dtype=self.torch_dtype,
|
||||
profile_name=self.profile_name,
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Phase-free spectral features for source/export-pipeline classification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpectralGeometry:
|
||||
"""Precomputed coordinates for one phase-free spectral representation."""
|
||||
|
||||
size: int
|
||||
grid: int
|
||||
radius: NDArray[np.int32]
|
||||
radius_counts: NDArray[np.int64]
|
||||
passband: NDArray[np.bool_]
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def make_geometry(size: int = 256, grid: int = 16) -> SpectralGeometry:
|
||||
"""Build radial detrending and pooling geometry."""
|
||||
if size < 16 or grid < 2 or size % grid:
|
||||
raise ValueError("size must be at least 16 and divisible by grid")
|
||||
yy, xx = np.mgrid[:size, :size]
|
||||
radius = np.rint(np.hypot(yy - size // 2, xx - size // 2)).astype(np.int32)
|
||||
radius_counts = np.bincount(radius.ravel())
|
||||
passband = (radius >= 4) & (radius <= size * 0.48)
|
||||
return SpectralGeometry(size, grid, radius, radius_counts, passband)
|
||||
|
||||
|
||||
def phase_free_log_spectrum(channel: NDArray[np.float64]) -> NDArray[np.float64]:
|
||||
"""Return centered log FFT magnitude, deliberately discarding phase."""
|
||||
if channel.ndim != 2:
|
||||
raise ValueError("channel must be a two-dimensional array")
|
||||
centered = channel - float(np.mean(channel))
|
||||
return np.log1p(np.abs(np.fft.fftshift(np.fft.fft2(centered))))
|
||||
|
||||
|
||||
def spectral_feature(pixels: NDArray[np.float64], geometry: SpectralGeometry) -> NDArray[np.float64]:
|
||||
"""Extract pooled radial-residual spectra from luma and opponent color."""
|
||||
expected = (geometry.size, geometry.size, 3)
|
||||
if pixels.shape != expected:
|
||||
raise ValueError(f"pixels must have shape {expected}, got {pixels.shape}")
|
||||
red, green, blue = np.moveaxis(pixels, 2, 0)
|
||||
channels = (
|
||||
0.299 * red + 0.587 * green + 0.114 * blue,
|
||||
red - green,
|
||||
blue - (red + green) / 2.0,
|
||||
)
|
||||
block = geometry.size // geometry.grid
|
||||
features: list[NDArray[np.float64]] = []
|
||||
for channel in channels:
|
||||
spectrum = phase_free_log_spectrum(channel)
|
||||
radial_sum = np.bincount(geometry.radius.ravel(), weights=spectrum.ravel())
|
||||
radial_mean = radial_sum / geometry.radius_counts
|
||||
residual = np.where(
|
||||
geometry.passband,
|
||||
spectrum - radial_mean[geometry.radius],
|
||||
0.0,
|
||||
)
|
||||
pooled = residual.reshape(geometry.grid, block, geometry.grid, block).mean(axis=(1, 3))
|
||||
features.append(pooled.ravel())
|
||||
return np.concatenate(features)
|
||||
|
||||
|
||||
def feature_from_image(image: Image.Image, geometry: SpectralGeometry) -> NDArray[np.float64]:
|
||||
"""Resize an RGB image canonically and extract its phase-free feature."""
|
||||
canonical = image.resize((geometry.size, geometry.size), Image.Resampling.LANCZOS)
|
||||
pixels = np.asarray(canonical, dtype=np.float64) / 255.0
|
||||
return spectral_feature(pixels, geometry)
|
||||
@@ -96,6 +96,41 @@ def edge_pad_to_grid(image: Image.Image, grid: int) -> Image.Image:
|
||||
)
|
||||
|
||||
|
||||
def diffusers_vae_roundtrip(
|
||||
pipe: Any,
|
||||
image: Image.Image,
|
||||
*,
|
||||
grid: int,
|
||||
device: Any,
|
||||
dtype: Any,
|
||||
profile_name: str,
|
||||
) -> Image.Image:
|
||||
"""Reconstruct pixels deterministically through a Diffusers pipeline VAE."""
|
||||
import torch
|
||||
|
||||
source_width, source_height = image.size
|
||||
padded = edge_pad_to_grid(image, grid)
|
||||
tensor = pipe.image_processor.preprocess(
|
||||
padded,
|
||||
height=padded.height,
|
||||
width=padded.width,
|
||||
).to(device=device, dtype=dtype)
|
||||
with torch.inference_mode():
|
||||
encoded = pipe.vae.encode(tensor)
|
||||
if hasattr(encoded, "latent_dist"):
|
||||
# A restoration donor must be deterministic. Diffusers samples this
|
||||
# distribution for img2img noise initialization; its mode is the
|
||||
# faithful VAE reconstruction needed here.
|
||||
latents = encoded.latent_dist.mode()
|
||||
elif hasattr(encoded, "latents"):
|
||||
latents = encoded.latents
|
||||
else:
|
||||
raise AttributeError(f"{profile_name} VAE encoder output contains no latents")
|
||||
decoded = pipe.vae.decode(latents, return_dict=False)[0]
|
||||
reconstructed = pipe.image_processor.postprocess(decoded, output_type="pil")[0]
|
||||
return reconstructed.crop((0, 0, source_width, source_height)).convert("RGB")
|
||||
|
||||
|
||||
def resolve_face_model_residency(
|
||||
requested: bool | None,
|
||||
*,
|
||||
|
||||
@@ -250,8 +250,6 @@ class WatermarkRemover:
|
||||
self.model_profile = resolved
|
||||
self.torch_dtype = _dtype_for_profile(resolved)
|
||||
self._qwen_zimage_pipeline = None
|
||||
if text_manifest is not None and self.model_profile == SDXL_ZIMAGE_PROFILE:
|
||||
raise ValueError("Verified text restoration is not supported by the sdxl-zimage profile")
|
||||
if text_manifest is not None and tile:
|
||||
raise ValueError("Verified text restoration is not calibrated with tiled diffusion")
|
||||
if fidelity_anchor and self.model_profile != QWEN_ZIMAGE_PROFILE:
|
||||
|
||||
@@ -318,9 +318,8 @@ _text_manifest_option = click.option(
|
||||
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
||||
default=None,
|
||||
help=(
|
||||
"Experimental verified-text restoration manifest. Requires qwen-zimage or "
|
||||
"chroma-zimage, the text-restoration extra, native untiled geometry, and no "
|
||||
"postprocessing."
|
||||
"Experimental verified-text restoration manifest. Requires the text-restoration "
|
||||
"extra, native untiled geometry, and no postprocessing."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ from typing import TYPE_CHECKING, Any
|
||||
from ._internal.watermark_profiles import (
|
||||
DEFAULT_PROFILE,
|
||||
REMOVAL_MODULES,
|
||||
SDXL_ZIMAGE_PROFILE,
|
||||
resolve_adaptive_polish,
|
||||
resolve_effective_profile,
|
||||
resolve_seed,
|
||||
@@ -192,9 +191,9 @@ class InvisibleEngine:
|
||||
def preload(self, *, global_only: bool = False) -> None:
|
||||
"""Eagerly load the pipeline so download progress is visible.
|
||||
|
||||
For ``qwen-zimage``, ``global_only=True`` loads the mandatory Qwen stage
|
||||
and leaves the optional Z-Image and SAM face stack lazy until a face is
|
||||
detected. Other profiles have no optional stage and ignore the flag.
|
||||
For every concrete profile, ``global_only=True`` loads its mandatory
|
||||
global stage and leaves the shared Z-Image and SAM face stack lazy until
|
||||
a face is detected.
|
||||
"""
|
||||
self._remover.preload(global_only=global_only)
|
||||
|
||||
@@ -269,13 +268,18 @@ class InvisibleEngine:
|
||||
|
||||
seed = resolve_seed(seed)
|
||||
effective_profile = resolve_effective_profile(self._remover.configured_profile, vendor)
|
||||
adaptive_polish = resolve_adaptive_polish(adaptive_polish, effective_profile)
|
||||
# Verified text must be the final pixel-writing stage. SDXL normally enables
|
||||
# adaptive polish by profile default, so an unset value becomes False here;
|
||||
# an explicit True remains an error below instead of being silently ignored.
|
||||
adaptive_polish = (
|
||||
False
|
||||
if text_manifest is not None and adaptive_polish is None
|
||||
else resolve_adaptive_polish(adaptive_polish, effective_profile)
|
||||
)
|
||||
|
||||
if fidelity_anchor and text_manifest is None:
|
||||
raise ValueError("fidelity_anchor requires a text manifest")
|
||||
if text_manifest is not None:
|
||||
if effective_profile == SDXL_ZIMAGE_PROFILE:
|
||||
raise ValueError("--text-manifest is not supported by the sdxl-zimage profile")
|
||||
if tile:
|
||||
raise ValueError("--text-manifest is not calibrated with --tile")
|
||||
if max_resolution != 0:
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Abstaining source/export-pipeline classification from image pixels.
|
||||
|
||||
This module does not detect or decode SynthID. It labels original-export-looking
|
||||
pixel pipelines as OpenAI, Google, or unknown after metadata may have been
|
||||
removed. Call it explicitly; ``identify`` never imports or runs it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from hashlib import sha256
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
|
||||
SOURCE_CLASSIFY_EXTRA = "'remove-ai-watermarks[source-classify]'"
|
||||
WEIGHTS_ENV = "RAIW_SOURCE_CLASSIFY_WEIGHTS"
|
||||
WEIGHTS_REPO = "wiltodelta/raiw-source-classify"
|
||||
WEIGHTS_REVISION = "4b8b408d03dac20231d46217977fd164414b6b9e"
|
||||
MODEL_FILE = "source-pipeline-mlp.npz"
|
||||
MODEL_SHA256 = "6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a"
|
||||
CLASSES = ("openai", "google", "unknown")
|
||||
PROVIDERS = CLASSES[:2]
|
||||
FEATURE_WIDTH = 892
|
||||
SPECTRAL_GRID = 16
|
||||
CLAIM = "source/export pipeline classification; not SynthID detection"
|
||||
# NumPy and the training runtime can differ by a few float32 ULPs at a matrix
|
||||
# boundary. Treat values within this tolerance as the stored threshold so the
|
||||
# exported runtime preserves the training runtime's >= decision.
|
||||
THRESHOLD_TOLERANCE = 1e-5
|
||||
|
||||
SourceLabel = Literal["openai", "google", "unknown"]
|
||||
SourceReason = Literal["classified", "abstained", "feature_unavailable"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceClassification:
|
||||
"""One narrow source-pipeline result with an explicit abstention."""
|
||||
|
||||
label: SourceLabel
|
||||
reason: SourceReason
|
||||
scores: dict[str, float]
|
||||
margins: dict[str, float]
|
||||
thresholds: dict[str, float]
|
||||
claim: str = CLAIM
|
||||
model_revision: str = WEIGHTS_REVISION
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"label": self.label,
|
||||
"reason": self.reason,
|
||||
"scores": self.scores,
|
||||
"margins": self.margins,
|
||||
"thresholds": self.thresholds,
|
||||
"claim": self.claim,
|
||||
"model_revision": self.model_revision,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Model:
|
||||
mean: NDArray[Any]
|
||||
scale: NDArray[Any]
|
||||
margins: NDArray[Any]
|
||||
layer0_weight: NDArray[Any]
|
||||
layer0_bias: NDArray[Any]
|
||||
layer1_weight: NDArray[Any]
|
||||
layer1_bias: NDArray[Any]
|
||||
output_weight: NDArray[Any]
|
||||
output_bias: NDArray[Any]
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Return whether the lightweight source-classify runtime is installed."""
|
||||
from remove_ai_watermarks.optional_deps import module_available
|
||||
|
||||
return module_available("numpy", "cv2", "huggingface_hub")
|
||||
|
||||
|
||||
def _model_path() -> Path:
|
||||
override = os.environ.get(WEIGHTS_ENV, "").strip()
|
||||
if override:
|
||||
path = Path(override).expanduser()
|
||||
candidate = path / MODEL_FILE if path.is_dir() else path
|
||||
if not candidate.is_file():
|
||||
raise RuntimeError(f"{WEIGHTS_ENV} does not contain {MODEL_FILE}: {path}")
|
||||
return candidate
|
||||
try:
|
||||
hub: Any = import_module("huggingface_hub")
|
||||
except ModuleNotFoundError as error:
|
||||
raise RuntimeError(f"Install {SOURCE_CLASSIFY_EXTRA} to download the source classifier") from error
|
||||
return Path(
|
||||
hub.hf_hub_download(
|
||||
repo_id=WEIGHTS_REPO,
|
||||
filename=MODEL_FILE,
|
||||
revision=WEIGHTS_REVISION,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def _load_model(path: Path, expected_sha256: str | None = MODEL_SHA256) -> _Model:
|
||||
import numpy as np
|
||||
|
||||
if expected_sha256 is not None:
|
||||
digest = sha256(path.read_bytes()).hexdigest()
|
||||
if digest != expected_sha256:
|
||||
raise ValueError(f"model SHA-256 mismatch: expected {expected_sha256}, got {digest}")
|
||||
required = {
|
||||
"classes",
|
||||
"kind",
|
||||
"mean",
|
||||
"scale",
|
||||
"margins",
|
||||
"layer0_weight",
|
||||
"layer0_bias",
|
||||
"layer1_weight",
|
||||
"layer1_bias",
|
||||
"output_weight",
|
||||
"output_bias",
|
||||
"activation",
|
||||
}
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
if set(payload.files) != required:
|
||||
raise ValueError(f"model fields must be exactly {sorted(required)}")
|
||||
if tuple(str(value) for value in payload["classes"]) != CLASSES:
|
||||
raise ValueError(f"model classes must be {CLASSES}")
|
||||
if str(payload["kind"]) != "fused":
|
||||
raise ValueError("model kind must be fused")
|
||||
if str(payload["activation"]) != "gelu_exact":
|
||||
raise ValueError("model activation must be gelu_exact")
|
||||
arrays = {
|
||||
name: np.asarray(
|
||||
payload[name],
|
||||
dtype=np.float64 if name in {"mean", "scale", "margins"} else np.float32,
|
||||
)
|
||||
for name in required - {"classes", "kind", "activation"}
|
||||
}
|
||||
shapes = {
|
||||
"mean": (FEATURE_WIDTH,),
|
||||
"scale": (FEATURE_WIDTH,),
|
||||
"margins": (len(PROVIDERS),),
|
||||
"layer0_weight": (128, FEATURE_WIDTH),
|
||||
"layer0_bias": (128,),
|
||||
"layer1_weight": (64, 128),
|
||||
"layer1_bias": (64,),
|
||||
"output_weight": (len(CLASSES), 64),
|
||||
"output_bias": (len(CLASSES),),
|
||||
}
|
||||
for name, expected in shapes.items():
|
||||
if arrays[name].shape != expected:
|
||||
raise ValueError(f"model field {name} must have shape {expected}")
|
||||
if not np.all(np.isfinite(arrays[name])):
|
||||
raise ValueError(f"model field {name} must be finite")
|
||||
if np.any(arrays["scale"] <= 0) or np.any(arrays["margins"] < 0):
|
||||
raise ValueError("model scales must be positive and margins non-negative")
|
||||
return _Model(**arrays)
|
||||
|
||||
|
||||
def _gelu_exact(values: NDArray[Any]) -> NDArray[Any]:
|
||||
import numpy as np
|
||||
|
||||
flat = values.ravel()
|
||||
errors = np.fromiter((math.erf(float(value) / math.sqrt(2.0)) for value in flat), dtype=np.float32)
|
||||
return (0.5 * flat * (1.0 + errors)).reshape(values.shape).astype(np.float32)
|
||||
|
||||
|
||||
def _scores(features: NDArray[Any], model: _Model) -> NDArray[Any]:
|
||||
import numpy as np
|
||||
|
||||
values = np.asarray((features - model.mean) / model.scale, dtype=np.float32)
|
||||
hidden0 = _gelu_exact(model.layer0_weight @ values + model.layer0_bias)
|
||||
hidden1 = _gelu_exact(model.layer1_weight @ hidden0 + model.layer1_bias)
|
||||
return np.asarray(model.output_weight @ hidden1 + model.output_bias, dtype=np.float64)
|
||||
|
||||
|
||||
def _result(raw_scores: NDArray[Any], thresholds: NDArray[Any]) -> SourceClassification:
|
||||
import numpy as np
|
||||
|
||||
margins = np.empty(len(PROVIDERS), dtype=np.float64)
|
||||
for index in range(len(PROVIDERS)):
|
||||
competitors = np.delete(raw_scores, index)
|
||||
margins[index] = raw_scores[index] - float(np.max(competitors))
|
||||
excess = margins - thresholds
|
||||
best = int(np.argmax(excess))
|
||||
accepted = bool(excess[best] >= -THRESHOLD_TOLERANCE)
|
||||
label: SourceLabel = PROVIDERS[best] if accepted else "unknown"
|
||||
return SourceClassification(
|
||||
label=label,
|
||||
reason="classified" if accepted else "abstained",
|
||||
scores={name: float(raw_scores[index]) for index, name in enumerate(CLASSES)},
|
||||
margins={name: float(margins[index]) for index, name in enumerate(PROVIDERS)},
|
||||
thresholds={name: float(thresholds[index]) for index, name in enumerate(PROVIDERS)},
|
||||
)
|
||||
|
||||
|
||||
def _unavailable(model: _Model) -> SourceClassification:
|
||||
return SourceClassification(
|
||||
label="unknown",
|
||||
reason="feature_unavailable",
|
||||
scores={},
|
||||
margins={},
|
||||
thresholds={name: float(model.margins[index]) for index, name in enumerate(PROVIDERS)},
|
||||
)
|
||||
|
||||
|
||||
def classify_source(source: str | os.PathLike[str]) -> SourceClassification:
|
||||
"""Classify an original-looking image export, abstaining as ``unknown``.
|
||||
|
||||
The result describes a complete source/export pipeline. It is not evidence
|
||||
that SynthID is present or absent.
|
||||
"""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks import image_io
|
||||
from remove_ai_watermarks._internal.forensic_124d import image_features
|
||||
from remove_ai_watermarks._internal.source_spectral import feature_from_image, make_geometry
|
||||
|
||||
model = _load_model(_model_path(), expected_sha256=MODEL_SHA256)
|
||||
image_io._register_heif() # pyright: ignore[reportPrivateUsage]
|
||||
with Image.open(source) as opened:
|
||||
image = opened.convert("RGB")
|
||||
pixels = np.asarray(image, dtype=np.uint8)
|
||||
forensic = image_features(pixels)
|
||||
if forensic is None:
|
||||
return _unavailable(model)
|
||||
spectral = feature_from_image(image, make_geometry(grid=SPECTRAL_GRID))
|
||||
features = np.concatenate((forensic, spectral))
|
||||
if features.shape != (FEATURE_WIDTH,) or not np.all(np.isfinite(features)):
|
||||
return _unavailable(model)
|
||||
return _result(_scores(features, model), model.margins)
|
||||
@@ -23,6 +23,10 @@ class TestTopLevelExports:
|
||||
assert raiw.remove_visible_detailed is api.remove_visible_detailed
|
||||
assert raiw.VisibleRemovalResult is api.VisibleRemovalResult
|
||||
assert raiw.visible_provenance is api.visible_provenance
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
assert raiw.classify_source is source_classify.classify_source
|
||||
assert raiw.SourceClassification is source_classify.SourceClassification
|
||||
|
||||
def test_unknown_attribute_raises(self):
|
||||
with pytest.raises(AttributeError):
|
||||
|
||||
@@ -64,7 +64,6 @@ class TestVerifiedTextMode:
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text("{}", encoding="utf-8")
|
||||
cases = (
|
||||
("sdxl-zimage", {}, "not supported by the sdxl-zimage profile"),
|
||||
("qwen-zimage", {"tile": True}, "not calibrated with --tile"),
|
||||
("qwen-zimage", {"max_resolution": 1024}, "max-resolution 0"),
|
||||
("qwen-zimage", {"humanize": 1.0}, "humanize=0"),
|
||||
@@ -87,19 +86,6 @@ class TestVerifiedTextMode:
|
||||
fidelity_anchor=True,
|
||||
)
|
||||
|
||||
def test_auto_rejects_google_text_manifest_before_model_loading(self, tmp_path):
|
||||
import pytest
|
||||
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text("{}", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="not supported by the sdxl-zimage profile"):
|
||||
self._engine("auto").remove_watermark(
|
||||
tmp_path / "unused.png",
|
||||
text_manifest=manifest,
|
||||
vendor="google",
|
||||
)
|
||||
|
||||
def test_loads_and_forwards_verified_manifest(self, tmp_path, monkeypatch):
|
||||
import json
|
||||
|
||||
@@ -137,9 +123,10 @@ class TestVerifiedTextMode:
|
||||
engine._remover.remove_watermark = fake_remove
|
||||
monkeypatch.setattr(region_eraser, "lama_available", lambda: True)
|
||||
|
||||
engine.remove_watermark(source, output, text_manifest=manifest)
|
||||
engine.remove_watermark(source, output, text_manifest=manifest, vendor="google")
|
||||
|
||||
assert seen["text_manifest"].lines[0].text == "Exact"
|
||||
assert seen["vendor"] == "google"
|
||||
# Leak-safe default since 0.27.1: the global 15% donor blend is OFF unless
|
||||
# explicitly requested (measured to return detector-visible OpenAI SynthID
|
||||
# on poster-scale manifests; see docs/text-protection-research.md).
|
||||
|
||||
@@ -2,29 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _publish() -> Any:
|
||||
path = Path(__file__).resolve().parents[1] / "scripts" / "publish_photo_classify_hf.py"
|
||||
spec = importlib.util.spec_from_file_location("publish_photo_classify_hf", path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import publish_photo_classify_hf as publish # noqa: E402
|
||||
|
||||
|
||||
def test_hub_id_is_the_photo_classify_repo() -> None:
|
||||
assert _publish().HUB_REPO == "wiltodelta/raiw-photo-classify"
|
||||
assert publish.HUB_REPO == "wiltodelta/raiw-photo-classify"
|
||||
|
||||
|
||||
def test_publish_freeze_includes_both_clip_runtimes() -> None:
|
||||
publish = _publish()
|
||||
from remove_ai_watermarks.classify import _WEIGHT_FILES, CLIP_ONNX_FILE, WEIGHTS_ALLOW_PATTERNS
|
||||
|
||||
assert publish.CLIP_FILE in publish.WEIGHT_FILES
|
||||
@@ -41,7 +33,7 @@ def test_weight_paths_accepts_run1_layout(tmp_path: Path) -> None:
|
||||
run1.mkdir()
|
||||
(run1 / "detector.pt").write_bytes(b"det")
|
||||
(run1 / "provider.pt").write_bytes(b"prov")
|
||||
found = _publish().weight_paths(tmp_path)
|
||||
found = publish.weight_paths(tmp_path)
|
||||
assert found["detector.pt"] == run1 / "detector.pt"
|
||||
assert found["clip-l-ft.pt"] == tmp_path / "clip-l-ft.pt"
|
||||
|
||||
@@ -50,12 +42,12 @@ def test_weight_paths_requires_every_file(tmp_path: Path) -> None:
|
||||
(tmp_path / "clip-l-ft.pt").write_bytes(b"clip")
|
||||
(tmp_path / "clip-l-ft-vision-fp32.onnx").write_bytes(b"onnx")
|
||||
with pytest.raises(SystemExit, match=r"missing probe-weights-clip-l-ft\.npz"):
|
||||
_publish().weight_paths(tmp_path)
|
||||
publish.weight_paths(tmp_path)
|
||||
|
||||
|
||||
def test_stage_card_copies_tracked_files(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "stage"
|
||||
_publish().stage_card(dest)
|
||||
publish.stage_card(dest)
|
||||
assert (dest / "README.md").is_file()
|
||||
assert (dest / "operating-point.json").is_file()
|
||||
text = (dest / "operating-point.json").read_text()
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Local staging checks for the source-classifier Hub publisher."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import publish_source_classify_hf as publish # noqa: E402
|
||||
|
||||
|
||||
def test_hub_id_is_the_source_classify_repo() -> None:
|
||||
assert publish.HUB_REPO == "wiltodelta/raiw-source-classify"
|
||||
|
||||
|
||||
def test_stage_release_copies_only_public_artifacts(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
(source / "source-pipeline-mlp.npz").write_bytes(b"weights")
|
||||
destination = tmp_path / "stage"
|
||||
publish.stage_release(destination, source)
|
||||
assert sorted(path.name for path in destination.iterdir()) == sorted(publish.PUBLIC_FILES)
|
||||
|
||||
|
||||
def test_stage_release_requires_the_model(tmp_path: Path) -> None:
|
||||
with pytest.raises(SystemExit, match=r"source-pipeline-mlp\.npz"):
|
||||
publish.stage_release(tmp_path / "stage", tmp_path)
|
||||
|
||||
|
||||
def test_card_sha_matches_the_metrics_manifest() -> None:
|
||||
metrics = json.loads((publish.CARD_DIR / "metrics.json").read_text())
|
||||
card = (publish.CARD_DIR / "README.md").read_text()
|
||||
assert metrics["artifact_sha256"] in card
|
||||
@@ -996,6 +996,70 @@ def test_sdxl_zimage_inherits_the_face_stage_rather_than_copying_it():
|
||||
assert SdxlZImagePipeline._load_global is not QwenZImagePipeline._load_global
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pipeline_module", "pipeline_class", "grid"),
|
||||
[
|
||||
("sdxl_zimage_pipeline", "SdxlZImagePipeline", 8),
|
||||
("chroma_zimage_pipeline", "ChromaZImagePipeline", 16),
|
||||
],
|
||||
)
|
||||
def test_diffusers_vae_roundtrip_uses_deterministic_mode_and_restores_native_size(
|
||||
monkeypatch, pipeline_module, pipeline_class, grid
|
||||
):
|
||||
"""Diffusers donors pad without distorting geometry and never sample."""
|
||||
import importlib
|
||||
|
||||
import torch
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
padded_width = 40 if grid == 8 else 48
|
||||
padded_height = 24 if grid == 8 else 32
|
||||
latent = torch.ones(1, 4, padded_height // 8, padded_width // 8)
|
||||
|
||||
class LatentDistribution:
|
||||
def mode(self):
|
||||
seen["mode_calls"] = int(seen.get("mode_calls", 0)) + 1
|
||||
return latent
|
||||
|
||||
def sample(self, *_args, **_kwargs):
|
||||
raise AssertionError("A verified-text donor must not sample VAE latents")
|
||||
|
||||
class FakeVae:
|
||||
@staticmethod
|
||||
def encode(tensor):
|
||||
seen["encoded_shape"] = tuple(tensor.shape)
|
||||
return type("Encoded", (), {"latent_dist": LatentDistribution()})()
|
||||
|
||||
@staticmethod
|
||||
def decode(received, *, return_dict):
|
||||
assert received is latent
|
||||
assert return_dict is False
|
||||
return (torch.zeros(1, 3, padded_height, padded_width),)
|
||||
|
||||
class FakeProcessor:
|
||||
@staticmethod
|
||||
def preprocess(image, *, height, width):
|
||||
seen["preprocess"] = (image.size, height, width)
|
||||
return torch.zeros(1, 3, height, width)
|
||||
|
||||
@staticmethod
|
||||
def postprocess(decoded, *, output_type):
|
||||
assert output_type == "pil"
|
||||
return [Image.new("RGB", (decoded.shape[3], decoded.shape[2]))]
|
||||
|
||||
pipe = type("FakePipe", (), {"vae": FakeVae(), "image_processor": FakeProcessor()})()
|
||||
module = importlib.import_module(f"remove_ai_watermarks._internal.{pipeline_module}")
|
||||
pipeline = getattr(module, pipeline_class)(device="cpu", torch_dtype=torch.float32)
|
||||
monkeypatch.setattr(pipeline, "_load_global", lambda: pipe)
|
||||
|
||||
result = pipeline._vae_roundtrip(Image.new("RGB", (35, 19)))
|
||||
|
||||
assert seen["preprocess"] == ((padded_width, padded_height), padded_height, padded_width)
|
||||
assert seen["encoded_shape"] == (1, 3, padded_height, padded_width)
|
||||
assert seen["mode_calls"] == 1
|
||||
assert result.size == (35, 19)
|
||||
|
||||
|
||||
def test_chroma_zimage_strength_uses_measured_flat_floors():
|
||||
"""The chroma floors come from the four-cohort oracle calibration; they must not
|
||||
drift from their derivation in watermark_profiles."""
|
||||
@@ -1163,8 +1227,9 @@ def test_chroma_zimage_floors_to_its_own_latent_grid():
|
||||
assert chroma_target_size(3, 3) == (16, 16)
|
||||
|
||||
|
||||
def test_watermark_remover_accepts_text_manifest_for_chroma_but_not_sdxl(tmp_path, monkeypatch):
|
||||
"""Chroma owns a VAE donor; SDXL still fails at the public boundary."""
|
||||
@pytest.mark.parametrize("profile", ["chroma-zimage", "sdxl-zimage"])
|
||||
def test_watermark_remover_accepts_text_manifest_for_every_diffusers_profile(tmp_path, monkeypatch, profile):
|
||||
"""Both Diffusers profiles forward reviewed text to their VAE donor."""
|
||||
from remove_ai_watermarks._internal.text_restoration import VerifiedTextLine, VerifiedTextManifest
|
||||
from remove_ai_watermarks._internal.watermark_remover import WatermarkRemover
|
||||
|
||||
@@ -1173,22 +1238,18 @@ def test_watermark_remover_accepts_text_manifest_for_chroma_but_not_sdxl(tmp_pat
|
||||
Image.new("RGB", (96, 80)).save(source)
|
||||
manifest = VerifiedTextManifest("0" * 64, 96, 80, (VerifiedTextLine((4, 4, 20, 16), "x", "alphabetic"),))
|
||||
|
||||
chroma = WatermarkRemover(device="cuda", pipeline="chroma-zimage")
|
||||
chroma_runtime = MagicMock()
|
||||
chroma_runtime.run.return_value = Image.new("RGB", (96, 80))
|
||||
monkeypatch.setattr(chroma, "_load_qwen_zimage_pipeline", lambda: chroma_runtime)
|
||||
remover = WatermarkRemover(device="cuda", pipeline=profile)
|
||||
runtime = MagicMock()
|
||||
runtime.run.return_value = Image.new("RGB", (96, 80))
|
||||
monkeypatch.setattr(remover, "_load_qwen_zimage_pipeline", lambda: runtime)
|
||||
|
||||
chroma.remove_watermark(source, text_manifest=manifest)
|
||||
remover.remove_watermark(source, text_manifest=manifest)
|
||||
|
||||
assert chroma_runtime.run.call_count == 1
|
||||
assert chroma_runtime.run.call_args.kwargs["text_manifest"] is manifest
|
||||
assert runtime.run.call_count == 1
|
||||
assert runtime.run.call_args.kwargs["text_manifest"] is manifest
|
||||
with pytest.raises(ValueError, match=r"fidelity anchor.*qwen-zimage"):
|
||||
chroma.remove_watermark(source, text_manifest=manifest, fidelity_anchor=True)
|
||||
assert chroma_runtime.run.call_count == 1
|
||||
|
||||
sdxl = WatermarkRemover(device="cuda", pipeline="sdxl-zimage")
|
||||
with pytest.raises(ValueError, match="not supported by the sdxl-zimage profile"):
|
||||
sdxl.remove_watermark(source, text_manifest=manifest)
|
||||
remover.remove_watermark(source, text_manifest=manifest, fidelity_anchor=True)
|
||||
assert runtime.run.call_count == 1
|
||||
|
||||
|
||||
def test_watermark_remover_dispatches_to_chroma_pipeline(monkeypatch):
|
||||
@@ -1229,58 +1290,6 @@ def test_chroma_adaptive_polish_defaults_off():
|
||||
assert resolve_adaptive_polish(True, "chroma-zimage") is True
|
||||
|
||||
|
||||
def test_chroma_vae_roundtrip_uses_deterministic_mode_and_restores_native_size(monkeypatch):
|
||||
"""The Chroma donor pads without distorting geometry and never samples."""
|
||||
import torch
|
||||
|
||||
from remove_ai_watermarks._internal.chroma_zimage_pipeline import ChromaZImagePipeline
|
||||
|
||||
seen: dict[str, object] = {}
|
||||
latent = torch.ones(1, 4, 4, 6)
|
||||
|
||||
class LatentDistribution:
|
||||
def mode(self):
|
||||
seen["mode_calls"] = int(seen.get("mode_calls", 0)) + 1
|
||||
return latent
|
||||
|
||||
def sample(self, *_args, **_kwargs):
|
||||
raise AssertionError("A verified-text donor must not sample VAE latents")
|
||||
|
||||
class FakeVae:
|
||||
@staticmethod
|
||||
def encode(tensor):
|
||||
seen["encoded_shape"] = tuple(tensor.shape)
|
||||
return type("Encoded", (), {"latent_dist": LatentDistribution()})()
|
||||
|
||||
@staticmethod
|
||||
def decode(received, *, return_dict):
|
||||
assert received is latent
|
||||
assert return_dict is False
|
||||
return (torch.zeros(1, 3, 32, 48),)
|
||||
|
||||
class FakeProcessor:
|
||||
@staticmethod
|
||||
def preprocess(image, *, height, width):
|
||||
seen["preprocess"] = (image.size, height, width)
|
||||
return torch.zeros(1, 3, height, width)
|
||||
|
||||
@staticmethod
|
||||
def postprocess(decoded, *, output_type):
|
||||
assert output_type == "pil"
|
||||
return [Image.new("RGB", (decoded.shape[3], decoded.shape[2]))]
|
||||
|
||||
pipe = type("FakePipe", (), {"vae": FakeVae(), "image_processor": FakeProcessor()})()
|
||||
pipeline = ChromaZImagePipeline(device="cpu", torch_dtype=torch.float32)
|
||||
monkeypatch.setattr(pipeline, "_load_global", lambda: pipe)
|
||||
|
||||
result = pipeline._vae_roundtrip(Image.new("RGB", (35, 19)))
|
||||
|
||||
assert seen["preprocess"] == ((48, 32), 32, 48)
|
||||
assert seen["encoded_shape"] == (1, 3, 32, 48)
|
||||
assert seen["mode_calls"] == 1
|
||||
assert result.size == (35, 19)
|
||||
|
||||
|
||||
def test_chroma_google_uses_one_content_agnostic_floor():
|
||||
"""Face detection alone cannot safely select a lower Google floor.
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Unit contracts for the lightweight source-pipeline runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from hashlib import sha256
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_model_identity_is_immutable() -> None:
|
||||
assert re.fullmatch(r"[0-9a-f]{40}", source_classify.WEIGHTS_REVISION)
|
||||
assert re.fullmatch(r"[0-9a-f]{64}", source_classify.MODEL_SHA256)
|
||||
|
||||
|
||||
def _artifact(path: Path, *, activation: str = "gelu_exact", output_bias: tuple[float, ...] = (0.0, 0.0, 0.0)) -> None:
|
||||
np.savez_compressed(
|
||||
path,
|
||||
classes=np.asarray(source_classify.CLASSES),
|
||||
kind=np.asarray("fused"),
|
||||
mean=np.zeros(source_classify.FEATURE_WIDTH),
|
||||
scale=np.ones(source_classify.FEATURE_WIDTH),
|
||||
margins=np.asarray((0.5, 0.5)),
|
||||
layer0_weight=np.zeros((128, source_classify.FEATURE_WIDTH), dtype=np.float32),
|
||||
layer0_bias=np.zeros(128, dtype=np.float32),
|
||||
layer1_weight=np.zeros((64, 128), dtype=np.float32),
|
||||
layer1_bias=np.zeros(64, dtype=np.float32),
|
||||
output_weight=np.zeros((3, 64), dtype=np.float32),
|
||||
output_bias=np.asarray(output_bias, dtype=np.float32),
|
||||
activation=np.asarray(activation),
|
||||
)
|
||||
|
||||
|
||||
def test_loader_accepts_the_exact_pickle_free_schema(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path)
|
||||
|
||||
model = source_classify._load_model(path, expected_sha256=None)
|
||||
|
||||
assert model.mean.shape == (source_classify.FEATURE_WIDTH,)
|
||||
assert model.margins.tolist() == pytest.approx([0.5, 0.5])
|
||||
|
||||
|
||||
def test_loader_rejects_the_mislabeled_activation(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path, activation="gelu_tanh")
|
||||
|
||||
with pytest.raises(ValueError, match="activation must be gelu_exact"):
|
||||
source_classify._load_model(path, expected_sha256=None)
|
||||
|
||||
|
||||
def test_provider_threshold_mutation_changes_the_real_decision(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path, output_bias=(2.0, 0.0, 0.0))
|
||||
model = source_classify._load_model(path, expected_sha256=None)
|
||||
features = np.zeros(source_classify.FEATURE_WIDTH, dtype=np.float32)
|
||||
scores = source_classify._scores(features, model)
|
||||
|
||||
accepted = source_classify._result(scores, np.asarray((1.5, 0.5), dtype=np.float32))
|
||||
rejected = source_classify._result(scores, np.asarray((2.5, 0.5), dtype=np.float32))
|
||||
|
||||
assert accepted.label == "openai"
|
||||
assert accepted.reason == "classified"
|
||||
assert rejected.label == "unknown"
|
||||
assert rejected.reason == "abstained"
|
||||
|
||||
|
||||
def test_a_float32_threshold_tie_preserves_the_inclusive_training_decision() -> None:
|
||||
scores = np.asarray((2.0, 0.0, 0.0), dtype=np.float32)
|
||||
result = source_classify._result(scores, np.asarray((2.0, 0.5), dtype=np.float32))
|
||||
|
||||
assert result.label == "openai"
|
||||
assert result.reason == "classified"
|
||||
|
||||
|
||||
def test_small_image_returns_honest_unknown(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from PIL import Image
|
||||
|
||||
weights = tmp_path / "weights"
|
||||
weights.mkdir()
|
||||
_artifact(weights / source_classify.MODEL_FILE)
|
||||
monkeypatch.setenv(source_classify.WEIGHTS_ENV, str(weights))
|
||||
monkeypatch.setattr(
|
||||
source_classify,
|
||||
"MODEL_SHA256",
|
||||
sha256((weights / source_classify.MODEL_FILE).read_bytes()).hexdigest(),
|
||||
)
|
||||
image = tmp_path / "small.png"
|
||||
Image.new("RGB", (128, 128), "white").save(image)
|
||||
|
||||
result = source_classify.classify_source(image)
|
||||
|
||||
assert result.label == "unknown"
|
||||
assert result.reason == "feature_unavailable"
|
||||
assert result.scores == {}
|
||||
|
||||
|
||||
def test_model_hash_is_a_runtime_precondition(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path)
|
||||
|
||||
with pytest.raises(ValueError, match="model SHA-256 mismatch"):
|
||||
source_classify._load_model(path, expected_sha256="0" * 64)
|
||||
|
||||
|
||||
def test_spectral_feature_is_finite_and_has_the_frozen_width() -> None:
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks._internal.source_spectral import feature_from_image, make_geometry
|
||||
|
||||
rng = np.random.default_rng(20260916)
|
||||
pixels = rng.integers(0, 256, size=(300, 400, 3), dtype=np.uint8)
|
||||
feature = feature_from_image(Image.fromarray(pixels), make_geometry())
|
||||
|
||||
assert feature.shape == (768,)
|
||||
assert np.all(np.isfinite(feature))
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Tests for the abstaining source-pipeline classifier research tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import source_pipeline_classifier as classifier
|
||||
|
||||
|
||||
def test_command_help_exits_cleanly() -> None:
|
||||
result = CliRunner().invoke(classifier.main, ["--help"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Usage:" in result.output
|
||||
assert "train" in result.output
|
||||
assert "predict" in result.output
|
||||
|
||||
|
||||
def _training_rows() -> tuple[np.ndarray, np.ndarray]:
|
||||
rng = np.random.default_rng(20260915)
|
||||
rows = np.vstack(
|
||||
(
|
||||
rng.normal((-3.0, 0.0, 0.0), 0.1, size=(12, 3)),
|
||||
rng.normal((3.0, 0.0, 0.0), 0.1, size=(12, 3)),
|
||||
rng.normal((0.0, 3.0, 0.0), 0.1, size=(12, 3)),
|
||||
)
|
||||
)
|
||||
labels = np.repeat(np.arange(3), 12)
|
||||
return rows, labels
|
||||
|
||||
|
||||
def test_fit_and_abstention_separate_provider_rows() -> None:
|
||||
features, labels = _training_rows()
|
||||
model = classifier.fit_ridge(features, labels, penalty=1.0)
|
||||
raw_scores = classifier.scores(features, model)
|
||||
margin = classifier.calibrate_margin(raw_scores, labels, unknown_fpr=0.01)
|
||||
predictions = classifier.predict_scores(raw_scores, margin)
|
||||
|
||||
assert np.array_equal(predictions[:24], labels[:24])
|
||||
assert np.all(predictions[24:] == classifier.UNKNOWN_INDEX)
|
||||
|
||||
|
||||
def test_model_round_trip_is_pickle_free_and_strict(tmp_path: Path) -> None:
|
||||
grid = 4
|
||||
feature_count = classifier.FEATURE_WIDTH + 3 * grid * grid
|
||||
model = classifier.RidgeModel(
|
||||
mean=np.arange(feature_count, dtype=np.float64),
|
||||
scale=np.full(feature_count, 2.0),
|
||||
weights=np.arange(feature_count * 3, dtype=np.float64).reshape(feature_count, 3),
|
||||
margin=0.25,
|
||||
grid=grid,
|
||||
penalty=100.0,
|
||||
)
|
||||
path = tmp_path / "model.npz"
|
||||
|
||||
classifier.save_model(path, model)
|
||||
loaded = classifier.load_model(path)
|
||||
|
||||
assert loaded.grid == grid
|
||||
assert loaded.margin == model.margin
|
||||
assert np.array_equal(loaded.mean, model.mean)
|
||||
assert np.array_equal(loaded.scale, model.scale)
|
||||
assert np.array_equal(loaded.weights, model.weights)
|
||||
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
assert payload.files
|
||||
|
||||
|
||||
def test_model_loader_rejects_extra_fields(tmp_path: Path) -> None:
|
||||
path = tmp_path / "model.npz"
|
||||
np.savez(
|
||||
path,
|
||||
classes=np.asarray(classifier.CLASSES),
|
||||
mean=np.zeros(136),
|
||||
scale=np.ones(136),
|
||||
weights=np.zeros((136, 3)),
|
||||
margin=np.asarray(0.1),
|
||||
spectral_grid=np.asarray(2),
|
||||
penalty=np.asarray(100.0),
|
||||
unexpected=np.asarray(1),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="model fields must be exactly"):
|
||||
classifier.load_model(path)
|
||||
|
||||
|
||||
def test_model_loader_rejects_nonfinite_margin(tmp_path: Path) -> None:
|
||||
grid = 2
|
||||
feature_count = classifier.FEATURE_WIDTH + 3 * grid * grid
|
||||
path = tmp_path / "model.npz"
|
||||
np.savez(
|
||||
path,
|
||||
classes=np.asarray(classifier.CLASSES),
|
||||
mean=np.zeros(feature_count),
|
||||
scale=np.ones(feature_count),
|
||||
weights=np.zeros((feature_count, 3)),
|
||||
margin=np.asarray(np.nan),
|
||||
spectral_grid=np.asarray(grid),
|
||||
penalty=np.asarray(100.0),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="margin must be finite"):
|
||||
classifier.load_model(path)
|
||||
|
||||
|
||||
def test_calibration_requires_unknown_rows() -> None:
|
||||
with pytest.raises(ValueError, match="must contain unknown"):
|
||||
classifier.calibrate_margin(
|
||||
np.zeros((2, 3)),
|
||||
np.asarray([0, 1]),
|
||||
unknown_fpr=0.01,
|
||||
)
|
||||
@@ -800,7 +800,7 @@ name = "cuda-bindings"
|
||||
version = "13.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cuda-pathfinder", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "cuda-pathfinder" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" },
|
||||
@@ -833,43 +833,43 @@ wheels = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
cublas = [
|
||||
{ name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cudart = [
|
||||
{ name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cufft = [
|
||||
{ name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cufile = [
|
||||
{ name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cupti = [
|
||||
{ name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
curand = [
|
||||
{ name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cusolver = [
|
||||
{ name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cusparse = [
|
||||
{ name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvjitlink = [
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvrtc = [
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvtx = [
|
||||
{ name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1066,7 +1066,7 @@ wheels = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
http = [
|
||||
{ name = "aiohttp", marker = "python_full_version < '3.13'" },
|
||||
{ name = "aiohttp" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1198,8 +1198,8 @@ name = "httpcore2"
|
||||
version = "2.12.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "h11", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "truststore", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "h11" },
|
||||
{ name = "truststore" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" }
|
||||
wheels = [
|
||||
@@ -1461,15 +1461,15 @@ name = "lightning"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pytorch-lightning", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pyyaml", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchmetrics", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "fsspec", extra = ["http"] },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pytorch-lightning" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchmetrics" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/39/b8/fbf1469d917c3ed3b22e00ab3a852fb9a5f85b9dc7f7060eedc3449098e7/lightning-2.6.6.tar.gz", hash = "sha256:1c6329e241f8a85f29fedead89c1ee138b36caa08de60c260c6a7eaee81c8295", size = 659902, upload-time = "2026-09-10T09:41:03.009Z" }
|
||||
wheels = [
|
||||
@@ -1481,8 +1481,8 @@ name = "lightning-utilities"
|
||||
version = "0.15.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f1/45/7fa8f56b17dc0f0a41ec70dd307ecd6787254483549843bef4c30ab5adce/lightning_utilities-0.15.3.tar.gz", hash = "sha256:792ae0204c79f6859721ac7f386c237a33b0ed06ba775009cb894e010a842033", size = 33553, upload-time = "2026-02-22T14:48:53.348Z" }
|
||||
wheels = [
|
||||
@@ -1896,7 +1896,7 @@ name = "nvidia-cublas"
|
||||
version = "13.1.1.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cuda-nvrtc" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
|
||||
@@ -1935,7 +1935,7 @@ name = "nvidia-cudnn-cu13"
|
||||
version = "9.24.0.43"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cublas" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" },
|
||||
@@ -1947,7 +1947,7 @@ name = "nvidia-cufft"
|
||||
version = "12.0.0.61"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
|
||||
@@ -1977,9 +1977,9 @@ name = "nvidia-cusolver"
|
||||
version = "12.0.4.66"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cusparse", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cublas" },
|
||||
{ name = "nvidia-cusparse" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
|
||||
@@ -1991,7 +1991,7 @@ name = "nvidia-cusparse"
|
||||
version = "12.6.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
|
||||
@@ -2146,7 +2146,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" }
|
||||
wheels = [
|
||||
@@ -2175,7 +2175,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ed/e8/c2959c0f969ea4ae96b55ad10c108c0410de4e8b3550d9941a4cfdd52703/opencv_python-4.14.0.94.tar.gz", hash = "sha256:afadc76a38a07713a4c2b4702da917cf0ed489be57e2a40a34e670189731de7d", size = 96400225, upload-time = "2026-07-28T19:23:09.546Z" }
|
||||
wheels = [
|
||||
@@ -2206,7 +2206,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" }
|
||||
wheels = [
|
||||
@@ -2235,7 +2235,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/67/8a/51ff1c6287be0b6cd3070cf095875da4ff7d6eda38507e6ada5608786a78/opencv_python_headless-4.14.0.94.tar.gz", hash = "sha256:4afa2ea1214453648be88259f035712454faa9039b686de7753569ba8eec1577", size = 96405664, upload-time = "2026-07-28T19:23:48.758Z" }
|
||||
wheels = [
|
||||
@@ -3179,14 +3179,14 @@ name = "pytorch-lightning"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pyyaml", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchmetrics", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "fsspec", extra = ["http"] },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchmetrics" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/97/e4/5025cb42311b4f8cc6d1b12cd81399ca433159f8e3831ab229c5a6c5177e/pytorch_lightning-2.6.6.tar.gz", hash = "sha256:52c52b20522dd3d3feb3bb3aae2dbe3f291e8e836d4111a74cfd72e9df018834", size = 662507, upload-time = "2026-09-10T09:41:16.684Z" }
|
||||
wheels = [
|
||||
@@ -3205,7 +3205,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5a/75/50581633d199812205ea8cdd0f6d52f12a624886b74bf1486335b67f01ff/pywavelets-1.9.0.tar.gz", hash = "sha256:148d12203377772bea452a59211d98649c8ee4a05eff019a9021853a36babdc8", size = 3938340, upload-time = "2025-08-04T16:20:04.978Z" }
|
||||
wheels = [
|
||||
@@ -3281,7 +3281,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/25/84/a9e80e33eb114e78bd2f1829a90d72d8d4d660109c721a8aa2f0d58d7592/pywavelets-1.10.0.tar.gz", hash = "sha256:f3cb8640225a8f3fdcbc3f6f3bd112c78927c9407cbfa65a5ff303990a6a3e32", size = 3948163, upload-time = "2026-09-07T17:15:15.637Z" }
|
||||
@@ -3449,7 +3449,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.4"
|
||||
version = "0.41.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "c2pa-python" },
|
||||
@@ -3584,6 +3584,13 @@ qwen-zimage = [
|
||||
{ name = "torchvision" },
|
||||
{ name = "transformers" },
|
||||
]
|
||||
source-classify = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "opencv-python-headless", version = "4.14.0.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
]
|
||||
text-draft = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "paddleocr" },
|
||||
@@ -3637,6 +3644,7 @@ requires-dist = [
|
||||
{ name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'lama'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'migan'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'source-classify'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'text-draft'", specifier = ">=0.20.0" },
|
||||
{ name = "invisible-watermark", marker = "extra == 'dev'", specifier = ">=0.2.0" },
|
||||
{ name = "numpy", marker = "python_full_version >= '3.13' and extra == 'pixels'", specifier = ">=2" },
|
||||
@@ -3665,10 +3673,11 @@ requires-dist = [
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'classify'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'detect'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'diffusion'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'source-classify'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'visible'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["qwen-zimage", "lama"], marker = "extra == 'text-restoration'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video"], marker = "extra == 'dev'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video", "heif", "detect", "trustmark", "text-restoration", "migan", "classify"], marker = "extra == 'all'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video", "heif", "detect", "trustmark", "text-restoration", "migan", "classify", "source-classify"], marker = "extra == 'all'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'lama'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'migan'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'video'" },
|
||||
@@ -3684,7 +3693,7 @@ requires-dist = [
|
||||
{ name = "transformers", marker = "extra == 'diffusion'", specifier = ">=5,<6" },
|
||||
{ name = "trustmark", marker = "python_full_version < '3.13' and extra == 'trustmark'", specifier = ">=0.8.0" },
|
||||
]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "classify-onnx", "lama", "migan", "dev", "all"]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "classify-onnx", "source-classify", "lama", "migan", "dev", "all"]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
@@ -4067,10 +4076,10 @@ name = "torchmetrics"
|
||||
version = "1.9.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
{ name = "packaging" },
|
||||
{ name = "torch" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/34/39b8b749333db56c0585d7a11fa62a283c087bb1dfc897d69fb8cedbefb1/torchmetrics-1.9.0.tar.gz", hash = "sha256:a488609948600df52d3db4fcdab02e62aab2a85ef34da67037dc3e65b8512faa", size = 581765, upload-time = "2026-03-09T17:41:22.443Z" }
|
||||
wheels = [
|
||||
@@ -4165,13 +4174,13 @@ name = "trustmark"
|
||||
version = "0.9.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "einops", marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning", marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "omegaconf", marker = "python_full_version < '3.13'" },
|
||||
{ name = "six", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchvision", marker = "python_full_version < '3.13'" },
|
||||
{ name = "einops" },
|
||||
{ name = "lightning" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
{ name = "omegaconf" },
|
||||
{ name = "six" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchvision" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/53/41/32e84596af94e8696c55060d55eb9eed9d99722a673d12212a1c0c4ef1d2/trustmark-0.9.2.tar.gz", hash = "sha256:bcaa3bc38ead1c0558df81932672f34e306efaed31389da57dbed100e236b28a", size = 63780, upload-time = "2026-09-08T10:58:11.601Z" }
|
||||
|
||||
|
||||
Reference in new issue
Block a user