mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-19 15:37:16 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7021b900b | ||
|
|
3f55df6f99 | ||
|
|
4795149d34 | ||
|
|
cf30b4f95a | ||
|
|
8ce8bf14da | ||
|
|
3fb94b220d | ||
|
|
d2c5aa66fc | ||
|
|
12e9a87865 |
No files matched your search
@@ -129,6 +129,10 @@ Do not classify an entire module as untestable because its main path downloads a
|
||||
- the photo-classify gate in `test_classify.py`: DEFINITELY is ridge AND MLP,
|
||||
POSSIBLY does not emit a provider, and `identify` does not import
|
||||
`remove_ai_watermarks.classify`.
|
||||
- the source-export classifier in `test_source_classify.py`: the Hub revision
|
||||
and artifact hash are immutable, the pickle-free schema and exact GELU are
|
||||
validated, threshold mutation changes the real decision, and `identify`
|
||||
never consumes the result as watermark evidence.
|
||||
- the `InvisibleOptions` defaults, in `test_api.py`. When one signature promises to
|
||||
mirror another, compare them field by field rather than pinning the values you happen
|
||||
to know about, so the next field added on one side and not the other fails at the
|
||||
|
||||
@@ -7,8 +7,8 @@ name: Distribute on release
|
||||
# - HF Space: factory-rebuild the version already pinned in the Space repo.
|
||||
# - ComfyUI: sync, test, and publish the node against the exact library release.
|
||||
# - ClawHub: publish the agent skill when its own version moved.
|
||||
# All wait for the freshly published sdist to appear on PyPI first, since the
|
||||
# Release event fires in parallel with publish.yml's upload.
|
||||
# All wait for the freshly published distribution to appear in both PyPI APIs
|
||||
# first, since the Release event fires in parallel with publish.yml's upload.
|
||||
|
||||
on:
|
||||
release:
|
||||
@@ -22,6 +22,9 @@ on:
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
# Thirty attempts may spend up to 20 seconds each in the JSON request,
|
||||
# simple-index request, and inter-attempt delay. Leave bounded headroom.
|
||||
timeout-minutes: 25
|
||||
outputs:
|
||||
version: ${{ steps.v.outputs.version }}
|
||||
url: ${{ steps.sdist.outputs.url }}
|
||||
@@ -63,26 +66,35 @@ jobs:
|
||||
path: release-identity.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Wait for the sdist on PyPI, capture url + sha256
|
||||
- name: Wait for the PyPI JSON and simple indexes, capture sdist metadata
|
||||
id: sdist
|
||||
run: |
|
||||
set -euo pipefail
|
||||
v="${{ steps.v.outputs.version }}"
|
||||
for i in $(seq 1 30); do
|
||||
if rel=$(curl -fsSL "https://pypi.org/pypi/remove-ai-watermarks/$v/json" 2>/dev/null); then
|
||||
url=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
sha=$(echo "$rel" | python3 -c "import sys,json;d=json.load(sys.stdin);print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'),''))")
|
||||
if rel=$(curl --connect-timeout 10 --max-time 20 -fsSL "https://pypi.org/pypi/remove-ai-watermarks/$v/json" 2>/dev/null); then
|
||||
read -r url sha <<< "$(echo "$rel" | python3 -c "import sys,json;u=next((u for u in json.load(sys.stdin)['urls'] if u['packagetype']=='sdist'),{});print(u.get('url',''),u.get('digests',{}).get('sha256',''))")"
|
||||
if [ -n "$url" ] && [ -n "$sha" ]; then
|
||||
echo "url=$url" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "Found sdist for $v"
|
||||
exit 0
|
||||
# PyPI's JSON API can show a freshly uploaded file before the
|
||||
# simple index used by pip has caught up. Downstream release
|
||||
# jobs install by package version, so waiting on JSON alone
|
||||
# creates a release-time race for them.
|
||||
simple=$(curl --connect-timeout 10 --max-time 20 -fsSL https://pypi.org/simple/remove-ai-watermarks/ 2>/dev/null || true)
|
||||
artifact=${url##*/}
|
||||
if [[ "$simple" == *"$artifact"* ]]; then
|
||||
echo "url=$url" >> "$GITHUB_OUTPUT"
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "PyPI JSON and simple indexes list $v"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo "sdist for $v not on PyPI yet (attempt $i), waiting..."
|
||||
sleep 20
|
||||
if [ "$i" -lt 30 ]; then
|
||||
echo "PyPI indexes do not both list $v yet (attempt $i), waiting..."
|
||||
sleep 20
|
||||
fi
|
||||
done
|
||||
echo "::error::sdist for $v did not appear on PyPI in time"
|
||||
echo "::error::PyPI JSON and simple indexes did not both list $v in time"
|
||||
exit 1
|
||||
|
||||
homebrew:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Publish source classifier to Hugging Face
|
||||
|
||||
# Manual only. The derivative model weights live in a GitHub model-freeze
|
||||
# release; the private training corpus and caches never enter this repository.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
confirm:
|
||||
description: "Type publish"
|
||||
required: true
|
||||
weights_tag:
|
||||
description: "GitHub Release tag holding source-pipeline-mlp.npz"
|
||||
required: true
|
||||
default: source-classify-freeze-2026-09-16-r2
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: ${{ github.event.inputs.confirm == 'publish' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Download derivative model weights
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ github.event.inputs.weights_tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p weights
|
||||
gh release download "$TAG" \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pattern source-pipeline-mlp.npz \
|
||||
--dir weights
|
||||
test -s weights/source-pipeline-mlp.npz
|
||||
|
||||
- name: Verify the frozen artifact
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r .artifact_sha256 docs/source-classify-hf/metrics.json)"
|
||||
test "${#expected}" -eq 64
|
||||
echo "$expected weights/source-pipeline-mlp.npz" | sha256sum --check
|
||||
|
||||
- name: Publish to Hugging Face
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${HF_TOKEN:-}" ]; then
|
||||
echo "HF_TOKEN secret is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --no-project --with huggingface-hub \
|
||||
python scripts/publish_source_classify_hf.py \
|
||||
--src weights \
|
||||
--message "Source-classify freeze (GitHub ${SHA:0:7})"
|
||||
@@ -43,7 +43,7 @@ The published Agent Skill is [`skills/remove-ai-watermarks/`](skills/remove-ai-w
|
||||
|
||||
[`docs/module-internals.md`](docs/module-internals.md) is the canonical per-module map, including design decisions, thresholds, calibration history, incident records, and regression guards. Read the relevant section before changing a subsystem.
|
||||
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); it is not `identify`. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
Research and current constraints are routed through [`docs/index.md`](docs/index.md), especially [`docs/known-limitations.md`](docs/known-limitations.md), [`docs/supported-signals.md`](docs/supported-signals.md), [`docs/synthid.md`](docs/synthid.md), and [`docs/watermarking-landscape.md`](docs/watermarking-landscape.md). Pixel photo classification is [`docs/photo-classify.md`](docs/photo-classify.md); the separate OpenAI/Google/unknown export-pipeline signal is [`docs/source-classify.md`](docs/source-classify.md). Neither runs inside `identify` or constitutes SynthID detection. Classifier research is split between general [`AI-generated image classifiers`](docs/ai-generated-image-classifiers.md) and [`SynthID source classifiers`](docs/synthid-classifiers.md). Other SynthID campaign logs are [`docs/synthid-detector-research.md`](docs/synthid-detector-research.md) and [`docs/synthid-removal-research.md`](docs/synthid-removal-research.md). The pre-split chronological archive is [`docs/synthid-detector-removal-plan.md`](docs/synthid-detector-removal-plan.md). The benchmark kernel and its pinned local oracles (AudioSeal, VideoSeal) cover the audio/video watermark cohorts and research studies: [`docs/benchmark-kernel.md`](docs/benchmark-kernel.md).
|
||||
|
||||
## Data safety
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ specialized Python API can inspect and disrupt the validated local
|
||||
| --- | --- |
|
||||
| Metadata inspection and stripping | `remove-ai-watermarks` |
|
||||
| Photograph AI-versus-camera classification | `remove-ai-watermarks[classify]` |
|
||||
| OpenAI/Google/unknown source-export classification | `remove-ai-watermarks[source-classify]` |
|
||||
| Visible detection and removal | `remove-ai-watermarks[visible]` |
|
||||
| Visible video processing | `remove-ai-watermarks[video]` |
|
||||
| Video SynthID removal | `remove-ai-watermarks[video,diffusion]` |
|
||||
@@ -68,7 +69,7 @@ specialized Python API can inspect and disrupt the validated local
|
||||
| Every production feature available on the active Python | `remove-ai-watermarks[all]` |
|
||||
|
||||
Lower-level and specialized extras include `pixels`, `heif`, `trustmark`,
|
||||
`migan`, `lama`, `diffusion`, and `classify-onnx`. The
|
||||
`migan`, `lama`, `diffusion`, `classify-onnx`, and `source-classify`. The
|
||||
[installation guide](docs/installation.md#feature-extras) documents their exact
|
||||
dependency composition, Python compatibility, and model requirements.
|
||||
|
||||
@@ -96,6 +97,19 @@ remove-ai-watermarks classify image.png
|
||||
|
||||
Guide: [photo pixel classification](docs/photo-classify.md).
|
||||
|
||||
For a lightweight, abstaining OpenAI/Google/unknown source-export signal after
|
||||
metadata removal, use the separate Python API. It is not a SynthID detector:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("image.png")
|
||||
print(result.label, result.reason)
|
||||
```
|
||||
|
||||
Install `remove-ai-watermarks[source-classify]`. Guide:
|
||||
[source-pipeline classification](docs/source-classify.md).
|
||||
|
||||
Signed provenance is the supported route for SynthID and `identify` reads it.
|
||||
There is no local SynthID pixel detector in the package. Research on a
|
||||
periodic lattice expert is in `scripts/synthid_runtime/` and
|
||||
|
||||
@@ -13,6 +13,7 @@ to run the tool. Use the maintainer references only when changing the code.
|
||||
| [Supported signals](supported-signals.md) | You need to know which visible marks, metadata formats, and invisible signals are covered. |
|
||||
| [Known limitations](known-limitations.md) | You need the quality, device, format, or verification boundaries. |
|
||||
| [Photo pixel classification](photo-classify.md) | You want `classify` / `classify_pixels`: AI versus camera, optional provider. |
|
||||
| [Source-pipeline classification](source-classify.md) | You want an abstaining OpenAI/Google/unknown export-pipeline signal after metadata removal. |
|
||||
| [Photo classifier training](photo-classify-training.md) | You need the retrain pack, public sources, and what is not published. |
|
||||
| [Scope, safety, and legal notes](legal-and-safety.md) | You need the intended use and legal context. |
|
||||
|
||||
@@ -42,6 +43,7 @@ The current behavior is defined by the code, tests, README, and user guides.
|
||||
- [AI-generated image classifiers](ai-generated-image-classifiers.md) (photo AI-versus-camera freeze consumed by `classify_pixels`, not `identify`)
|
||||
- [Photo classifier Hugging Face card](photo-classify-hf/README.md) (Hub publication text of the 2026-08-31 freeze)
|
||||
- [SynthID source classifiers](synthid-classifiers.md) (metadata-free OpenAI/Gemini source finding and provider-lineage experiments)
|
||||
- [Source-pipeline classifier model card](source-classify-hf/README.md) (published derivative weights, aggregate evaluation, and intended-use boundary)
|
||||
- [SynthID local detector research](synthid-detector-research.md)
|
||||
- [SynthID mark removal research](synthid-removal-research.md)
|
||||
- [SynthID identity research](synthid-robust-identity-research.md)
|
||||
|
||||
@@ -129,6 +129,7 @@ application actually uses:
|
||||
| `trustmark` | Adobe TrustMark detection on Python 3.11-3.12 | trustmark | Yes |
|
||||
| `classify` | Metadata-free photo AI-versus-camera classifier plus gated provider | `pixels`, Torch, Transformers | Yes |
|
||||
| `classify-onnx` | Optional CPU-only ONNX vision runtime for `classify` | `classify`, ONNX Runtime | Yes |
|
||||
| `source-classify` | Abstaining OpenAI/Google/unknown source-export classifier, not SynthID detection | `pixels`, Hugging Face Hub | Model download, no Torch |
|
||||
| `diffusion` | Torch and Diffusers runtime; video SynthID regeneration | `pixels`, Torch, Diffusers | Yes |
|
||||
| `migan` | MI-GAN ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
| `lama` | big-LaMa ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch |
|
||||
@@ -156,6 +157,7 @@ flowchart LR
|
||||
trustmark
|
||||
classify --> pixels
|
||||
onnx["classify-onnx"] --> classify
|
||||
source["source-classify"] --> pixels
|
||||
```
|
||||
|
||||
`heif`, `trustmark`, and `text-draft` are independent branches. Combine them
|
||||
|
||||
@@ -20,6 +20,14 @@ The command is never started by `identify`. A no-signal provenance result stays
|
||||
unknown. This is not a clean verdict and it does not run cleanup. Full guide:
|
||||
[photo pixel classification](photo-classify.md).
|
||||
|
||||
`classify_source` is a different, lightweight OpenAI/Google/unknown
|
||||
source-export classifier. It is useful only while the pixels still resemble an
|
||||
original export. A 95% resize reduced measured Google recall to 1%, and a JPEG
|
||||
quality-75 round trip made every public evaluation row abstain. Its provider
|
||||
label is not evidence that SynthID is present, and `unknown` is not evidence
|
||||
that SynthID is absent. It is also never started by `identify`. Full guide:
|
||||
[source-pipeline classification](source-classify.md).
|
||||
|
||||
## Visible removal
|
||||
|
||||
### Fill quality depends on the background
|
||||
|
||||
@@ -1116,6 +1116,30 @@ both backends explicitly (`tests/test_classify.py` pins the seam). User guide:
|
||||
[photo-classify.md](photo-classify.md). Hub card:
|
||||
[photo-classify-hf/README.md](photo-classify-hf/README.md).
|
||||
|
||||
### Source-pipeline classifier
|
||||
|
||||
[`source_classify.py`](../src/remove_ai_watermarks/source_classify.py) is a
|
||||
separate, lightweight OpenAI/Google/unknown source-export classifier. It fuses
|
||||
the existing 124-dimensional patch descriptor with the 768-dimensional
|
||||
phase-free spectrum in
|
||||
[`_internal/source_spectral.py`](../src/remove_ai_watermarks/_internal/source_spectral.py),
|
||||
then evaluates the frozen 892 -> 128 -> 64 -> 3 GELU MLP in NumPy. Provider
|
||||
labels clear independent calibrated margins; every other row is `unknown`.
|
||||
The NumPy decision treats values within `1e-5` of the stored margin as the
|
||||
inclusive training threshold, absorbing float32 BLAS differences; this matches
|
||||
the frozen training-runtime decisions on all 27,811 cached internal and
|
||||
external feature rows.
|
||||
|
||||
The runtime pins both the immutable Hub revision and the NPZ SHA-256, loads
|
||||
with `allow_pickle=False`, and rejects any field, shape, class, activation, or
|
||||
non-finite-value drift. `RAIW_SOURCE_CLASSIFY_WEIGHTS` is the offline override.
|
||||
The `source-classify` extra adds `pixels` and Hugging Face Hub but no Torch or
|
||||
Transformers. `identify` never imports or runs this path. Its output names a
|
||||
complete source/export pipeline and is not SynthID detection. Unit contracts
|
||||
and the threshold mutation guard live in `tests/test_source_classify.py`;
|
||||
evaluation and attack limits live in
|
||||
[`synthid-classifiers.md`](synthid-classifiers.md).
|
||||
|
||||
The CLIP loader suppresses discarded random parameter initialization with
|
||||
Transformers' `no_init_weights` context before applying the complete frozen
|
||||
state dict. Normal `CLIPModel(config)` construction initialized 427 million
|
||||
@@ -2272,6 +2296,17 @@ resolution case -- every verdict so far comes from one fixture and one seed.
|
||||
Both ends of the shipped curve now have oracle verdicts, and the shipped curve clears
|
||||
everything it has been tested at:
|
||||
|
||||
A source-fresh GPT Image 2.5 check on 2026-09-15 exercised the complete shipped
|
||||
profile rather than only a strength sweep. One 1024 x 1024 API generation from
|
||||
Flare and one from Sunburst both validated as current OpenAI C2PA, routed to the
|
||||
flat OpenAI strength, and remained OpenAI SynthID-positive after pixel-identical
|
||||
metadata removal. The complete `qwen-zimage` profile at 0.07675, seed 0, cleared
|
||||
both in the official OpenAI Content Provenance API. The Flare output measured
|
||||
37.48 dB PSNR / 0.9329 SSIM and the Sunburst output 37.01 dB / 0.9179 against
|
||||
their sources. Those fidelity numbers describe two neutral studio-photo
|
||||
carriers, not a content-wide quality estimate, and the source-positive/output-negative
|
||||
oracle pairs do not replace the wider OpenAI calibration that set the floor.
|
||||
|
||||
| oracle | fixture size | detected at | clean from |
|
||||
|---|---|---|---|
|
||||
| openai.com/verify | 1.57 MP | 0.06 | 0.08 |
|
||||
|
||||
+18
-1
@@ -6,12 +6,29 @@ and pipeline modules are intended for maintainers and specialized workflows.
|
||||
Dependency groups are identical for the CLI and Python API. The default install
|
||||
covers metadata extraction, normalization, verdict logic, and stripping.
|
||||
Array/pixel APIs use `pixels`; visible removal uses `visible`; DWT-DCT detection
|
||||
uses `detect`; pixel photo classification uses `classify`; invisible image removal uses `qwen-zimage` and an NVIDIA GPU; and
|
||||
uses `detect`; pixel photo classification uses `classify`; lightweight source-pipeline classification uses `source-classify`; invisible image removal uses `qwen-zimage` and an NVIDIA GPU; and
|
||||
visible video processing uses `video`. The video-pixel SynthID-removal profile is
|
||||
a separate VAE path that still runs on CPU and combines `video` and `diffusion`. Add `heif`
|
||||
independently when path-based pixel APIs must decode HEIC, HEIF, or AVIF. See
|
||||
the complete [feature-extra matrix](installation.md#feature-extras).
|
||||
|
||||
## Classify a source/export pipeline
|
||||
|
||||
Install `remove-ai-watermarks[source-classify]`, then call the explicit API:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("metadata-stripped.png")
|
||||
print(result.label) # "openai" | "google" | "unknown"
|
||||
print(result.reason) # "classified" | "abstained" | "feature_unavailable"
|
||||
```
|
||||
|
||||
This signal classifies a complete image-generation and export pipeline. It is
|
||||
not a SynthID detector and does not establish watermark presence or absence.
|
||||
See [source-pipeline classification](source-classify.md) for the result fields,
|
||||
model pin, evaluation boundary, and offline weights override.
|
||||
|
||||
## Remove visible marks
|
||||
|
||||
Install `remove-ai-watermarks[visible]` before using the visible-removal API.
|
||||
|
||||
@@ -92,7 +92,17 @@ alone does not upload new heads -- bump the pinned Hub revision when a new
|
||||
head must reach installed clients. Pin a Hub revision in production so a
|
||||
later freeze cannot silently change installed clients.
|
||||
|
||||
The workflow can also be started manually with an optional version input.
|
||||
The separate `wiltodelta/raiw-source-classify` model is published with
|
||||
`.github/workflows/publish-source-classify-hf.yml`. The workflow downloads the
|
||||
derivative `source-pipeline-mlp.npz` artifact from the declared GitHub model
|
||||
freeze, verifies its SHA-256, and uploads only that file plus the tracked card
|
||||
and aggregate metrics under `docs/source-classify-hf/`. The private corpus,
|
||||
catalog, paths, hashes, embeddings, and feature caches are outside this flow.
|
||||
|
||||
The workflow is manual and requires the model-freeze release tag plus an
|
||||
explicit `publish` confirmation. Keep model-freeze GitHub releases marked as
|
||||
pre-releases so a model artifact cannot replace the latest package release in
|
||||
GitHub's release UI.
|
||||
|
||||
If a distribution job fails because a repository or Hugging Face credential is
|
||||
invalid, rotate the corresponding GitHub secret and rerun the failed job. A
|
||||
@@ -255,13 +265,15 @@ lists it: uv serves its cached pre-release view of the index. Re-run with a
|
||||
fresh cache (`UV_CACHE_DIR=$(mktemp -d)`) before treating it as a release
|
||||
failure.
|
||||
|
||||
The ComfyUI sync run can fail the same way on its own: it resolves the version
|
||||
from the PyPI JSON API (which updates first) but installs the test dependency
|
||||
with pip against the simple index, whose CDN edges lag by minutes. The
|
||||
`distribute.yml` waiter also has a clock: 0.40.0's node suite finished after
|
||||
the 15-minute poll, so the job now waits 30 minutes and then accepts a
|
||||
registry row that already requires this library version. Rerun the failed
|
||||
`comfyui` job only when the registry still has no such row.
|
||||
The ComfyUI sync resolves the version from the PyPI JSON API but installs its
|
||||
test dependency through pip's simple index, whose CDN edges can lag by
|
||||
minutes. `distribute.yml` gates every downstream job on both APIs listing the
|
||||
release, so it cannot dispatch that race again. The ComfyUI waiter itself has
|
||||
a clock: 0.40.0's node suite finished after the 15-minute poll, so it now
|
||||
waits 30 minutes and then accepts a registry row that already requires this
|
||||
library version. For an older distribution that started before the simple
|
||||
index caught up, rerun its failed `comfyui` job only when the registry still
|
||||
has no such row.
|
||||
|
||||
The automatic `verify-release.yml` run after `distribute.yml` checks the
|
||||
Hugging Face Space pin. When `RAIW_HF_SPACE_TOKEN` is set, that pin lands
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
library_name: numpy
|
||||
pipeline_tag: image-classification
|
||||
tags:
|
||||
- image-forensics
|
||||
- source-attribution
|
||||
- open-set-classification
|
||||
---
|
||||
|
||||
# RAIW source-pipeline classifier
|
||||
|
||||
This is a compact, abstaining pixel-forensics classifier for three labels:
|
||||
`openai`, `google`, and `unknown`. It is intended for original-export-looking
|
||||
images whose metadata may have been removed without changing the pixels.
|
||||
|
||||
This model is **not a SynthID detector**. It does not decode or verify a
|
||||
watermark payload. Its labels describe complete generation and export
|
||||
pipelines, so renderer, geometry, encoding, post-processing, and watermark
|
||||
state may be confounded. `unknown` does not mean that SynthID is absent, and a
|
||||
provider label does not prove that SynthID is present.
|
||||
|
||||
## Model
|
||||
|
||||
- Input: a 124-dimensional residual/spectral forensic feature vector plus a
|
||||
768-dimensional phase-free spectral vector.
|
||||
- Head: 892 -> 128 -> 64 -> 3 MLP with GELU activations.
|
||||
- Output: `openai`, `google`, or abstain as `unknown` using independent
|
||||
provider margins.
|
||||
- Artifact: `source-pipeline-mlp.npz`
|
||||
- SHA-256: `6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a`
|
||||
|
||||
The artifact contains only learned parameters, normalization constants,
|
||||
labels, and thresholds. It does not contain training images, paths, image
|
||||
hashes, embeddings, or a training catalog.
|
||||
|
||||
## Evaluation
|
||||
|
||||
The operating point was selected on an internal calibration split. The locked
|
||||
internal test and the public `bigdatamark/synthid-research` dataset were not
|
||||
used for architecture, checkpoint, or threshold selection.
|
||||
|
||||
| Evaluation | Class | Recall | Precision |
|
||||
|---|---|---:|---:|
|
||||
| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% |
|
||||
| Locked internal test (5,369 images) | Google | 57.6% | 87.9% |
|
||||
| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% |
|
||||
| Public external originals (900 images) | OpenAI | 68.7% | 100.0% |
|
||||
| Public external originals (900 images) | Google | 81.3% | 98.8% |
|
||||
| Public external originals (900 images) | Unknown/Flux | 99.0% | 66.4% |
|
||||
|
||||
On the public external set, a 95% resize reduced Google recall to 1.0%, and a
|
||||
JPEG quality-75 round trip caused the model to abstain on every image. These
|
||||
are expected abstentions, not evidence that a watermark was removed.
|
||||
|
||||
## Intended use
|
||||
|
||||
Use this model as one probabilistic signal in a broader forensic report or to
|
||||
suggest an appropriate processing path. Do not use it to claim watermark
|
||||
presence, authorship, policy compliance, fraud, or provenance. Applications
|
||||
should expose the abstention and the model's narrow source-pipeline claim.
|
||||
|
||||
Feature extraction and inference are versioned in the
|
||||
`remove-ai-watermarks` Python project rather than in this model repository.
|
||||
Use its explicit `classify_source` API; `identify` does not run this model.
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"claim": "abstaining OpenAI/Google source-pipeline classification; not SynthID detection",
|
||||
"artifact_sha256": "6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a",
|
||||
"classes": ["openai", "google", "unknown"],
|
||||
"locked_internal_test": {
|
||||
"n": 5369,
|
||||
"confusion_actual_by_predicted": [[230, 6, 145], [0, 217, 160], [41, 24, 4546]],
|
||||
"recall": {"openai": 0.6036745407, "google": 0.5755968170, "unknown": 0.9859032748},
|
||||
"precision": {"openai": 0.8487084871, "google": 0.8785425101, "unknown": 0.9371263657}
|
||||
},
|
||||
"public_external_originals": {
|
||||
"dataset": "bigdatamark/synthid-research",
|
||||
"n": 900,
|
||||
"confusion_actual_by_predicted": [[206, 0, 94], [0, 244, 56], [0, 3, 297]],
|
||||
"recall": {"openai": 0.6866666667, "google": 0.8133333333, "unknown": 0.99},
|
||||
"precision": {"openai": 1.0, "google": 0.9878542510, "unknown": 0.6644295302}
|
||||
},
|
||||
"selection": {
|
||||
"minimum_calibration_precision": {"openai": 0.99, "google": 0.95},
|
||||
"maximum_calibration_unknown_false_positive_rate": 0.01
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# Source-pipeline classification
|
||||
|
||||
`classify_source` is an opt-in, metadata-free pixel classifier for images that
|
||||
still resemble an original provider export. It returns `openai`, `google`, or
|
||||
an explicit `unknown` abstention.
|
||||
|
||||
This is not a SynthID detector. It does not decode or verify a watermark
|
||||
payload, and it must not be used to claim that SynthID is present or absent.
|
||||
The model describes the complete source/export pipeline, where renderer,
|
||||
geometry, encoding, post-processing, and watermark state can be confounded.
|
||||
|
||||
Install the lightweight runtime:
|
||||
|
||||
```bash
|
||||
uv add "remove-ai-watermarks[source-classify]"
|
||||
```
|
||||
|
||||
Call the Python API explicitly:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.classify_source("metadata-stripped.png")
|
||||
print(result.label) # "openai" | "google" | "unknown"
|
||||
print(result.reason) # "classified" | "abstained" | "feature_unavailable"
|
||||
print(result.margins) # provider margins before thresholding
|
||||
print(result.thresholds) # frozen provider thresholds
|
||||
```
|
||||
|
||||
`feature_unavailable` is returned for images below the required 256x256
|
||||
geometry or for a degenerate feature vector. `abstained` means neither provider
|
||||
cleared its independent calibrated margin. Both are honest `unknown` results.
|
||||
|
||||
The first call downloads the exact pinned revision of
|
||||
[`wiltodelta/raiw-source-classify`](https://huggingface.co/wiltodelta/raiw-source-classify).
|
||||
Set `RAIW_SOURCE_CLASSIFY_WEIGHTS` to the NPZ itself or a directory containing
|
||||
`source-pipeline-mlp.npz` for an offline deployment.
|
||||
|
||||
The model is useful as a routing suggestion for a processing workflow or as one
|
||||
probabilistic signal in a broader forensic report. Do not use it as evidence of
|
||||
authorship, fraud, policy compliance, provenance, or watermark presence. The
|
||||
published model card records the evaluation and robustness limits.
|
||||
@@ -121,6 +121,10 @@ entries enter the watermark inventory, while a fingerprint remains a
|
||||
re-linkability warning, never selects pixel regeneration, and does not suppress
|
||||
independent SynthID evidence.
|
||||
|
||||
The opt-in `classify_source` API is not an `identify` signal. It can suggest an
|
||||
OpenAI or Google source/export pipeline after metadata removal, but it neither
|
||||
detects nor decodes a watermark and never changes the provenance verdict.
|
||||
|
||||
## File and container formats
|
||||
|
||||
Pixel based image commands discover these extensions:
|
||||
@@ -171,7 +175,11 @@ and `scripts/synthid_runtime/`.
|
||||
The tool recognizes presence from supported provenance: Google AI C2PA
|
||||
under Google's all-media watermark policy, and current OpenAI C2PA carrying an
|
||||
explicit `c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action
|
||||
does not assert SynthID.
|
||||
does not assert SynthID. GPT Image 2.5 Flare and Sunburst were verified on
|
||||
2026-09-15: both API outputs carried valid OpenAI C2PA with
|
||||
`c2pa.watermarked.unbound` and exercised the existing model-independent OpenAI
|
||||
route. Automatic routing selected `qwen-zimage` with the OpenAI strength cohort,
|
||||
and both outputs cleared the official OpenAI SynthID check.
|
||||
|
||||
Provider verifiers and provenance APIs are reserved for development-time
|
||||
calibration. They are not exposed through the installed CLI, the top-level
|
||||
|
||||
@@ -62,6 +62,91 @@ support treating augmentation and architecture shift as first-class tests,
|
||||
rather than reading an untransformed source-classification score as watermark
|
||||
evidence.
|
||||
|
||||
### Abstaining fused source classifier, 2026-09-15
|
||||
|
||||
The follow-up experiment combined the existing 124-d forensic descriptor with
|
||||
the 16x16 phase-free spectral grid in a class-balanced ridge head. Its three
|
||||
outputs are `openai`, `google`, and `unknown`; a provider result is emitted only
|
||||
when it beats both the other provider and `unknown` by a margin selected on a
|
||||
separate calibration split. The public implementation is
|
||||
[`scripts/source_pipeline_classifier.py`](../scripts/source_pipeline_classifier.py).
|
||||
It includes training, strict pickle-free model loading, and JSON inference, but
|
||||
no fitted weights: the experimental training freeze contains inputs that are
|
||||
not redistributable. This keeps the code reproducible without publishing a
|
||||
derivative artifact whose data provenance does not permit redistribution.
|
||||
|
||||
Penalty 100 and margin 0.6392407698660646 were selected using only the training
|
||||
and calibration partitions. The 2,669-image locked internal test and Mark's
|
||||
900-image public set were not used for feature, penalty, or threshold selection.
|
||||
Feature-extraction failures count as `unknown` in every result below.
|
||||
|
||||
| Evaluation | Accuracy | OpenAI recall | Google recall | Unknown recall |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Locked internal, 2,669 images | 84.2% | 60.9% | 48.8% | 95.8% |
|
||||
| Mark original exports, 900 images | 90.1% | 76.3% | 94.3% | 99.7% |
|
||||
| Mark crop 2 px | 89.8% | 76.7% | 94.3% | 98.3% |
|
||||
| Mark resize 95% | 59.7% | 81.7% | 0.0% | 97.3% |
|
||||
| Mark JPEG quality 75 | 33.3% | 0.0% | 0.0% | 100.0% |
|
||||
| Mark blur radius 0.7 | 68.3% | 42.3% | 63.0% | 99.7% |
|
||||
|
||||
The untouched-export result is useful source-attribution evidence, but the
|
||||
transforms decide the claim. Resizing eliminated the Google branch, JPEG made
|
||||
the classifier abstain on every row, and blur materially reduced both provider
|
||||
recalls. The locked unknown false-attribution rate was 4.2%, versus 0.9% on the
|
||||
calibration split. This is therefore an inspectable research baseline, not a
|
||||
runtime detector or a product watermark verdict.
|
||||
|
||||
Training requires hash-disjoint `train` and `calibration` directories, each
|
||||
with `openai`, `google`, and `unknown` subdirectories:
|
||||
|
||||
```bash
|
||||
uv run python scripts/source_pipeline_classifier.py train \
|
||||
/path/to/train /path/to/calibration --model-out .local-eval/source-pipeline.npz
|
||||
|
||||
uv run python scripts/source_pipeline_classifier.py predict \
|
||||
.local-eval/source-pipeline.npz image.png
|
||||
```
|
||||
|
||||
A stronger public release needs a publication-cleared multi-era training set
|
||||
and a causal marked/unmarked contrast. Mark's same-prompt triples remain an
|
||||
external source/export test, not evidence that the fitted features read the
|
||||
SynthID payload.
|
||||
|
||||
### Full-corpus source-pipeline model, 2026-09-16
|
||||
|
||||
The next experiment trained a compact nonlinear head on the complete available
|
||||
private corpus, including target-provider exports, photographs, and foreign
|
||||
generator negatives. The corpus itself is not published. The released
|
||||
`source-pipeline-mlp.npz` is a derivative artifact containing only learned
|
||||
parameters, normalization constants, labels, and abstention thresholds; it
|
||||
contains no images, paths, image hashes, embeddings, or training catalog.
|
||||
|
||||
The model joins the 124-dimensional forensic descriptor with the
|
||||
768-dimensional phase-free spectral descriptor and applies a
|
||||
892 -> 128 -> 64 -> 3 GELU MLP. Checkpoint and thresholds were selected on a
|
||||
separate calibration partition. The 5,369-image hash-disjoint locked test and
|
||||
Mark's public 900-image originals were not used for architecture, checkpoint,
|
||||
or threshold selection.
|
||||
|
||||
| Evaluation | Class | Recall | Precision |
|
||||
| --- | --- | ---: | ---: |
|
||||
| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% |
|
||||
| Locked internal test (5,369 images) | Google | 57.6% | 87.9% |
|
||||
| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% |
|
||||
| Mark public originals (900 images) | OpenAI | 68.7% | 100.0% |
|
||||
| Mark public originals (900 images) | Google | 81.3% | 98.8% |
|
||||
| Mark public originals (900 images) | Unknown/FLUX | 99.0% | 66.4% |
|
||||
|
||||
A 95% resize reduced Google recall on the public set to 1%, and a JPEG
|
||||
quality-75 round trip made every row abstain. This behavior matches the narrow
|
||||
claim: the model classifies original-looking source/export pipelines after
|
||||
metadata removal. It neither decodes nor detects SynthID, and its `unknown`
|
||||
label does not mean that SynthID is absent. The model card and aggregate
|
||||
metrics are tracked under
|
||||
[`docs/source-classify-hf/`](source-classify-hf/README.md). Runtime integration
|
||||
is provided by the explicit `classify_source` library API and never feeds the
|
||||
`identify` watermark verdict.
|
||||
|
||||
Krawetz's Gemini-chat TPR critique is a verifier-quality claim, not a
|
||||
feature we can ship. [Lead Stories, 2026-07](https://leadstories.com/analysis/2026/07/google-gemini-synthid-detector-confuses-results-within-same-chat.html)
|
||||
documented Gemini repeating the first file's SynthID verdict inside a
|
||||
|
||||
@@ -775,6 +775,20 @@ SynthID. Current OpenAI manifests distinguish the watermarked output with the
|
||||
signed `c2pa.watermarked.unbound` action. The detector requires that action, so
|
||||
old OpenAI C2PA remains an origin signal without becoming a SynthID claim.
|
||||
|
||||
**GPT Image 2.5 compatibility, verified 2026-09-15.** One fresh low-quality
|
||||
1024 x 1024 API generation from each released model, `gpt-image-2.5-flare` and
|
||||
`gpt-image-2.5-sunburst`, carried a valid bound OpenAI C2PA claim generated by
|
||||
`OpenAI Media Service API`, with `trainedAlgorithmicMedia` and the action set
|
||||
`created, converted, watermarked.unbound`. `identify` reported high-confidence
|
||||
OpenAI provenance plus SynthID, `has_invisible_target` returned true, and the
|
||||
automatic strength cohort resolved to OpenAI for both files. After C2PA was
|
||||
removed without changing decoded pixels, the official OpenAI Content Provenance
|
||||
API still returned SynthID `detected` for both sources. The complete shipped
|
||||
`qwen-zimage` profile at strength 0.07675 and seed 0 then produced two outputs
|
||||
for which the same metadata-independent API returned SynthID `not_detected`.
|
||||
This is one controlled source per model and proves the current routing and
|
||||
operating point on those two sources; it is not a provider-wide removal rate.
|
||||
|
||||
### 4.3 Other vendors
|
||||
|
||||
- **Kakao** (South Korea): SynthID adopter as of May 2026 (Google announcement)
|
||||
|
||||
+9
-2
@@ -4,7 +4,7 @@
|
||||
# on it, including the ComfyUI node package. The console script below carries
|
||||
# the same weight, since users have it on PATH.
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.5"
|
||||
version = "0.41.0"
|
||||
description = "AI watermark remover for visible, invisible, and provenance marks in images and video"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11,<3.15"
|
||||
@@ -160,6 +160,13 @@ classify-onnx = [
|
||||
"remove-ai-watermarks[classify]",
|
||||
"onnxruntime>=1.24.0",
|
||||
]
|
||||
# Compact OpenAI/Google/unknown source-export classifier. This is intentionally
|
||||
# separate from the heavier photo AI-versus-camera classifier and is not a
|
||||
# SynthID detector. Weights download on first use.
|
||||
source-classify = [
|
||||
"remove-ai-watermarks[pixels]",
|
||||
"huggingface-hub>=0.20.0",
|
||||
]
|
||||
# Universal region eraser backend -- big-LaMa via onnxruntime (Carve/LaMa-ONNX,
|
||||
# Apache-2.0). CPU, no torch. Model (~200 MB) is downloaded on first use and
|
||||
# cached by huggingface_hub; it is never bundled in this repo. The default cv2
|
||||
@@ -205,7 +212,7 @@ dev = [
|
||||
]
|
||||
# ``qwen-zimage`` already pulls ``diffusion``; naming both would suggest diffusion is
|
||||
# independently sufficient for a removal, which it is not.
|
||||
all = ["remove-ai-watermarks[video,heif,detect,trustmark,text-restoration,migan,classify]"]
|
||||
all = ["remove-ai-watermarks[video,heif,detect,trustmark,text-restoration,migan,classify,source-classify]"]
|
||||
|
||||
[project.scripts]
|
||||
remove-ai-watermarks = "remove_ai_watermarks.cli:main"
|
||||
|
||||
@@ -51,9 +51,11 @@ explicitly names a tracked canonical result.
|
||||
| `visible_sheets.py` | Build blinded contact sheets for relaxation candidates. |
|
||||
| `watermark_benchmark_cohort.py` | Build the deterministic synthetic DWT-DCT and TrustMark cohort consumed by `watermark_benchmark.py`. |
|
||||
| `spectral_pipeline_probe.py` | Measure a prompt-grouped, phase-free spectral source-pipeline hypothesis without presenting it as SynthID detection. |
|
||||
| `source_pipeline_classifier.py` | Train or run an abstaining OpenAI/Google/unknown source-export classifier; research-only and not SynthID detection. |
|
||||
| `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. |
|
||||
| `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. |
|
||||
| `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. |
|
||||
| `publish_source_classify_hf.py` | Upload the derivative source-pipeline model and its public card to Hugging Face `wiltodelta/raiw-source-classify`. Manual; the Action `publish-source-classify-hf.yml` is the write-token path. |
|
||||
| `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. |
|
||||
|
||||
## Research and diagnostic prototypes
|
||||
@@ -86,5 +88,7 @@ running the sync command without `--check` is the explicit snapshot update.
|
||||
|
||||
`_plain_console.py` provides plain-text fallbacks for Rich output, and
|
||||
`_text_eval.py` contains normalization helpers shared by text-evaluation scripts.
|
||||
`hf_publish.py` owns the common authenticated, allowlisted Hugging Face model
|
||||
repository upload used by the two model publishers.
|
||||
`engine_selection_manifest.py` owns the content-matrix schema and validation
|
||||
shared by its integrity check and the real-image benchmark builder.
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Shared Hugging Face model-repository publication helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def publish_model_folder(
|
||||
stage: Path,
|
||||
*,
|
||||
repo_id: str,
|
||||
token: str,
|
||||
message: str,
|
||||
allow_patterns: Iterable[str],
|
||||
) -> str:
|
||||
"""Upload an explicit file allowlist and return the Hub revision."""
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(repo_id, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
commit = HfApi(token=token).upload_folder(
|
||||
folder_path=str(stage),
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
allow_patterns=list(allow_patterns),
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
@@ -24,6 +24,8 @@ import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-photo-classify"
|
||||
@@ -92,18 +94,13 @@ def stage_weights(dest: Path, src: Path) -> None:
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(HUB_REPO, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
api = HfApi(token=token)
|
||||
commit = api.upload_folder(
|
||||
folder_path=str(stage),
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=["README.md", "operating-point.json", *WEIGHT_FILES, GATE_FILE, GATE_STABLE_FILE],
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish the source-pipeline model to Hugging Face.
|
||||
|
||||
The derivative weight file is downloaded by the GitHub Action from a model
|
||||
freeze release. The private corpus, feature caches, and training catalog never
|
||||
enter this repository or the Hub staging directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-source-classify"
|
||||
MODEL_FILE = "source-pipeline-mlp.npz"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
CARD_DIR = REPO_ROOT / "docs" / "source-classify-hf"
|
||||
PUBLIC_FILES = ("README.md", "metrics.json", MODEL_FILE)
|
||||
|
||||
|
||||
def stage_release(dest: Path, src: Path) -> None:
|
||||
"""Stage the two tracked documents and one derivative model artifact."""
|
||||
sources = {
|
||||
"README.md": CARD_DIR / "README.md",
|
||||
"metrics.json": CARD_DIR / "metrics.json",
|
||||
MODEL_FILE: src.expanduser().resolve() / MODEL_FILE,
|
||||
}
|
||||
missing = [str(path) for path in sources.values() if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit(f"missing publication files: {', '.join(missing)}")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for name, path in sources.items():
|
||||
shutil.copy2(path, dest / name)
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
"""Upload the exact staged allowlist and return the Hub revision."""
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=PUBLIC_FILES,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--src", type=Path, required=True)
|
||||
parser.add_argument("--message", default="Publish source-pipeline classifier")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
token = os.environ.get("HF_TOKEN", "").strip()
|
||||
if not token:
|
||||
raise SystemExit("HF_TOKEN is not set (write role required)")
|
||||
with tempfile.TemporaryDirectory(prefix="raiw-source-hf-") as tmp:
|
||||
stage = Path(tmp)
|
||||
stage_release(stage, args.src)
|
||||
revision = publish(stage, token=token, message=args.message)
|
||||
log.info("published %s revision=%s", HUB_REPO, revision or "(unknown)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,371 @@
|
||||
"""Train or run an abstaining source/export-pipeline classifier.
|
||||
|
||||
This research tool combines the library's 124-d forensic descriptor with the
|
||||
phase-free spectral descriptor from ``spectral_pipeline_probe.py``. Its labels
|
||||
describe complete generation and export pipelines. It does not decode SynthID
|
||||
or establish that a watermark is present.
|
||||
|
||||
Expected training layout::
|
||||
|
||||
train/openai/*.png
|
||||
train/google/*.png
|
||||
train/unknown/*.png
|
||||
calibration/openai/*.png
|
||||
calibration/google/*.png
|
||||
calibration/unknown/*.png
|
||||
|
||||
Examples::
|
||||
|
||||
uv run python scripts/source_pipeline_classifier.py train \
|
||||
train calibration --model-out model.npz
|
||||
uv run python scripts/source_pipeline_classifier.py predict \
|
||||
model.npz image.png
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import click
|
||||
import numpy as np
|
||||
import spectral_pipeline_probe as spectral
|
||||
from numpy.typing import NDArray
|
||||
from PIL import Image
|
||||
from watermark_benchmark import sha256_file
|
||||
|
||||
from remove_ai_watermarks._internal.forensic_124d import FEATURE_WIDTH, PATCH, image_features
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CLASSES = ("openai", "google", "unknown")
|
||||
UNKNOWN_INDEX = CLASSES.index("unknown")
|
||||
DEFAULT_GRID = 16
|
||||
DEFAULT_PENALTY = 100.0
|
||||
DEFAULT_UNKNOWN_FPR = 0.01
|
||||
CLAIM = "source/export pipeline classification; not SynthID detection"
|
||||
|
||||
FloatArray = NDArray[np.float64]
|
||||
IntArray = NDArray[np.int64]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RidgeModel:
|
||||
"""One standardized three-class ridge head with an abstention margin."""
|
||||
|
||||
mean: FloatArray
|
||||
scale: FloatArray
|
||||
weights: FloatArray
|
||||
margin: float
|
||||
grid: int
|
||||
penalty: float
|
||||
|
||||
|
||||
def discover_labeled_paths(root: Path) -> tuple[list[Path], IntArray, list[str]]:
|
||||
"""Find images under the three fixed label directories."""
|
||||
paths: list[Path] = []
|
||||
labels: list[int] = []
|
||||
hashes: list[str] = []
|
||||
for class_index, class_name in enumerate(CLASSES):
|
||||
directory = root / class_name
|
||||
if not directory.is_dir():
|
||||
raise ValueError(f"class directory does not exist: {directory}")
|
||||
class_paths = sorted(
|
||||
path for path in directory.rglob("*") if path.is_file() and path.suffix.lower() in spectral.IMAGE_SUFFIXES
|
||||
)
|
||||
if not class_paths:
|
||||
raise ValueError(f"class directory has no supported images: {directory}")
|
||||
for path in class_paths:
|
||||
paths.append(path)
|
||||
labels.append(class_index)
|
||||
hashes.append(sha256_file(path))
|
||||
if len(hashes) != len(set(hashes)):
|
||||
raise ValueError(f"duplicate image bytes found inside {root}")
|
||||
return paths, np.asarray(labels, dtype=np.int64), hashes
|
||||
|
||||
|
||||
def _feature_from_path(path: Path, geometry: spectral.SpectralGeometry) -> FloatArray | None:
|
||||
"""Return the fused representation for one path and prepared geometry."""
|
||||
with Image.open(path) as source:
|
||||
image = source.convert("RGB")
|
||||
forensic = image_features(np.asarray(image, dtype=np.uint8))
|
||||
if forensic is None:
|
||||
return None
|
||||
phase_free = spectral.feature_from_image(image, geometry, "original")
|
||||
return np.concatenate(
|
||||
(
|
||||
np.asarray(forensic, dtype=np.float64),
|
||||
np.asarray(phase_free, dtype=np.float64),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def feature_from_path(path: Path, grid: int) -> FloatArray | None:
|
||||
"""Return the fused forensic and phase-free spectral representation."""
|
||||
return _feature_from_path(path, spectral.make_geometry(size=PATCH, grid=grid))
|
||||
|
||||
|
||||
def extract_features(paths: list[Path], labels: IntArray, grid: int) -> tuple[FloatArray, IntArray, list[Path]]:
|
||||
"""Extract valid rows and report files whose forensic feature abstains."""
|
||||
geometry = spectral.make_geometry(size=PATCH, grid=grid)
|
||||
features: list[FloatArray] = []
|
||||
kept_labels: list[int] = []
|
||||
skipped: list[Path] = []
|
||||
for index, (path, label) in enumerate(zip(paths, labels, strict=True), start=1):
|
||||
try:
|
||||
feature = _feature_from_path(path, geometry)
|
||||
except (OSError, ValueError) as error:
|
||||
log.warning("Could not extract %s: %s", path, error)
|
||||
feature = None
|
||||
if feature is None:
|
||||
skipped.append(path)
|
||||
continue
|
||||
features.append(feature)
|
||||
kept_labels.append(int(label))
|
||||
if index % 100 == 0:
|
||||
log.info("Extracted %s/%s images", index, len(paths))
|
||||
if not features:
|
||||
raise ValueError("no image produced a valid feature vector")
|
||||
return np.asarray(features), np.asarray(kept_labels, dtype=np.int64), skipped
|
||||
|
||||
|
||||
def fit_ridge(
|
||||
features: FloatArray,
|
||||
labels: IntArray,
|
||||
penalty: float,
|
||||
grid: int = DEFAULT_GRID,
|
||||
) -> RidgeModel:
|
||||
"""Fit a class-balanced standardized ridge head before calibration."""
|
||||
if features.ndim != 2 or len(features) != len(labels):
|
||||
raise ValueError("features and labels must describe the same rows")
|
||||
counts = np.bincount(labels, minlength=len(CLASSES)).astype(np.float64)
|
||||
if np.any(counts == 0):
|
||||
raise ValueError("training data must contain all three classes")
|
||||
if penalty <= 0:
|
||||
raise ValueError("penalty must be positive")
|
||||
mean = features.mean(axis=0)
|
||||
raw_scale = features.std(axis=0)
|
||||
active = raw_scale > 1e-10
|
||||
scale = np.where(active, raw_scale, 1.0)
|
||||
normalized = (features[:, active] - mean[active]) / scale[active]
|
||||
sample_weight = len(labels) / (len(CLASSES) * counts[labels])
|
||||
root_weight = np.sqrt(sample_weight)[:, None]
|
||||
weighted = normalized * root_weight
|
||||
targets = np.eye(len(CLASSES), dtype=np.float64)[labels] * root_weight
|
||||
gram = weighted.T @ weighted + penalty * np.eye(weighted.shape[1])
|
||||
active_weights = cast("FloatArray", np.linalg.solve(gram, weighted.T @ targets))
|
||||
weights = np.zeros((features.shape[1], len(CLASSES)), dtype=np.float64)
|
||||
weights[active] = active_weights
|
||||
return RidgeModel(mean, scale, weights, 0.0, grid, penalty)
|
||||
|
||||
|
||||
def scores(features: FloatArray, model: RidgeModel) -> FloatArray:
|
||||
"""Return the uncalibrated ridge scores."""
|
||||
return ((features - model.mean) / model.scale) @ model.weights
|
||||
|
||||
|
||||
def decision_margins(raw_scores: FloatArray) -> FloatArray:
|
||||
"""Return each row's best-provider advantage over every alternative."""
|
||||
target_scores = raw_scores[:, :UNKNOWN_INDEX]
|
||||
best = np.argmax(target_scores, axis=1)
|
||||
other = 1 - best
|
||||
rows = np.arange(len(raw_scores))
|
||||
reference = np.maximum(raw_scores[rows, other], raw_scores[:, UNKNOWN_INDEX])
|
||||
return target_scores[rows, best] - reference
|
||||
|
||||
|
||||
def calibrate_margin(
|
||||
calibration_scores: FloatArray,
|
||||
calibration_labels: IntArray,
|
||||
unknown_fpr: float,
|
||||
) -> float:
|
||||
"""Choose the smallest empirical cut meeting the unknown false-positive target."""
|
||||
if not 0 <= unknown_fpr < 1:
|
||||
raise ValueError("unknown_fpr must be in [0, 1)")
|
||||
unknown_scores = calibration_scores[calibration_labels == UNKNOWN_INDEX]
|
||||
if not len(unknown_scores):
|
||||
raise ValueError("calibration data must contain unknown rows")
|
||||
return max(
|
||||
0.0,
|
||||
float(
|
||||
np.quantile(
|
||||
decision_margins(unknown_scores),
|
||||
1.0 - unknown_fpr,
|
||||
method="higher",
|
||||
)
|
||||
+ 1e-12
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def predict_scores(raw_scores: FloatArray, margin: float) -> IntArray:
|
||||
"""Return provider labels only when the calibrated margin is met."""
|
||||
best = np.argmax(raw_scores[:, :UNKNOWN_INDEX], axis=1)
|
||||
accepted = decision_margins(raw_scores) >= margin
|
||||
return np.where(accepted, best, UNKNOWN_INDEX).astype(np.int64)
|
||||
|
||||
|
||||
def save_model(path: Path, model: RidgeModel) -> None:
|
||||
"""Write a pickle-free model artifact."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
np.savez_compressed(
|
||||
path,
|
||||
classes=np.asarray(CLASSES),
|
||||
mean=model.mean,
|
||||
scale=model.scale,
|
||||
weights=model.weights,
|
||||
margin=np.asarray(model.margin),
|
||||
spectral_grid=np.asarray(model.grid),
|
||||
penalty=np.asarray(model.penalty),
|
||||
)
|
||||
|
||||
|
||||
def load_model(path: Path) -> RidgeModel:
|
||||
"""Load and validate a pickle-free model artifact."""
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
required = {
|
||||
"classes",
|
||||
"mean",
|
||||
"scale",
|
||||
"weights",
|
||||
"margin",
|
||||
"spectral_grid",
|
||||
"penalty",
|
||||
}
|
||||
if set(payload.files) != required:
|
||||
raise ValueError(f"model fields must be exactly {sorted(required)}")
|
||||
classes = tuple(str(value) for value in payload["classes"])
|
||||
mean = np.asarray(payload["mean"], dtype=np.float64)
|
||||
scale = np.asarray(payload["scale"], dtype=np.float64)
|
||||
weights = np.asarray(payload["weights"], dtype=np.float64)
|
||||
margin = float(payload["margin"])
|
||||
grid = int(payload["spectral_grid"])
|
||||
penalty = float(payload["penalty"])
|
||||
expected_features = FEATURE_WIDTH + 3 * grid * grid
|
||||
if classes != CLASSES:
|
||||
raise ValueError(f"model classes must be {CLASSES}")
|
||||
if mean.shape != (expected_features,) or scale.shape != mean.shape:
|
||||
raise ValueError("model normalization vectors have the wrong shape")
|
||||
if weights.shape != (expected_features, len(CLASSES)):
|
||||
raise ValueError("model weights have the wrong shape")
|
||||
if not np.all(np.isfinite(mean)) or not np.all(np.isfinite(scale)):
|
||||
raise ValueError("model normalization vectors must be finite")
|
||||
if not np.all(np.isfinite(weights)) or np.any(scale <= 0):
|
||||
raise ValueError("model weights must be finite and scales positive")
|
||||
if not np.isfinite(margin) or margin < 0:
|
||||
raise ValueError("model margin must be finite and non-negative")
|
||||
if grid < 2 or PATCH % grid or not np.isfinite(penalty) or penalty <= 0:
|
||||
raise ValueError("model calibration fields are invalid")
|
||||
return RidgeModel(mean, scale, weights, margin, grid, penalty)
|
||||
|
||||
|
||||
@click.group()
|
||||
def main() -> None:
|
||||
"""Train or run the research-only source-pipeline classifier."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
|
||||
|
||||
@main.command("train")
|
||||
@click.argument("train_root", type=click.Path(exists=True, file_okay=False, path_type=Path))
|
||||
@click.argument("calibration_root", type=click.Path(exists=True, file_okay=False, path_type=Path))
|
||||
@click.option("--model-out", type=click.Path(dir_okay=False, path_type=Path), required=True)
|
||||
@click.option("--grid", type=click.IntRange(min=2), default=DEFAULT_GRID, show_default=True)
|
||||
@click.option("--penalty", type=click.FloatRange(min=0.0, min_open=True), default=DEFAULT_PENALTY, show_default=True)
|
||||
@click.option(
|
||||
"--unknown-fpr",
|
||||
type=click.FloatRange(min=0.0, max=1.0, max_open=True),
|
||||
default=DEFAULT_UNKNOWN_FPR,
|
||||
show_default=True,
|
||||
)
|
||||
def train_command(
|
||||
train_root: Path,
|
||||
calibration_root: Path,
|
||||
model_out: Path,
|
||||
grid: int,
|
||||
penalty: float,
|
||||
unknown_fpr: float,
|
||||
) -> None:
|
||||
"""Fit on TRAIN_ROOT and calibrate abstention on CALIBRATION_ROOT."""
|
||||
if PATCH % grid:
|
||||
raise click.UsageError(f"--grid must divide {PATCH}")
|
||||
try:
|
||||
train_paths, train_labels, train_hashes = discover_labeled_paths(train_root)
|
||||
calibration_paths, calibration_labels, calibration_hashes = discover_labeled_paths(calibration_root)
|
||||
overlap = set(train_hashes) & set(calibration_hashes)
|
||||
if overlap:
|
||||
raise ValueError(f"train/calibration byte overlap: {len(overlap)} images")
|
||||
train_features, kept_train_labels, train_skipped = extract_features(train_paths, train_labels, grid)
|
||||
calibration_features, kept_calibration_labels, calibration_skipped = extract_features(
|
||||
calibration_paths, calibration_labels, grid
|
||||
)
|
||||
provisional = fit_ridge(train_features, kept_train_labels, penalty, grid)
|
||||
margin = calibrate_margin(
|
||||
scores(calibration_features, provisional),
|
||||
kept_calibration_labels,
|
||||
unknown_fpr,
|
||||
)
|
||||
model = replace(provisional, margin=margin)
|
||||
save_model(model_out, model)
|
||||
except ValueError as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
click.echo(
|
||||
json.dumps(
|
||||
{
|
||||
"model": str(model_out),
|
||||
"claim": CLAIM,
|
||||
"train_rows": len(train_features),
|
||||
"calibration_rows": len(calibration_features),
|
||||
"feature_abstentions": len(train_skipped) + len(calibration_skipped),
|
||||
"margin": margin,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@main.command("predict")
|
||||
@click.argument("model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
def predict_command(model_path: Path, images: tuple[Path, ...]) -> None:
|
||||
"""Classify original-export IMAGES with MODEL_PATH."""
|
||||
try:
|
||||
model = load_model(model_path)
|
||||
except (OSError, ValueError) as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
results: list[dict[str, object]] = []
|
||||
geometry = spectral.make_geometry(size=PATCH, grid=model.grid)
|
||||
for path in images:
|
||||
try:
|
||||
feature = _feature_from_path(path, geometry)
|
||||
except (OSError, ValueError) as error:
|
||||
raise click.ClickException(f"could not extract {path}: {error}") from error
|
||||
if feature is None:
|
||||
results.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"label": "unknown",
|
||||
"reason": "124-d feature unavailable",
|
||||
"claim": CLAIM,
|
||||
}
|
||||
)
|
||||
continue
|
||||
raw_scores = scores(feature[None, :], model)
|
||||
prediction = int(predict_scores(raw_scores, model.margin)[0])
|
||||
results.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"label": CLASSES[prediction],
|
||||
"decision_margin": float(decision_margins(raw_scores)[0]),
|
||||
"required_margin": model.margin,
|
||||
"scores": {name: float(raw_scores[0, index]) for index, name in enumerate(CLASSES)},
|
||||
"claim": CLAIM,
|
||||
}
|
||||
)
|
||||
click.echo(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -30,6 +30,14 @@ import numpy as np
|
||||
from numpy.typing import NDArray
|
||||
from PIL import Image, ImageFilter
|
||||
|
||||
from remove_ai_watermarks._internal import source_spectral as _spectral
|
||||
|
||||
SpectralGeometry = _spectral.SpectralGeometry
|
||||
make_geometry = _spectral.make_geometry
|
||||
phase_free_log_spectrum = _spectral.phase_free_log_spectrum
|
||||
spectral_feature = _spectral.spectral_feature
|
||||
_source_feature_from_image = _spectral.feature_from_image
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg", ".png", ".webp"})
|
||||
@@ -40,17 +48,6 @@ IntArray = NDArray[np.int64]
|
||||
BoolArray = NDArray[np.bool_]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpectralGeometry:
|
||||
"""Precomputed coordinates for one phase-free spectral representation."""
|
||||
|
||||
size: int
|
||||
grid: int
|
||||
radius: NDArray[np.int32]
|
||||
radius_counts: IntArray
|
||||
passband: BoolArray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FoldModel:
|
||||
"""One standardized dual-ridge model and its held-out rows."""
|
||||
@@ -62,51 +59,6 @@ class FoldModel:
|
||||
weights: FloatArray
|
||||
|
||||
|
||||
def make_geometry(size: int = 256, grid: int = 32) -> SpectralGeometry:
|
||||
"""Build the radial detrending and pooling geometry."""
|
||||
if size < 16 or grid < 2 or size % grid:
|
||||
raise ValueError("size must be at least 16 and divisible by grid")
|
||||
yy, xx = np.mgrid[:size, :size]
|
||||
radius = np.rint(np.hypot(yy - size // 2, xx - size // 2)).astype(np.int32)
|
||||
radius_counts = np.bincount(radius.ravel())
|
||||
passband = (radius >= 4) & (radius <= size * 0.48)
|
||||
return SpectralGeometry(size, grid, radius, radius_counts, passband)
|
||||
|
||||
|
||||
def phase_free_log_spectrum(channel: FloatArray) -> FloatArray:
|
||||
"""Return centered log FFT magnitude, deliberately discarding phase."""
|
||||
if channel.ndim != 2:
|
||||
raise ValueError("channel must be a two-dimensional array")
|
||||
centered = np.asarray(channel, dtype=np.float64) - float(np.mean(channel))
|
||||
return np.log1p(np.abs(np.fft.fftshift(np.fft.fft2(centered))))
|
||||
|
||||
|
||||
def spectral_feature(pixels: FloatArray, geometry: SpectralGeometry) -> FloatArray:
|
||||
"""Extract pooled radial-residual spectra from luma and opponent color."""
|
||||
if pixels.shape != (geometry.size, geometry.size, 3):
|
||||
raise ValueError(f"pixels must have shape {(geometry.size, geometry.size, 3)}, got {pixels.shape}")
|
||||
red, green, blue = np.moveaxis(np.asarray(pixels, dtype=np.float64), 2, 0)
|
||||
channels = (
|
||||
0.299 * red + 0.587 * green + 0.114 * blue,
|
||||
red - green,
|
||||
blue - (red + green) / 2.0,
|
||||
)
|
||||
block = geometry.size // geometry.grid
|
||||
features: list[FloatArray] = []
|
||||
for channel in channels:
|
||||
spectrum = phase_free_log_spectrum(channel)
|
||||
radial_sum = np.bincount(geometry.radius.ravel(), weights=spectrum.ravel())
|
||||
radial_mean = radial_sum / geometry.radius_counts
|
||||
residual = np.where(
|
||||
geometry.passband,
|
||||
spectrum - radial_mean[geometry.radius],
|
||||
0.0,
|
||||
)
|
||||
pooled = residual.reshape(geometry.grid, block, geometry.grid, block).mean(axis=(1, 3))
|
||||
features.append(pooled.ravel())
|
||||
return np.concatenate(features)
|
||||
|
||||
|
||||
def _apply_attack(image: Image.Image, attack: str) -> Image.Image:
|
||||
"""Apply one declared robustness transform before canonical resizing."""
|
||||
if attack == "original":
|
||||
@@ -134,15 +86,13 @@ def _apply_attack(image: Image.Image, attack: str) -> Image.Image:
|
||||
def feature_from_path(path: Path, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
"""Decode one path, apply ATTACK, and return its phase-free feature."""
|
||||
with Image.open(path) as source:
|
||||
return _feature_from_image(source.convert("RGB"), geometry, attack)
|
||||
return feature_from_image(source.convert("RGB"), geometry, attack)
|
||||
|
||||
|
||||
def _feature_from_image(image: Image.Image, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
def feature_from_image(image: Image.Image, geometry: SpectralGeometry, attack: str) -> FloatArray:
|
||||
"""Extract one attacked feature from an already decoded RGB image."""
|
||||
attacked = _apply_attack(image, attack)
|
||||
canonical = attacked.resize((geometry.size, geometry.size), Image.Resampling.LANCZOS)
|
||||
pixels = np.asarray(canonical, dtype=np.float64) / 255.0
|
||||
return spectral_feature(pixels, geometry)
|
||||
return _source_feature_from_image(attacked, geometry)
|
||||
|
||||
|
||||
def discover_grouped_paths(
|
||||
@@ -253,7 +203,7 @@ def _extract_all_attacks(
|
||||
with Image.open(path) as source:
|
||||
image = source.convert("RGB")
|
||||
for attack in attacks:
|
||||
rows[attack].append(_feature_from_image(image, geometry, attack))
|
||||
rows[attack].append(feature_from_image(image, geometry, attack))
|
||||
return {attack: np.asarray(features) for attack, features in rows.items()}
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ High-level API (lazy, so ``import remove_ai_watermarks`` stays cheap)::
|
||||
raiw.remove_video_metadata("in.mp4", "out.mp4") # verified metadata strip
|
||||
raiw.remove_video_invisible("in.mp4", "out.mp4") # calibrated video-pixel regeneration
|
||||
raiw.remove_video_visible("in.mp4", "out.mp4") # stable visible video-mark removal
|
||||
raiw.classify_source("in.png") # source pipeline, not SynthID detection
|
||||
|
||||
For a provenance verdict use the ``identify`` submodule::
|
||||
|
||||
@@ -33,7 +34,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.40.5"
|
||||
__version__ = "0.41.0"
|
||||
|
||||
__all__ = [
|
||||
"BatchItemResult",
|
||||
@@ -41,8 +42,10 @@ __all__ = [
|
||||
"InvisibleOptions",
|
||||
"MetadataStripIncomplete",
|
||||
"RemoveAllResult",
|
||||
"SourceClassification",
|
||||
"VisibleRemovalResult",
|
||||
"__version__",
|
||||
"classify_source",
|
||||
"identify_video",
|
||||
"inspect_video_metadata",
|
||||
"remove_all",
|
||||
@@ -71,6 +74,7 @@ if TYPE_CHECKING:
|
||||
remove_visible_detailed,
|
||||
visible_provenance,
|
||||
)
|
||||
from remove_ai_watermarks.source_classify import SourceClassification, classify_source
|
||||
from remove_ai_watermarks.video import (
|
||||
identify_video,
|
||||
inspect_video_metadata,
|
||||
@@ -101,6 +105,10 @@ def __getattr__(name: str) -> object:
|
||||
from remove_ai_watermarks import api
|
||||
|
||||
return getattr(api, name)
|
||||
if name in ("SourceClassification", "classify_source"):
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
return getattr(source_classify, name)
|
||||
if name in (
|
||||
"identify_video",
|
||||
"inspect_video_metadata",
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Phase-free spectral features for source/export-pipeline classification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpectralGeometry:
|
||||
"""Precomputed coordinates for one phase-free spectral representation."""
|
||||
|
||||
size: int
|
||||
grid: int
|
||||
radius: NDArray[np.int32]
|
||||
radius_counts: NDArray[np.int64]
|
||||
passband: NDArray[np.bool_]
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def make_geometry(size: int = 256, grid: int = 16) -> SpectralGeometry:
|
||||
"""Build radial detrending and pooling geometry."""
|
||||
if size < 16 or grid < 2 or size % grid:
|
||||
raise ValueError("size must be at least 16 and divisible by grid")
|
||||
yy, xx = np.mgrid[:size, :size]
|
||||
radius = np.rint(np.hypot(yy - size // 2, xx - size // 2)).astype(np.int32)
|
||||
radius_counts = np.bincount(radius.ravel())
|
||||
passband = (radius >= 4) & (radius <= size * 0.48)
|
||||
return SpectralGeometry(size, grid, radius, radius_counts, passband)
|
||||
|
||||
|
||||
def phase_free_log_spectrum(channel: NDArray[np.float64]) -> NDArray[np.float64]:
|
||||
"""Return centered log FFT magnitude, deliberately discarding phase."""
|
||||
if channel.ndim != 2:
|
||||
raise ValueError("channel must be a two-dimensional array")
|
||||
centered = channel - float(np.mean(channel))
|
||||
return np.log1p(np.abs(np.fft.fftshift(np.fft.fft2(centered))))
|
||||
|
||||
|
||||
def spectral_feature(pixels: NDArray[np.float64], geometry: SpectralGeometry) -> NDArray[np.float64]:
|
||||
"""Extract pooled radial-residual spectra from luma and opponent color."""
|
||||
expected = (geometry.size, geometry.size, 3)
|
||||
if pixels.shape != expected:
|
||||
raise ValueError(f"pixels must have shape {expected}, got {pixels.shape}")
|
||||
red, green, blue = np.moveaxis(pixels, 2, 0)
|
||||
channels = (
|
||||
0.299 * red + 0.587 * green + 0.114 * blue,
|
||||
red - green,
|
||||
blue - (red + green) / 2.0,
|
||||
)
|
||||
block = geometry.size // geometry.grid
|
||||
features: list[NDArray[np.float64]] = []
|
||||
for channel in channels:
|
||||
spectrum = phase_free_log_spectrum(channel)
|
||||
radial_sum = np.bincount(geometry.radius.ravel(), weights=spectrum.ravel())
|
||||
radial_mean = radial_sum / geometry.radius_counts
|
||||
residual = np.where(
|
||||
geometry.passband,
|
||||
spectrum - radial_mean[geometry.radius],
|
||||
0.0,
|
||||
)
|
||||
pooled = residual.reshape(geometry.grid, block, geometry.grid, block).mean(axis=(1, 3))
|
||||
features.append(pooled.ravel())
|
||||
return np.concatenate(features)
|
||||
|
||||
|
||||
def feature_from_image(image: Image.Image, geometry: SpectralGeometry) -> NDArray[np.float64]:
|
||||
"""Resize an RGB image canonically and extract its phase-free feature."""
|
||||
canonical = image.resize((geometry.size, geometry.size), Image.Resampling.LANCZOS)
|
||||
pixels = np.asarray(canonical, dtype=np.float64) / 255.0
|
||||
return spectral_feature(pixels, geometry)
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Abstaining source/export-pipeline classification from image pixels.
|
||||
|
||||
This module does not detect or decode SynthID. It labels original-export-looking
|
||||
pixel pipelines as OpenAI, Google, or unknown after metadata may have been
|
||||
removed. Call it explicitly; ``identify`` never imports or runs it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from hashlib import sha256
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
|
||||
SOURCE_CLASSIFY_EXTRA = "'remove-ai-watermarks[source-classify]'"
|
||||
WEIGHTS_ENV = "RAIW_SOURCE_CLASSIFY_WEIGHTS"
|
||||
WEIGHTS_REPO = "wiltodelta/raiw-source-classify"
|
||||
WEIGHTS_REVISION = "4b8b408d03dac20231d46217977fd164414b6b9e"
|
||||
MODEL_FILE = "source-pipeline-mlp.npz"
|
||||
MODEL_SHA256 = "6430b0738869ff07b6845b7c6149c2a83bc3f3785878082886c241c6f7ca0c4a"
|
||||
CLASSES = ("openai", "google", "unknown")
|
||||
PROVIDERS = CLASSES[:2]
|
||||
FEATURE_WIDTH = 892
|
||||
SPECTRAL_GRID = 16
|
||||
CLAIM = "source/export pipeline classification; not SynthID detection"
|
||||
# NumPy and the training runtime can differ by a few float32 ULPs at a matrix
|
||||
# boundary. Treat values within this tolerance as the stored threshold so the
|
||||
# exported runtime preserves the training runtime's >= decision.
|
||||
THRESHOLD_TOLERANCE = 1e-5
|
||||
|
||||
SourceLabel = Literal["openai", "google", "unknown"]
|
||||
SourceReason = Literal["classified", "abstained", "feature_unavailable"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceClassification:
|
||||
"""One narrow source-pipeline result with an explicit abstention."""
|
||||
|
||||
label: SourceLabel
|
||||
reason: SourceReason
|
||||
scores: dict[str, float]
|
||||
margins: dict[str, float]
|
||||
thresholds: dict[str, float]
|
||||
claim: str = CLAIM
|
||||
model_revision: str = WEIGHTS_REVISION
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"label": self.label,
|
||||
"reason": self.reason,
|
||||
"scores": self.scores,
|
||||
"margins": self.margins,
|
||||
"thresholds": self.thresholds,
|
||||
"claim": self.claim,
|
||||
"model_revision": self.model_revision,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Model:
|
||||
mean: NDArray[Any]
|
||||
scale: NDArray[Any]
|
||||
margins: NDArray[Any]
|
||||
layer0_weight: NDArray[Any]
|
||||
layer0_bias: NDArray[Any]
|
||||
layer1_weight: NDArray[Any]
|
||||
layer1_bias: NDArray[Any]
|
||||
output_weight: NDArray[Any]
|
||||
output_bias: NDArray[Any]
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Return whether the lightweight source-classify runtime is installed."""
|
||||
from remove_ai_watermarks.optional_deps import module_available
|
||||
|
||||
return module_available("numpy", "cv2", "huggingface_hub")
|
||||
|
||||
|
||||
def _model_path() -> Path:
|
||||
override = os.environ.get(WEIGHTS_ENV, "").strip()
|
||||
if override:
|
||||
path = Path(override).expanduser()
|
||||
candidate = path / MODEL_FILE if path.is_dir() else path
|
||||
if not candidate.is_file():
|
||||
raise RuntimeError(f"{WEIGHTS_ENV} does not contain {MODEL_FILE}: {path}")
|
||||
return candidate
|
||||
try:
|
||||
hub: Any = import_module("huggingface_hub")
|
||||
except ModuleNotFoundError as error:
|
||||
raise RuntimeError(f"Install {SOURCE_CLASSIFY_EXTRA} to download the source classifier") from error
|
||||
return Path(
|
||||
hub.hf_hub_download(
|
||||
repo_id=WEIGHTS_REPO,
|
||||
filename=MODEL_FILE,
|
||||
revision=WEIGHTS_REVISION,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def _load_model(path: Path, expected_sha256: str | None = MODEL_SHA256) -> _Model:
|
||||
import numpy as np
|
||||
|
||||
if expected_sha256 is not None:
|
||||
digest = sha256(path.read_bytes()).hexdigest()
|
||||
if digest != expected_sha256:
|
||||
raise ValueError(f"model SHA-256 mismatch: expected {expected_sha256}, got {digest}")
|
||||
required = {
|
||||
"classes",
|
||||
"kind",
|
||||
"mean",
|
||||
"scale",
|
||||
"margins",
|
||||
"layer0_weight",
|
||||
"layer0_bias",
|
||||
"layer1_weight",
|
||||
"layer1_bias",
|
||||
"output_weight",
|
||||
"output_bias",
|
||||
"activation",
|
||||
}
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
if set(payload.files) != required:
|
||||
raise ValueError(f"model fields must be exactly {sorted(required)}")
|
||||
if tuple(str(value) for value in payload["classes"]) != CLASSES:
|
||||
raise ValueError(f"model classes must be {CLASSES}")
|
||||
if str(payload["kind"]) != "fused":
|
||||
raise ValueError("model kind must be fused")
|
||||
if str(payload["activation"]) != "gelu_exact":
|
||||
raise ValueError("model activation must be gelu_exact")
|
||||
arrays = {
|
||||
name: np.asarray(
|
||||
payload[name],
|
||||
dtype=np.float64 if name in {"mean", "scale", "margins"} else np.float32,
|
||||
)
|
||||
for name in required - {"classes", "kind", "activation"}
|
||||
}
|
||||
shapes = {
|
||||
"mean": (FEATURE_WIDTH,),
|
||||
"scale": (FEATURE_WIDTH,),
|
||||
"margins": (len(PROVIDERS),),
|
||||
"layer0_weight": (128, FEATURE_WIDTH),
|
||||
"layer0_bias": (128,),
|
||||
"layer1_weight": (64, 128),
|
||||
"layer1_bias": (64,),
|
||||
"output_weight": (len(CLASSES), 64),
|
||||
"output_bias": (len(CLASSES),),
|
||||
}
|
||||
for name, expected in shapes.items():
|
||||
if arrays[name].shape != expected:
|
||||
raise ValueError(f"model field {name} must have shape {expected}")
|
||||
if not np.all(np.isfinite(arrays[name])):
|
||||
raise ValueError(f"model field {name} must be finite")
|
||||
if np.any(arrays["scale"] <= 0) or np.any(arrays["margins"] < 0):
|
||||
raise ValueError("model scales must be positive and margins non-negative")
|
||||
return _Model(**arrays)
|
||||
|
||||
|
||||
def _gelu_exact(values: NDArray[Any]) -> NDArray[Any]:
|
||||
import numpy as np
|
||||
|
||||
flat = values.ravel()
|
||||
errors = np.fromiter((math.erf(float(value) / math.sqrt(2.0)) for value in flat), dtype=np.float32)
|
||||
return (0.5 * flat * (1.0 + errors)).reshape(values.shape).astype(np.float32)
|
||||
|
||||
|
||||
def _scores(features: NDArray[Any], model: _Model) -> NDArray[Any]:
|
||||
import numpy as np
|
||||
|
||||
values = np.asarray((features - model.mean) / model.scale, dtype=np.float32)
|
||||
hidden0 = _gelu_exact(model.layer0_weight @ values + model.layer0_bias)
|
||||
hidden1 = _gelu_exact(model.layer1_weight @ hidden0 + model.layer1_bias)
|
||||
return np.asarray(model.output_weight @ hidden1 + model.output_bias, dtype=np.float64)
|
||||
|
||||
|
||||
def _result(raw_scores: NDArray[Any], thresholds: NDArray[Any]) -> SourceClassification:
|
||||
import numpy as np
|
||||
|
||||
margins = np.empty(len(PROVIDERS), dtype=np.float64)
|
||||
for index in range(len(PROVIDERS)):
|
||||
competitors = np.delete(raw_scores, index)
|
||||
margins[index] = raw_scores[index] - float(np.max(competitors))
|
||||
excess = margins - thresholds
|
||||
best = int(np.argmax(excess))
|
||||
accepted = bool(excess[best] >= -THRESHOLD_TOLERANCE)
|
||||
label: SourceLabel = PROVIDERS[best] if accepted else "unknown"
|
||||
return SourceClassification(
|
||||
label=label,
|
||||
reason="classified" if accepted else "abstained",
|
||||
scores={name: float(raw_scores[index]) for index, name in enumerate(CLASSES)},
|
||||
margins={name: float(margins[index]) for index, name in enumerate(PROVIDERS)},
|
||||
thresholds={name: float(thresholds[index]) for index, name in enumerate(PROVIDERS)},
|
||||
)
|
||||
|
||||
|
||||
def _unavailable(model: _Model) -> SourceClassification:
|
||||
return SourceClassification(
|
||||
label="unknown",
|
||||
reason="feature_unavailable",
|
||||
scores={},
|
||||
margins={},
|
||||
thresholds={name: float(model.margins[index]) for index, name in enumerate(PROVIDERS)},
|
||||
)
|
||||
|
||||
|
||||
def classify_source(source: str | os.PathLike[str]) -> SourceClassification:
|
||||
"""Classify an original-looking image export, abstaining as ``unknown``.
|
||||
|
||||
The result describes a complete source/export pipeline. It is not evidence
|
||||
that SynthID is present or absent.
|
||||
"""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks import image_io
|
||||
from remove_ai_watermarks._internal.forensic_124d import image_features
|
||||
from remove_ai_watermarks._internal.source_spectral import feature_from_image, make_geometry
|
||||
|
||||
model = _load_model(_model_path(), expected_sha256=MODEL_SHA256)
|
||||
image_io._register_heif() # pyright: ignore[reportPrivateUsage]
|
||||
with Image.open(source) as opened:
|
||||
image = opened.convert("RGB")
|
||||
pixels = np.asarray(image, dtype=np.uint8)
|
||||
forensic = image_features(pixels)
|
||||
if forensic is None:
|
||||
return _unavailable(model)
|
||||
spectral = feature_from_image(image, make_geometry(grid=SPECTRAL_GRID))
|
||||
features = np.concatenate((forensic, spectral))
|
||||
if features.shape != (FEATURE_WIDTH,) or not np.all(np.isfinite(features)):
|
||||
return _unavailable(model)
|
||||
return _result(_scores(features, model), model.margins)
|
||||
@@ -23,6 +23,10 @@ class TestTopLevelExports:
|
||||
assert raiw.remove_visible_detailed is api.remove_visible_detailed
|
||||
assert raiw.VisibleRemovalResult is api.VisibleRemovalResult
|
||||
assert raiw.visible_provenance is api.visible_provenance
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
assert raiw.classify_source is source_classify.classify_source
|
||||
assert raiw.SourceClassification is source_classify.SourceClassification
|
||||
|
||||
def test_unknown_attribute_raises(self):
|
||||
with pytest.raises(AttributeError):
|
||||
|
||||
@@ -2,29 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _publish() -> Any:
|
||||
path = Path(__file__).resolve().parents[1] / "scripts" / "publish_photo_classify_hf.py"
|
||||
spec = importlib.util.spec_from_file_location("publish_photo_classify_hf", path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import publish_photo_classify_hf as publish # noqa: E402
|
||||
|
||||
|
||||
def test_hub_id_is_the_photo_classify_repo() -> None:
|
||||
assert _publish().HUB_REPO == "wiltodelta/raiw-photo-classify"
|
||||
assert publish.HUB_REPO == "wiltodelta/raiw-photo-classify"
|
||||
|
||||
|
||||
def test_publish_freeze_includes_both_clip_runtimes() -> None:
|
||||
publish = _publish()
|
||||
from remove_ai_watermarks.classify import _WEIGHT_FILES, CLIP_ONNX_FILE, WEIGHTS_ALLOW_PATTERNS
|
||||
|
||||
assert publish.CLIP_FILE in publish.WEIGHT_FILES
|
||||
@@ -41,7 +33,7 @@ def test_weight_paths_accepts_run1_layout(tmp_path: Path) -> None:
|
||||
run1.mkdir()
|
||||
(run1 / "detector.pt").write_bytes(b"det")
|
||||
(run1 / "provider.pt").write_bytes(b"prov")
|
||||
found = _publish().weight_paths(tmp_path)
|
||||
found = publish.weight_paths(tmp_path)
|
||||
assert found["detector.pt"] == run1 / "detector.pt"
|
||||
assert found["clip-l-ft.pt"] == tmp_path / "clip-l-ft.pt"
|
||||
|
||||
@@ -50,12 +42,12 @@ def test_weight_paths_requires_every_file(tmp_path: Path) -> None:
|
||||
(tmp_path / "clip-l-ft.pt").write_bytes(b"clip")
|
||||
(tmp_path / "clip-l-ft-vision-fp32.onnx").write_bytes(b"onnx")
|
||||
with pytest.raises(SystemExit, match=r"missing probe-weights-clip-l-ft\.npz"):
|
||||
_publish().weight_paths(tmp_path)
|
||||
publish.weight_paths(tmp_path)
|
||||
|
||||
|
||||
def test_stage_card_copies_tracked_files(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "stage"
|
||||
_publish().stage_card(dest)
|
||||
publish.stage_card(dest)
|
||||
assert (dest / "README.md").is_file()
|
||||
assert (dest / "operating-point.json").is_file()
|
||||
text = (dest / "operating-point.json").read_text()
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Local staging checks for the source-classifier Hub publisher."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import publish_source_classify_hf as publish # noqa: E402
|
||||
|
||||
|
||||
def test_hub_id_is_the_source_classify_repo() -> None:
|
||||
assert publish.HUB_REPO == "wiltodelta/raiw-source-classify"
|
||||
|
||||
|
||||
def test_stage_release_copies_only_public_artifacts(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
source.mkdir()
|
||||
(source / "source-pipeline-mlp.npz").write_bytes(b"weights")
|
||||
destination = tmp_path / "stage"
|
||||
publish.stage_release(destination, source)
|
||||
assert sorted(path.name for path in destination.iterdir()) == sorted(publish.PUBLIC_FILES)
|
||||
|
||||
|
||||
def test_stage_release_requires_the_model(tmp_path: Path) -> None:
|
||||
with pytest.raises(SystemExit, match=r"source-pipeline-mlp\.npz"):
|
||||
publish.stage_release(tmp_path / "stage", tmp_path)
|
||||
|
||||
|
||||
def test_card_sha_matches_the_metrics_manifest() -> None:
|
||||
metrics = json.loads((publish.CARD_DIR / "metrics.json").read_text())
|
||||
card = (publish.CARD_DIR / "README.md").read_text()
|
||||
assert metrics["artifact_sha256"] in card
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Unit contracts for the lightweight source-pipeline runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from hashlib import sha256
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from remove_ai_watermarks import source_classify
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_model_identity_is_immutable() -> None:
|
||||
assert re.fullmatch(r"[0-9a-f]{40}", source_classify.WEIGHTS_REVISION)
|
||||
assert re.fullmatch(r"[0-9a-f]{64}", source_classify.MODEL_SHA256)
|
||||
|
||||
|
||||
def _artifact(path: Path, *, activation: str = "gelu_exact", output_bias: tuple[float, ...] = (0.0, 0.0, 0.0)) -> None:
|
||||
np.savez_compressed(
|
||||
path,
|
||||
classes=np.asarray(source_classify.CLASSES),
|
||||
kind=np.asarray("fused"),
|
||||
mean=np.zeros(source_classify.FEATURE_WIDTH),
|
||||
scale=np.ones(source_classify.FEATURE_WIDTH),
|
||||
margins=np.asarray((0.5, 0.5)),
|
||||
layer0_weight=np.zeros((128, source_classify.FEATURE_WIDTH), dtype=np.float32),
|
||||
layer0_bias=np.zeros(128, dtype=np.float32),
|
||||
layer1_weight=np.zeros((64, 128), dtype=np.float32),
|
||||
layer1_bias=np.zeros(64, dtype=np.float32),
|
||||
output_weight=np.zeros((3, 64), dtype=np.float32),
|
||||
output_bias=np.asarray(output_bias, dtype=np.float32),
|
||||
activation=np.asarray(activation),
|
||||
)
|
||||
|
||||
|
||||
def test_loader_accepts_the_exact_pickle_free_schema(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path)
|
||||
|
||||
model = source_classify._load_model(path, expected_sha256=None)
|
||||
|
||||
assert model.mean.shape == (source_classify.FEATURE_WIDTH,)
|
||||
assert model.margins.tolist() == pytest.approx([0.5, 0.5])
|
||||
|
||||
|
||||
def test_loader_rejects_the_mislabeled_activation(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path, activation="gelu_tanh")
|
||||
|
||||
with pytest.raises(ValueError, match="activation must be gelu_exact"):
|
||||
source_classify._load_model(path, expected_sha256=None)
|
||||
|
||||
|
||||
def test_provider_threshold_mutation_changes_the_real_decision(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path, output_bias=(2.0, 0.0, 0.0))
|
||||
model = source_classify._load_model(path, expected_sha256=None)
|
||||
features = np.zeros(source_classify.FEATURE_WIDTH, dtype=np.float32)
|
||||
scores = source_classify._scores(features, model)
|
||||
|
||||
accepted = source_classify._result(scores, np.asarray((1.5, 0.5), dtype=np.float32))
|
||||
rejected = source_classify._result(scores, np.asarray((2.5, 0.5), dtype=np.float32))
|
||||
|
||||
assert accepted.label == "openai"
|
||||
assert accepted.reason == "classified"
|
||||
assert rejected.label == "unknown"
|
||||
assert rejected.reason == "abstained"
|
||||
|
||||
|
||||
def test_a_float32_threshold_tie_preserves_the_inclusive_training_decision() -> None:
|
||||
scores = np.asarray((2.0, 0.0, 0.0), dtype=np.float32)
|
||||
result = source_classify._result(scores, np.asarray((2.0, 0.5), dtype=np.float32))
|
||||
|
||||
assert result.label == "openai"
|
||||
assert result.reason == "classified"
|
||||
|
||||
|
||||
def test_small_image_returns_honest_unknown(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from PIL import Image
|
||||
|
||||
weights = tmp_path / "weights"
|
||||
weights.mkdir()
|
||||
_artifact(weights / source_classify.MODEL_FILE)
|
||||
monkeypatch.setenv(source_classify.WEIGHTS_ENV, str(weights))
|
||||
monkeypatch.setattr(
|
||||
source_classify,
|
||||
"MODEL_SHA256",
|
||||
sha256((weights / source_classify.MODEL_FILE).read_bytes()).hexdigest(),
|
||||
)
|
||||
image = tmp_path / "small.png"
|
||||
Image.new("RGB", (128, 128), "white").save(image)
|
||||
|
||||
result = source_classify.classify_source(image)
|
||||
|
||||
assert result.label == "unknown"
|
||||
assert result.reason == "feature_unavailable"
|
||||
assert result.scores == {}
|
||||
|
||||
|
||||
def test_model_hash_is_a_runtime_precondition(tmp_path: Path) -> None:
|
||||
path = tmp_path / source_classify.MODEL_FILE
|
||||
_artifact(path)
|
||||
|
||||
with pytest.raises(ValueError, match="model SHA-256 mismatch"):
|
||||
source_classify._load_model(path, expected_sha256="0" * 64)
|
||||
|
||||
|
||||
def test_spectral_feature_is_finite_and_has_the_frozen_width() -> None:
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks._internal.source_spectral import feature_from_image, make_geometry
|
||||
|
||||
rng = np.random.default_rng(20260916)
|
||||
pixels = rng.integers(0, 256, size=(300, 400, 3), dtype=np.uint8)
|
||||
feature = feature_from_image(Image.fromarray(pixels), make_geometry())
|
||||
|
||||
assert feature.shape == (768,)
|
||||
assert np.all(np.isfinite(feature))
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Tests for the abstaining source-pipeline classifier research tool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import source_pipeline_classifier as classifier
|
||||
|
||||
|
||||
def test_command_help_exits_cleanly() -> None:
|
||||
result = CliRunner().invoke(classifier.main, ["--help"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Usage:" in result.output
|
||||
assert "train" in result.output
|
||||
assert "predict" in result.output
|
||||
|
||||
|
||||
def _training_rows() -> tuple[np.ndarray, np.ndarray]:
|
||||
rng = np.random.default_rng(20260915)
|
||||
rows = np.vstack(
|
||||
(
|
||||
rng.normal((-3.0, 0.0, 0.0), 0.1, size=(12, 3)),
|
||||
rng.normal((3.0, 0.0, 0.0), 0.1, size=(12, 3)),
|
||||
rng.normal((0.0, 3.0, 0.0), 0.1, size=(12, 3)),
|
||||
)
|
||||
)
|
||||
labels = np.repeat(np.arange(3), 12)
|
||||
return rows, labels
|
||||
|
||||
|
||||
def test_fit_and_abstention_separate_provider_rows() -> None:
|
||||
features, labels = _training_rows()
|
||||
model = classifier.fit_ridge(features, labels, penalty=1.0)
|
||||
raw_scores = classifier.scores(features, model)
|
||||
margin = classifier.calibrate_margin(raw_scores, labels, unknown_fpr=0.01)
|
||||
predictions = classifier.predict_scores(raw_scores, margin)
|
||||
|
||||
assert np.array_equal(predictions[:24], labels[:24])
|
||||
assert np.all(predictions[24:] == classifier.UNKNOWN_INDEX)
|
||||
|
||||
|
||||
def test_model_round_trip_is_pickle_free_and_strict(tmp_path: Path) -> None:
|
||||
grid = 4
|
||||
feature_count = classifier.FEATURE_WIDTH + 3 * grid * grid
|
||||
model = classifier.RidgeModel(
|
||||
mean=np.arange(feature_count, dtype=np.float64),
|
||||
scale=np.full(feature_count, 2.0),
|
||||
weights=np.arange(feature_count * 3, dtype=np.float64).reshape(feature_count, 3),
|
||||
margin=0.25,
|
||||
grid=grid,
|
||||
penalty=100.0,
|
||||
)
|
||||
path = tmp_path / "model.npz"
|
||||
|
||||
classifier.save_model(path, model)
|
||||
loaded = classifier.load_model(path)
|
||||
|
||||
assert loaded.grid == grid
|
||||
assert loaded.margin == model.margin
|
||||
assert np.array_equal(loaded.mean, model.mean)
|
||||
assert np.array_equal(loaded.scale, model.scale)
|
||||
assert np.array_equal(loaded.weights, model.weights)
|
||||
|
||||
with np.load(path, allow_pickle=False) as payload:
|
||||
assert payload.files
|
||||
|
||||
|
||||
def test_model_loader_rejects_extra_fields(tmp_path: Path) -> None:
|
||||
path = tmp_path / "model.npz"
|
||||
np.savez(
|
||||
path,
|
||||
classes=np.asarray(classifier.CLASSES),
|
||||
mean=np.zeros(136),
|
||||
scale=np.ones(136),
|
||||
weights=np.zeros((136, 3)),
|
||||
margin=np.asarray(0.1),
|
||||
spectral_grid=np.asarray(2),
|
||||
penalty=np.asarray(100.0),
|
||||
unexpected=np.asarray(1),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="model fields must be exactly"):
|
||||
classifier.load_model(path)
|
||||
|
||||
|
||||
def test_model_loader_rejects_nonfinite_margin(tmp_path: Path) -> None:
|
||||
grid = 2
|
||||
feature_count = classifier.FEATURE_WIDTH + 3 * grid * grid
|
||||
path = tmp_path / "model.npz"
|
||||
np.savez(
|
||||
path,
|
||||
classes=np.asarray(classifier.CLASSES),
|
||||
mean=np.zeros(feature_count),
|
||||
scale=np.ones(feature_count),
|
||||
weights=np.zeros((feature_count, 3)),
|
||||
margin=np.asarray(np.nan),
|
||||
spectral_grid=np.asarray(grid),
|
||||
penalty=np.asarray(100.0),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="margin must be finite"):
|
||||
classifier.load_model(path)
|
||||
|
||||
|
||||
def test_calibration_requires_unknown_rows() -> None:
|
||||
with pytest.raises(ValueError, match="must contain unknown"):
|
||||
classifier.calibrate_margin(
|
||||
np.zeros((2, 3)),
|
||||
np.asarray([0, 1]),
|
||||
unknown_fpr=0.01,
|
||||
)
|
||||
@@ -3449,7 +3449,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.5"
|
||||
version = "0.41.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "c2pa-python" },
|
||||
@@ -3584,6 +3584,13 @@ qwen-zimage = [
|
||||
{ name = "torchvision" },
|
||||
{ name = "transformers" },
|
||||
]
|
||||
source-classify = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "opencv-python-headless", version = "4.11.0.86", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "opencv-python-headless", version = "4.14.0.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
]
|
||||
text-draft = [
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "paddleocr" },
|
||||
@@ -3637,6 +3644,7 @@ requires-dist = [
|
||||
{ name = "huggingface-hub", marker = "extra == 'dev'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'lama'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'migan'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'source-classify'", specifier = ">=0.20.0" },
|
||||
{ name = "huggingface-hub", marker = "extra == 'text-draft'", specifier = ">=0.20.0" },
|
||||
{ name = "invisible-watermark", marker = "extra == 'dev'", specifier = ">=0.2.0" },
|
||||
{ name = "numpy", marker = "python_full_version >= '3.13' and extra == 'pixels'", specifier = ">=2" },
|
||||
@@ -3665,10 +3673,11 @@ requires-dist = [
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'classify'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'detect'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'diffusion'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'source-classify'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["pixels"], marker = "extra == 'visible'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["qwen-zimage", "lama"], marker = "extra == 'text-restoration'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video"], marker = "extra == 'dev'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video", "heif", "detect", "trustmark", "text-restoration", "migan", "classify"], marker = "extra == 'all'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["video", "heif", "detect", "trustmark", "text-restoration", "migan", "classify", "source-classify"], marker = "extra == 'all'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'lama'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'migan'" },
|
||||
{ name = "remove-ai-watermarks", extras = ["visible"], marker = "extra == 'video'" },
|
||||
@@ -3684,7 +3693,7 @@ requires-dist = [
|
||||
{ name = "transformers", marker = "extra == 'diffusion'", specifier = ">=5,<6" },
|
||||
{ name = "trustmark", marker = "python_full_version < '3.13' and extra == 'trustmark'", specifier = ">=0.8.0" },
|
||||
]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "classify-onnx", "lama", "migan", "dev", "all"]
|
||||
provides-extras = ["pixels", "heif", "visible", "video", "detect", "diffusion", "qwen-zimage", "text-restoration", "text-draft", "trustmark", "classify", "classify-onnx", "source-classify", "lama", "migan", "dev", "all"]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
|
||||
Reference in new issue
Block a user