mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-15 05:50:45 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf63e8e187 | ||
|
|
d4181f3277 | ||
|
|
bd14f40c7d | ||
|
|
c4edc2820d | ||
|
|
927429d349 |
No files matched your search
@@ -58,7 +58,10 @@ finding.
|
||||
3. Run the applicable fresh and historical discovery modes from the reference.
|
||||
Use `corpus_gap_scan.py --since YYYY-MM-DD` for the overlapping fresh window
|
||||
and omit `--since` for the complete history. Report counts per input shape
|
||||
and per outcome; an example does not establish coverage.
|
||||
and per outcome; an example does not establish coverage. Give every code and
|
||||
corpus snapshot a distinct `--report` path. The adjacent
|
||||
`.progress.jsonl` checkpoint is the live progress source and makes the same
|
||||
command resumable; use `--restart` only when intentionally discarding it.
|
||||
4. Triage every candidate into one of: parser/serialization gap, missing vendor
|
||||
attribution, new visible mark, possible invisible signal, format/read error,
|
||||
expected non-AI provenance, or false candidate. Preserve unresolved items as
|
||||
|
||||
@@ -8,7 +8,7 @@ historical data is sufficient.
|
||||
| Surface | Discovery evidence | Regression check | Removal proof |
|
||||
| --- | --- | --- | --- |
|
||||
| C2PA, EXIF, XMP, IPTC, PNG text, TC260, soft bindings, and container metadata | `scripts/corpus_gap_scan.py` plus review of unattributed and partially attributed reports | `scripts/sidecar_regression.py`; compare record-based and file-based identification when collection changes | `scripts/metadata_removal_audit.py`; signal absent after strip and decoded pixels/streams unchanged |
|
||||
| Registered visible image marks | `scripts/visible_positives.py`; metadata-derived vendor cohorts and blinded sheets for uncovered labels | `scripts/visible_eval.py`, registered examples, and full-corpus crossfire | `scripts/visible_removal_audit.py`; same-mark redetection plus visual residual review and fill-quality checks |
|
||||
| Registered visible image marks | `scripts/visible_positives.py` records strict and metadata-corroborated results, metadata cohorts, and blinded native-detail bands for uncovered cohorts plus an unbiased quiet sample | `scripts/visible_eval.py` exposes registry members without adjudicated scope, registered examples, and full-corpus crossfire | `scripts/visible_removal_audit.py`; same-mark redetection plus visual residual review and fill-quality checks |
|
||||
| Candidate China-AIGC visible marks | `scripts/vendor_cohort_harvest.py` partitions by producer identity without using pixels; inspect full-width top and bottom sheets | `scripts/vendor_mark_calibrate.py` against independently labelled positives, clean negatives, and neighbouring marks | Register only after the candidate gate survives full-corpus crossfire; then use the normal visible-removal audit |
|
||||
| Registered visible video marks | Review provider-labelled or controlled complete clips, not isolated frames alone | Full-clip tests and the local real-provider audit described in `docs/verification-plan.md` | Preserve sequence, timing, duration, and audio; second-pass detection must be quiet |
|
||||
| Locally decodable invisible image marks, including open DWT-DCT and TrustMark | Positive-control-gated corpus or constructed carriers; a negative on an uncontrolled carrier is inconclusive | `scripts/watermark_benchmark.py` and the relevant decoder tests with the required extras installed | Re-run the same decoder on output and retain a positive control in the same run |
|
||||
@@ -42,6 +42,12 @@ compressed pixels, but its inventory should be reconciled with the current
|
||||
metadata constants and registry on every audit. Do not infer completeness from
|
||||
`No gap candidates` until the other rows of the matrix also ran.
|
||||
|
||||
For a visible discovery pass, use a fresh report path and pass `--sheets` to
|
||||
`visible_positives.py`. Review the numbered PNG files before opening
|
||||
`MANIFEST_DO_NOT_OPEN.csv`; metadata cohorts select where a mark may occur but do
|
||||
not label the pixels. The separate `unbiased-quiet` stratum is required because a
|
||||
new provider may carry neither a mapped producer code nor readable provenance.
|
||||
|
||||
## When to replay old data
|
||||
|
||||
Yes, old data must be rerun. Current code can learn to parse an old
|
||||
|
||||
@@ -142,17 +142,8 @@ jobs:
|
||||
--event workflow_dispatch \
|
||||
--limit 10 \
|
||||
--json databaseId,createdAt \
|
||||
| python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
dispatched_at = sys.argv[1]
|
||||
runs = json.load(sys.stdin)
|
||||
print(next(
|
||||
(run["databaseId"] for run in runs if run["createdAt"] >= dispatched_at),
|
||||
"",
|
||||
))
|
||||
' "$dispatched_at"
|
||||
| python3 -c "import json,sys; t=sys.argv[1]; print(next((r['databaseId'] for r in json.load(sys.stdin) if r['createdAt']>=t), ''))" \
|
||||
"$dispatched_at"
|
||||
)
|
||||
if [ -n "$run_id" ]; then
|
||||
break
|
||||
|
||||
@@ -133,6 +133,8 @@ inspection includes the native TC260 `AIGC` tag in
|
||||
`moov.udta.meta.keys/ilst`, including a `moov` placed after the media payload,
|
||||
plus the QuickTime-form `meta` variants Doubao's iOS export writes (a bare
|
||||
`meta` box as a direct `moov` child, and a keyless `hdlr=mdir` metadata list).
|
||||
Keyed `workflow` and `prompt` entries in the same metadata-list structure are
|
||||
also detected and removed, including ComfyUI exports.
|
||||
MKV and WebM inspection reads the normative
|
||||
`Segment.Tags.Tag.SimpleTag` placement. AVI uses `LIST/INFO/AIGC`, while FLV
|
||||
uses `script.onMetaData.AIGC`. The non-ISOBMFF formats are remuxed with stream
|
||||
@@ -350,9 +352,9 @@ Visible mark support includes:
|
||||
- one calibrated Samsung Galaxy AI label variant.
|
||||
|
||||
Metadata and provenance inspection covers C2PA, EXIF, XMP, IPTC, common
|
||||
generator parameters, China TC260 AIGC labels, and several vendor specific
|
||||
signals. Optional decoders add support for open DWT-DCT watermarks and Adobe
|
||||
TrustMark.
|
||||
generator parameters in image and video containers, China TC260 AIGC labels,
|
||||
and several vendor specific signals. Optional decoders add support for open
|
||||
DWT-DCT watermarks and Adobe TrustMark.
|
||||
|
||||
The exact support matrix, including important locale and detector limits, lives
|
||||
in [supported signals](docs/supported-signals.md). The
|
||||
@@ -369,8 +371,9 @@ Visible removal follows three steps:
|
||||
|
||||
Metadata removal uses format aware stripping. JPEG metadata removal preserves
|
||||
the encoded image scan instead of recompressing it. Native MP4/MOV TC260 values
|
||||
are blanked without changing box sizes or media offsets. Other supported
|
||||
containers use their corresponding metadata path.
|
||||
and keyed generation parameters are blanked without changing box sizes or
|
||||
media offsets. Other supported containers use their corresponding metadata
|
||||
path.
|
||||
|
||||
Invisible removal is different. It regenerates the image through a diffusion
|
||||
pipeline to disrupt pixel and frequency domain watermarks. This changes the
|
||||
|
||||
+4
-3
@@ -316,10 +316,11 @@ delegates to the same verified metadata scanner and stripper as the generic
|
||||
streams are not transcoded. For MP4 and MOV, this includes the native TC260
|
||||
`AIGC` key and JSON value stored in `moov.udta.meta.keys/ilst`, plus the
|
||||
QuickTime-form `meta` variants Doubao's iOS export writes (a bare `meta` box
|
||||
as a direct `moov` child, and a keyless `hdlr=mdir` metadata list). The inspector
|
||||
seeks past a large `mdat` to find a tail `moov`. Removal stream-copies the
|
||||
as a direct `moov` child, and a keyless `hdlr=mdir` metadata list). Keyed
|
||||
`workflow` and `prompt` entries in the same metadata list are also inspected.
|
||||
The inspector seeks past a large `mdat` to find a tail `moov`. Removal stream-copies the
|
||||
container in bounded chunks, converts supported top-level provenance boxes to
|
||||
same-size `free` boxes, and blanks the TC260 key/value in place. Box sizes,
|
||||
same-size `free` boxes, and blanks the TC260 or generation key/value in place. Box sizes,
|
||||
media offsets, and encoded stream bytes do not move; the result is atomically
|
||||
published only after the complete copy succeeds.
|
||||
|
||||
|
||||
@@ -222,7 +222,9 @@ alone is spaced out). The MP4/MOV/M4V/M4A removal path first validates the top-l
|
||||
box walk, then copies the source to a sibling temporary file in bounded chunks.
|
||||
Supported C2PA/JUMBF/AI-label boxes become same-size `free` boxes with blank
|
||||
payloads; TC260 removal changes the four-byte key to `free` and blanks only the
|
||||
validated JSON value with same-length spaces. This preserves every box size,
|
||||
validated JSON value with same-length spaces. Keyed generation metadata such
|
||||
as `workflow` and `prompt` uses the same structural walk; removal blanks the
|
||||
complete key and value in place. This preserves every box size,
|
||||
`stco`/`co64` offset, encoded stream byte, and source-sized memory bound.
|
||||
Publication is atomic, and a malformed top-level walk is copied unchanged. A
|
||||
generic `AIGC` key whose value has no TC260 field is ignored.
|
||||
@@ -508,13 +510,14 @@ convert `signingCredential.untrusted` into trusted based on a vendor-name match.
|
||||
Vendor attribution comes from the registry in
|
||||
[`_internal/constants.py`](../src/remove_ai_watermarks/_internal/constants.py). Derived
|
||||
issuer and platform maps should not be maintained separately.
|
||||
For an AI C2PA claim, a recognized product in `claim_generator` takes precedence
|
||||
over the certificate issuer: an application can sign through an upstream model
|
||||
provider without becoming that provider's product. Only exact product mappings
|
||||
receive this precedence; an unknown claim generator still falls back to issuer
|
||||
attribution. An unmapped issuer org reads as unknown-signer C2PA with no platform;
|
||||
that is how Ideogram was surfaced (4 private-corpus files signed "Ideogram, Inc",
|
||||
2026-08-08) before its vendor row was added on 2026-08-27.
|
||||
A recognized product in `claim_generator` takes precedence over the certificate
|
||||
issuer: an application can sign through an upstream model provider without
|
||||
becoming that provider's product. Exact products are also useful provenance on a
|
||||
non-AI edit, such as CapCut; recognizing the product does not itself change the AI
|
||||
verdict. An unknown claim generator falls back to issuer attribution. An unmapped
|
||||
issuer org reads as unknown-signer C2PA with no platform; that is how Ideogram was
|
||||
surfaced (4 private-corpus files signed "Ideogram, Inc", 2026-08-08) before its
|
||||
vendor row was added on 2026-08-27.
|
||||
|
||||
### Metadata scanning and stripping
|
||||
|
||||
@@ -541,10 +544,18 @@ Key contracts:
|
||||
asserting that the pixels were generated; explicit `aigc_info` discriminator
|
||||
values and Dreamina `exportType=generation` do assert AI origin. Ordinary
|
||||
Aweme, retouch, and `lv` editor exports are preserved.
|
||||
- EXIF reads fall back to Pillow's decoded `getexif()` mapping when a container
|
||||
exposes no ordinary `info["exif"]` blob. This covers ImageMagick PNG raw EXIF
|
||||
profiles and keeps portable-record identification aligned with the file path.
|
||||
- Structured `generationParams.modelName` JSON is reduced to its recognized
|
||||
model name before attribution, so prompts and the surrounding JSON never
|
||||
become a platform label.
|
||||
- ISOBMFF containers use
|
||||
[`_internal/isobmff.py`](../src/remove_ai_watermarks/_internal/isobmff.py).
|
||||
- Native MP4/MOV TC260 `AIGC` entries are read from
|
||||
`moov.udta.meta.keys/ilst` and blanked without changing box sizes.
|
||||
- Native MP4/MOV `workflow` and `prompt` entries use that keyed metadata-list
|
||||
reader and the same offset-preserving removal path.
|
||||
- Native MKV/WebM TC260 `AIGC` entries are read from
|
||||
`Segment.Tags.Tag.SimpleTag` and removed through the ffmpeg stream-copy path.
|
||||
- Native AVI and FLV TC260 entries are read from `LIST/INFO/AIGC` and
|
||||
|
||||
+2
-1
@@ -568,7 +568,8 @@ field records that source audio is copied if present and not watermark-verified.
|
||||
MP4/MOV
|
||||
inspection recognizes the native TC260 `AIGC` entry in
|
||||
`moov.udta.meta.keys/ilst` and the QuickTime-form `meta` variants Doubao's iOS
|
||||
export writes; its removal preserves container size and encoded
|
||||
export writes. It also recognizes keyed `workflow` and `prompt` generation
|
||||
metadata in the same metadata-list structure; removal preserves container size and encoded
|
||||
stream bytes. MP4/MOV/M4V are copied in bounded chunks, so a large `mdat` is not
|
||||
loaded into memory; publication is atomic. MKV/WebM inspection recognizes the corresponding
|
||||
`Segment.Tags.Tag.SimpleTag` representation; its removal requires ffmpeg for a
|
||||
|
||||
@@ -80,17 +80,21 @@ The learned backends download model files on first use.
|
||||
The inspection and stripping code handles signals in these groups:
|
||||
|
||||
- C2PA Content Credentials and supported cloud manifest references;
|
||||
- EXIF and XMP generator fields;
|
||||
- EXIF and XMP generator fields, including structured
|
||||
`generationParams.modelName` values from supported generators;
|
||||
- exact app-export provenance and AIGC disclosures from supported
|
||||
ByteDance-family products, with product-only provenance excluded from the
|
||||
generated-image verdict;
|
||||
- IPTC AI disclosure fields;
|
||||
- PNG text chunks and embedded generation parameters;
|
||||
- PNG text chunks and embedded generation parameters, including keyed
|
||||
`workflow` and `prompt` tags in MP4/MOV metadata lists;
|
||||
- China TC260 AIGC labels in supported image placements and the normative
|
||||
MP4/MOV `moov.udta.meta.keys/ilst`, MKV/WebM
|
||||
`Segment.Tags.Tag.SimpleTag`, AVI `LIST/INFO/AIGC`, and FLV
|
||||
`script.onMetaData.AIGC` placements;
|
||||
- xAI and Grok EXIF signature fields;
|
||||
- xAI and Grok signature-plus-UUID pairs in ordinary EXIF, ImageMagick PNG raw
|
||||
EXIF profiles, XMP description/creator fields, PNG Description/Author text,
|
||||
and IPTC Caption-Abstract/By-line fields;
|
||||
- Samsung AI editing markers;
|
||||
- Hugging Face job metadata;
|
||||
- open Stable Diffusion style DWT-DCT watermarks with the `detect` extra;
|
||||
|
||||
@@ -498,6 +498,15 @@ the metadata-serialization probe inside that workflow. It now records raw
|
||||
markers, input shapes, identify errors, integrity clashes, unattributed results,
|
||||
and blind-marker candidates, but it does not cover visible marks, proprietary
|
||||
invisible marks, detection/removal parity, or historical crossfire.
|
||||
The scan runs files in parallel and checkpoints each completed row beside the
|
||||
requested CSV as `.progress.jsonl`; repeat the same command to resume an
|
||||
interrupted run, and use a distinct report path for every code/corpus snapshot.
|
||||
The visible counterpart, `scripts/visible_positives.py`, records both strict and
|
||||
metadata-corroborated registry results in a resumable JSONL. With `--sheets` it
|
||||
adds blinded, native-resolution top and bottom bands from quiet metadata cohorts
|
||||
and from an unbiased quiet sample. The latter keeps a provider with no mapped
|
||||
metadata from being excluded by construction; neither stratum is ground truth
|
||||
until the pixels are adjudicated.
|
||||
|
||||
Run a fresh-delta discovery review weekly after the retained corpus is
|
||||
refreshed, using `corpus_gap_scan.py --since` with an overlap from the preceding
|
||||
|
||||
@@ -27,6 +27,18 @@ provider. For AI claims, exact product mappings in `claim_generator` therefore
|
||||
take precedence over issuer attribution. Supported mappings include Higgsfield
|
||||
AI, Recraft, Topaz Labs Image API, and TikTok Ad Creative Toolbox; an unknown
|
||||
claim generator still falls back to the certificate issuer.
|
||||
Exact claim-generator mappings also cover `Grok Imagine` as xAI Grok Imagine
|
||||
and `Suno` as Suno. Both can carry a valid AI source claim without a separately
|
||||
readable issuer, so issuer-only attribution leaves the platform blank.
|
||||
|
||||
Some signers expose the product in `signature_info` while retaining a generic
|
||||
claim generator. xAI Grok Imagine is recognized from that signed identity even
|
||||
when the generator is only `c2pa-rs`; Producer.ai is handled the same way.
|
||||
FastVid has an exact claim-generator mapping. SPRING (SG) PTE. LTD. is retained
|
||||
as the signer on AI-source claims without guessing which of its products created
|
||||
the file. TikTok, ByteDance, CapCut, Anthropic Claude file provenance, and vivo
|
||||
Camera are neutral signer or capture attributions: without an AI source assertion,
|
||||
none of them changes the AI verdict.
|
||||
|
||||
**OpenRouter raster metadata survey (2026-09-02):** a paid Recraft V3 API
|
||||
output carried a valid C2PA claim with `claim_generator` `recraft.ai`, a
|
||||
@@ -36,20 +48,23 @@ Riverflow V2 Fast API outputs carried no locally readable C2PA, XMP, EXIF, or
|
||||
container metadata. Their absence is evidence about those exact files and API
|
||||
routes, not a claim that every Krea or Riverflow export is metadata-free.
|
||||
|
||||
**ByteDance Volcano Engine (Volcengine)** — the cloud behind Doubao / Jimeng — signs its AI image output with a cert from `certificate_center@volcengine.com` + `trainedAlgorithmicMedia` (issuer `b"volcengine"` → "ByteDance (Volcano Engine)", platform "ByteDance Volcano Engine"); note this is the C2PA-signed surface, distinct from the XMP/PNG TC260 `AIGC` label Doubao also uses. ByteDance's **international brand (BytePlus / Seedream / Seededit)** signs the same content as **"Byteplus Pte. Ltd."**. The bare `volcengine` needle missed it, so BytePlus output was mis-attributed to "Adobe Firefly" through an incidental "Adobe XMP" toolkit string. Issuer `b"Byteplus"` maps directly to "BytePlus (ByteDance)". ByteDance's consumer app **Dreamina** (the international Jimeng brand) signs as **"Bytedance Pte. Ltd."** with a `Dreamina/x.y` claim generator but, unlike the Volcano Engine surface, ships **no `trainedAlgorithmicMedia`**. Issuer `b"Dreamina"` maps to "ByteDance Dreamina" with **`asserts_ai=True`**. Registering the broader **issuer** `b"Bytedance Pte"` was deliberately avoided because that same entity also signs non-AI CapCut edits; keying on the `Dreamina` generator token is precise.
|
||||
- **EXIF/XMP/PNG-text generator tag (caught by `exif_generator`):** **Ideogram** writes EXIF `Make="Ideogram AI"` (collected 2026-05-24 — no C2PA, no SynthID, no imwatermark; the Make tag is the only signal). Additional verified generator stamps include **NovelAI** (`Software`, `Source`, and `Title` PNG text chunks), **Reve** (`Software` or XMP `CreatorTool` = `reve.com`), and **Aphrodite AI** (`Make` or `Software` = `Aphrodite AI`).
|
||||
**ByteDance Volcano Engine (Volcengine)** — the cloud behind Doubao / Jimeng — signs its AI image output with a cert from `certificate_center@volcengine.com` + `trainedAlgorithmicMedia` (issuer `b"volcengine"` → "ByteDance (Volcano Engine)", platform "ByteDance Volcano Engine"); note this is the C2PA-signed surface, distinct from the XMP/PNG TC260 `AIGC` label Doubao also uses. ByteDance's **international brand (BytePlus / Seedream / Seededit)** signs the same content as **"Byteplus Pte. Ltd."**. The bare `volcengine` needle missed it, so BytePlus output was mis-attributed to "Adobe Firefly" through an incidental "Adobe XMP" toolkit string. Issuer `b"Byteplus"` maps directly to "BytePlus (ByteDance)". ByteDance's consumer app **Dreamina** (the international Jimeng brand) signs as **"Bytedance Pte. Ltd."** with a `Dreamina/x.y` claim generator but, unlike the Volcano Engine surface, ships **no `trainedAlgorithmicMedia`**. Issuer `b"Dreamina"` maps to "ByteDance Dreamina" with **`asserts_ai=True`**. The broader **issuer** `b"Bytedance Pte"` is registered only as a neutral signer because that same entity also signs non-AI CapCut edits; it has no AI platform and cannot assert an AI verdict. Keying the product attribution on the `Dreamina` generator token remains the precise path.
|
||||
- **EXIF/XMP/PNG-text generator tag (caught by `exif_generator`):** **Ideogram** writes EXIF `Make="Ideogram AI"` (collected 2026-05-24 — no C2PA, no SynthID, no imwatermark; the Make tag is the only signal). Additional verified generator stamps include **NovelAI** (`Software`, `Source`, and `Title` PNG text chunks), **Reve** (`Software` or XMP `CreatorTool` = `reve.com`), and **Aphrodite AI** (`Make` or `Software` = `Aphrodite AI`). A JSON `generationParams.modelName` value is parsed structurally and reduced to a recognized model name, rather than exposing its surrounding prompt JSON as a platform label.
|
||||
- **App-export provenance and AIGC JSON:** supported ByteDance-family exports can place a JSON object in EXIF `ImageDescription` or `UserComment`, independently of C2PA or TC260. Exact `product` values for Doubao, Xinghui, and Dreamina are removable product provenance, but do not alone prove that the pixels were generated. Dreamina additionally requires `exportType=generation` for that verdict. A nested Aweme `aigc_type=1` or private ByteDance `aigc_label_type=1` / `2` is an AIGC disclosure; `0` is inconclusive and can occur on a Dreamina generation export. Plain Aweme, retouch, and `lv` exports are preserved. The lower-case private field is deliberately not interpreted as the normative TC260 `Label`, whose values `1` / `2` / `3` mean generated / possibly generated / suspected generated under [GB 45438-2025](https://www.tc260.org.cn/upload/2025-03-15/1742009439794081593.pdf).
|
||||
- **xAI / Grok — its own EXIF signature scheme, NOT C2PA (DETECTED by `metadata.xai_signature`, built 2026-05-26).**
|
||||
- **xAI / Grok signature scheme (DETECTED by `metadata.xai_signature`, built 2026-05-26, expanded 2026-09-13).**
|
||||
|
||||
Grok JPEG downloads (Aurora model) carry **no C2PA, no XMP, no SynthID, no IPTC** — only EXIF `Artist` = a UUID and EXIF `ImageDescription` = `Signature: <base64>` (a crypto signature, unverifiable locally without xAI's public key). This empirically kills the earlier unverified "xAI signs C2PA as xAI" lead — xAI is not even a C2PA member. `exif_generator` misses it (neither field holds an `AI_GENERATOR_TOKENS` token), so a dedicated detector `xai_signature(path)` matches the pair (`ImageDescription ~ ^Signature: [A-Za-z0-9+/=]{64,}` AND UUID `Artist`); wired into `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify` (signal `xai_signature`, platform "xAI (Grok / Aurora)").
|
||||
The sampled Grok JPEG downloads from the Aurora route carry **no C2PA, no XMP, no SynthID, no IPTC** — only EXIF `Artist` = a UUID and EXIF `ImageDescription` = `Signature: <base64>` (a crypto signature, unverifiable locally without xAI's public key). This is route-specific, not a product-wide no-C2PA claim: newer Grok Imagine output can carry a valid AI-source C2PA claim whose `claim_generator` is `Grok Imagine`. `exif_generator` misses the legacy pair (neither field holds an `AI_GENERATOR_TOKENS` token), so a dedicated detector `xai_signature(path)` requires both the long signature shape and a UUID; it is wired into `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify` (signal `xai_signature`, platform "xAI (Grok / Aurora)").
|
||||
|
||||
Metadata-preserving transcodes can move the same two values out of EXIF. Confirmed equivalents are XMP `dc:description` + `dc:creator`, PNG `Description` + `Author`, and IPTC `Caption-Abstract` + `By-line`. Detection requires one of those exact field pairs and keeps XMP values within the same RDF description, so an unrelated document UUID cannot complete a signature-shaped caption. A signature-shaped caption without its paired creator UUID, or a UUID without the signature blob, remains ordinary metadata.
|
||||
|
||||
**Format confirmed stable across n=3 genuine generations:** exactly three EXIF tags (`Artist`, `ExifOffset`, `ImageDescription`), `Signature:` prefix constant, base64 payload 300-1004 chars. Two capture facts: (a) the `Artist` UUID **equals the public image id** in the asset URL (`https://imagine-public.x.ai/imagine-public/images/<uuid>.jpg`), so it is NOT a private per-user secret — only the `Signature` blob is; (b) the Grok web-UI image is a re-encoded **WebP with no signature** — the EXIF survives only in the *original* JPEG (download button or that public tokenless URL), which is why screenshots / re-encodes are metadata-stripped. A real fixture `data/fixtures/provenance/grok-1.jpg` plus **synthetic** JPEG fixtures (fake UUID + fake `Signature:` blob) cover the detector; never add a real Grok image carrying private content (the repo is public).
|
||||
|
||||
**Stripped on removal too:** `remove_ai_metadata` calls `_scrub_ai_exif` on
|
||||
JPEG EXIF, which deletes the xAI Signature and UUID Artist pair plus supported
|
||||
AI generator values while retaining unrelated camera and editor EXIF. The
|
||||
shared `xai_signature_pair` helper is the single source of truth for the
|
||||
pair. On the ISOBMFF path, `blank_ai_exif_tokens` provides the corresponding
|
||||
**Stripped on removal too:** `remove_ai_metadata` deletes both members of the
|
||||
pair from EXIF, XMP, PNG text, or IPTC while retaining unrelated camera and
|
||||
editor metadata. Converted PNGs can retain the original pair in an ImageMagick
|
||||
raw EXIF profile; Pillow exposes that form through `getexif()` even when
|
||||
`info["exif"]` is absent, so detection, removal, and portable records share that
|
||||
fallback. On the ISOBMFF path, `blank_ai_exif_tokens` provides the corresponding
|
||||
in-place scrub for supported EXIF values, TC260 AIGC blocks, and the xAI pair.
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). The label says only "this is AI", but its `ContentProducer` names the signing entity — `001` + `1` + an 18-char Unified Social Credit Code + a 5-digit product suffix, normalized by `metadata.uscc_of`, or for a few generators a bare product name. `KnownMark.tc260_producer_codes` maps the codes settled per vendor by `scripts/vendor_cohort_harvest.py` to registry mark keys, so an AIGC image relaxes only the detector of the product it actually carries; an unmapped or absent producer leaves every product strict. The same mapping supplies the manufacturer for provenance-conflict checks. A code identifies a legal entity, not necessarily one brand, so a hosting or aggregating platform that signs for several apps is a recall bet rather than a proof.
|
||||
|
||||
@@ -88,7 +103,10 @@ payloads. Removal remuxes either container through ffmpeg with stream copy.
|
||||
|
||||
Native TC260 MP4/MOV tags do not live in those top-level provenance boxes.
|
||||
`tc260_aigc_payloads` separately seeks through `moov.udta.meta.keys/ilst`, so
|
||||
the normative tag is also found when a large `mdat` precedes `moov`.
|
||||
the normative tag is also found when a large `mdat` precedes `moov`. The parser
|
||||
canonicalizes known fields across PascalCase and lowerCamelCase serializations.
|
||||
The same keyed metadata-list walk exposes generation `workflow` and `prompt`
|
||||
entries, including ComfyUI exports, and blanks them in place during removal.
|
||||
|
||||
**Meta-box XMP and EXIF removal are handled in place:** an AI-label XMP packet
|
||||
stored as a meta-box `mime` item is blanked by
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
# on it, including the ComfyUI node package. The console script below carries
|
||||
# the same weight, since users have it on PATH.
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.1"
|
||||
version = "0.40.3"
|
||||
description = "AI watermark remover for visible, invisible, and provenance marks in images and video"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11,<3.15"
|
||||
|
||||
+3
-3
@@ -9,7 +9,7 @@ explicitly names a tracked canonical result.
|
||||
|
||||
| Script | Purpose |
|
||||
| --- | --- |
|
||||
| `corpus_gap_scan.py` | Inventory `identify`, metadata markers, unattributed outcomes, errors, and gap candidates over a full or date-bounded corpus. |
|
||||
| `corpus_gap_scan.py` | Inventory `identify`, metadata markers, normalized C2PA review fields, unattributed outcomes, errors, and gap candidates over a full or date-bounded corpus. Its schema-versioned checkpoint resumes safely after interruption. |
|
||||
| `detection_timing.py` | Record per-method metadata and verdict timings. |
|
||||
| `detection_timing_report.py` | Aggregate timing records by method and segment. |
|
||||
| `fidelity_metrics.py` | Compute objective image-fidelity metrics for paired outputs. |
|
||||
@@ -26,7 +26,7 @@ explicitly names a tracked canonical result.
|
||||
| `sidecar_regression.py` | Compare current identification with recorded sidecars. |
|
||||
| `smoke_matrix.py` | Exercise CLI parameter choices on real local data. |
|
||||
| `video_fidelity_probe.py` | Compare delivered video fidelity with its source. |
|
||||
| `visible_eval.py` | Benchmark registered visible-mark detectors. |
|
||||
| `visible_eval.py` | Benchmark the adjudicated registry subset and expose unmeasured marks as scope-zero rows. |
|
||||
| `visible_removal_audit.py` | Audit visible-removal results over a local corpus. |
|
||||
| `watermark_benchmark.py` | Run hash-pinned image cases while keeping detection, removal observation, and fidelity separate. |
|
||||
| `watermark_benchmark_report.py` | Aggregate repeated benchmark results with artifact-aware counts and separate cold/warm timing. |
|
||||
@@ -46,7 +46,7 @@ explicitly names a tracked canonical result.
|
||||
| `vendor_mark_calibrate.py` | Calibrate a candidate vendor text detector. |
|
||||
| `visible_alpha_solve.py` | Rebuild visible-watermark alpha assets from controlled captures and the cleared Qwen symbol fixture. |
|
||||
| `visible_groundtruth.py` | Consolidate blinded contact-sheet labels into ground truth. |
|
||||
| `visible_positives.py` | List corpus images carrying a registered visible mark. |
|
||||
| `visible_positives.py` | Resumably scan registered marks plus metadata cohorts, and build blinded native-detail bands for uncovered and unbiased quiet samples. |
|
||||
| `visible_recall_sample.py` | Build an unbiased blinded sample for recall measurement. |
|
||||
| `visible_sheets.py` | Build blinded contact sheets for relaxation candidates. |
|
||||
| `watermark_benchmark_cohort.py` | Build the deterministic synthetic DWT-DCT and TrustMark cohort consumed by `watermark_benchmark.py`. |
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Shared deterministic corpus walks and append-safe JSONL checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Collection
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def corpus_day(path: Path, corpus: Path) -> datetime.date | None:
|
||||
"""Read the leading YYYY-MM-DD corpus segment, if present."""
|
||||
try:
|
||||
return datetime.date.fromisoformat(path.relative_to(corpus).parts[0])
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def corpus_files(corpus: Path, since: datetime.date | None, *, suffixes: Collection[str] | None = None) -> list[Path]:
|
||||
"""Return a deterministic recursive walk, optionally date- and suffix-bounded."""
|
||||
if since is None:
|
||||
candidates = corpus.rglob("*")
|
||||
else:
|
||||
roots = [
|
||||
path
|
||||
for path in corpus.iterdir()
|
||||
if path.is_dir() and (day := corpus_day(path, corpus)) is not None and day >= since
|
||||
]
|
||||
candidates = (path for root in roots for path in root.rglob("*"))
|
||||
return sorted(
|
||||
path for path in candidates if path.is_file() and (suffixes is None or path.suffix.lower() in suffixes)
|
||||
)
|
||||
|
||||
|
||||
def repair_jsonl_tail(path: Path) -> None:
|
||||
"""Truncate an interrupted final JSONL fragment before another append."""
|
||||
if not path.exists():
|
||||
return
|
||||
with path.open("rb+") as stream:
|
||||
data = stream.read()
|
||||
if data and not data.endswith(b"\n"):
|
||||
stream.truncate(data.rfind(b"\n") + 1)
|
||||
+215
-107
@@ -2,9 +2,10 @@
|
||||
|
||||
Two jobs in one pass:
|
||||
|
||||
1. **Report** -- run ``identify`` over every image and write one CSV row per file
|
||||
(verdict, platform, confidence, watermarks, signals, raw metadata markers,
|
||||
candidate classes, integrity clashes, and errors).
|
||||
1. **Report** -- run metadata-only ``identify`` over every file and write one CSV
|
||||
row per file (verdict, platform, confidence, watermarks, signals, raw metadata
|
||||
markers, candidate classes, integrity clashes, and errors). Visible and
|
||||
invisible pixel detectors belong to their own corpus audits.
|
||||
2. **Gap audit** -- for every ``unknown``-verdict file, scan only its *metadata
|
||||
region* (PNG text/eXIf chunks, JPEG APPn segments before SOS, or the file
|
||||
head for other containers) for known provenance markers. A marker found there
|
||||
@@ -20,27 +21,51 @@ This is how new detector gaps get found (it is what surfaced the JPEG-EXIF
|
||||
Usage:
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals \\
|
||||
--report .local-eval/detector-report.csv
|
||||
--workers 8 --report .local-eval/detector-report.csv
|
||||
uv run python scripts/corpus_gap_scan.py --corpus .local-eval/originals \\
|
||||
--since 2026-09-01 --report .local-eval/detector-report-weekly.csv
|
||||
|
||||
Rows stream into ``<report>.progress.jsonl`` and resume by relative path after an
|
||||
interruption. A schema version prevents an older checkpoint from silently
|
||||
producing blank fields after the report grows. Use a new report name for a new
|
||||
code/corpus snapshot, or pass ``--restart`` when intentionally replacing the
|
||||
checkpoint. The final CSV is written atomically after every selected row is
|
||||
present. C2PA candidates include normalized signer, claim-generator, action,
|
||||
source-type, and validation fields for aggregate review without reopening every
|
||||
manifest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
from concurrent.futures import FIRST_COMPLETED, Future, ProcessPoolExecutor, wait
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import date
|
||||
|
||||
import click
|
||||
from _corpus_scan import corpus_day, corpus_files, repair_jsonl_tail
|
||||
from _plain_console import Console, Table
|
||||
|
||||
from remove_ai_watermarks.identify import _metadata_region as identify_metadata_region
|
||||
from remove_ai_watermarks.identify import identify
|
||||
from remove_ai_watermarks.metadata import scan_head
|
||||
from remove_ai_watermarks.identify import extract_provenance_evidence, identify_from_evidence
|
||||
from remove_ai_watermarks.metadata import (
|
||||
IPTC_AI_FIELD_MARKERS,
|
||||
IPTC_AI_MARKERS,
|
||||
aigc_label_from_metadata,
|
||||
c2pa_marker_in,
|
||||
samsung_genai_in,
|
||||
scan_head,
|
||||
xai_signature_in_metadata,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
@@ -51,22 +76,10 @@ console = Console()
|
||||
# below is the primary guard, this list is the second. Group: C2PA/JUMBF infra,
|
||||
# AI source-type / labeling schemes, and distinctive generator name strings.
|
||||
MARKERS: tuple[bytes, ...] = (
|
||||
# C2PA / JUMBF infrastructure and AI source-type / labeling schemes.
|
||||
b"c2pa",
|
||||
b"jumbf",
|
||||
b"contentauth",
|
||||
# AI source-type / labeling schemes that are meaningful by presence.
|
||||
b"trainedAlgorithmicMedia",
|
||||
b"digitalSourceType",
|
||||
b'"AIGC"',
|
||||
b"<TC260:AIGC>",
|
||||
b"TC260:AIGC",
|
||||
b"tc260.org.cn",
|
||||
b"AISystemUsed",
|
||||
b"SynthID",
|
||||
b"hf-job-id",
|
||||
b"genAIType",
|
||||
b"PhotoEditor_Re_Edit",
|
||||
b"Signature:",
|
||||
# Distinctive multi-word generator strings only. Bare single words (Luma,
|
||||
# Gemini, Sora, ...) are omitted: they collide with unrelated metadata prose
|
||||
# (e.g. "Luma" in Lightroom's EnhanceDenoiseLumaAmount), defeating precision.
|
||||
@@ -80,14 +93,11 @@ MARKERS: tuple[bytes, ...] = (
|
||||
b"Adobe Firefly",
|
||||
b"Black Forest",
|
||||
b"volcengine",
|
||||
b"Doubao",
|
||||
b"\xe8\xb1\x86\xe5\x8c\x85",
|
||||
b"Nano Banana",
|
||||
b"Stability AI",
|
||||
b"Samsung Galaxy",
|
||||
)
|
||||
|
||||
REPORT_FIELDS: tuple[str, ...] = (
|
||||
"report_schema_version",
|
||||
"path",
|
||||
"suffix",
|
||||
"lib_version",
|
||||
@@ -96,16 +106,24 @@ REPORT_FIELDS: tuple[str, ...] = (
|
||||
"confidence",
|
||||
"watermarks",
|
||||
"signals",
|
||||
"c2pa_issuer",
|
||||
"c2pa_claim_generator",
|
||||
"c2pa_actions",
|
||||
"c2pa_source_type",
|
||||
"c2pa_validation",
|
||||
"integrity_clashes",
|
||||
"markers",
|
||||
"candidate_classes",
|
||||
"error",
|
||||
)
|
||||
_REPORT_SCHEMA_VERSION = "2"
|
||||
_DISPLAY_CANDIDATE_LIMIT = 50
|
||||
_WORKER_BATCH_SIZE = 16
|
||||
|
||||
|
||||
def _base_row(path: str, suffix: str, lib_version: str) -> dict[str, str]:
|
||||
row = dict.fromkeys(REPORT_FIELDS, "")
|
||||
row.update(path=path, suffix=suffix, lib_version=lib_version)
|
||||
row.update(report_schema_version=_REPORT_SCHEMA_VERSION, path=path, suffix=suffix, lib_version=lib_version)
|
||||
return row
|
||||
|
||||
|
||||
@@ -122,9 +140,20 @@ def _row(rep, *, path: str, suffix: str, lib_version: str) -> dict[str, str]: #
|
||||
|
||||
|
||||
def _marker_hits(region: bytes) -> list[str]:
|
||||
"""Return marker labels found case-insensitively in one metadata region."""
|
||||
"""Return high-precision marker labels found in one metadata region."""
|
||||
folded = region.lower()
|
||||
return sorted({marker.decode("latin-1", "replace") for marker in MARKERS if marker.lower() in folded})
|
||||
hits = {marker.decode("latin-1", "replace") for marker in MARKERS if marker.lower() in folded}
|
||||
if c2pa_marker_in(region):
|
||||
hits.add("C2PA/JUMBF")
|
||||
if aigc_label_from_metadata(region) is not None:
|
||||
hits.add("TC260 AIGC")
|
||||
if any(marker in region for marker in IPTC_AI_MARKERS + IPTC_AI_FIELD_MARKERS):
|
||||
hits.add("IPTC AI disclosure")
|
||||
if samsung_genai_in(region) is not None:
|
||||
hits.add("Samsung genAIType")
|
||||
if xai_signature_in_metadata(region):
|
||||
hits.add("xAI signature pair")
|
||||
return sorted(hits)
|
||||
|
||||
|
||||
def _marker_hits_for_path(path: Path) -> list[str]:
|
||||
@@ -151,23 +180,119 @@ def _candidate_classes(rep: Any, hits: list[str]) -> list[str]:
|
||||
|
||||
def _day_of(path: Path, corpus: Path) -> date | None:
|
||||
"""Read the leading YYYY-MM-DD corpus segment, if this layout has one."""
|
||||
try:
|
||||
segment = path.relative_to(corpus).parts[0]
|
||||
return date.fromisoformat(segment)
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
return corpus_day(path, corpus)
|
||||
|
||||
|
||||
def _files(corpus: Path, since: date | None) -> list[Path]:
|
||||
"""Return the deterministic corpus walk, optionally bounded by its date segment."""
|
||||
if since is None:
|
||||
return sorted(path for path in corpus.rglob("*") if path.is_file())
|
||||
roots = [
|
||||
path
|
||||
for path in corpus.iterdir()
|
||||
if path.is_dir() and (day := _day_of(path, corpus)) is not None and day >= since
|
||||
]
|
||||
return sorted(path for root in roots for path in root.rglob("*") if path.is_file())
|
||||
return corpus_files(corpus, since)
|
||||
|
||||
|
||||
def _scan_one(args: tuple[str, str, str]) -> dict[str, str]:
|
||||
"""Scan one path in a worker process and always return a report row."""
|
||||
path_str, relative_path, lib_version = args
|
||||
path = Path(path_str)
|
||||
suffix = path.suffix.lower()
|
||||
try:
|
||||
evidence = extract_provenance_evidence(path)
|
||||
rep = identify_from_evidence(evidence)
|
||||
except Exception as exc:
|
||||
log.warning("identify failed on %s: %s", relative_path, exc)
|
||||
row = _base_row(relative_path, suffix, lib_version)
|
||||
row["candidate_classes"] = "identify_error"
|
||||
row["error"] = f"{type(exc).__name__}: {exc}"[:300].replace("\n", " ")
|
||||
else:
|
||||
row = _row(rep, path=relative_path, suffix=suffix, lib_version=lib_version)
|
||||
c2pa = evidence.c2pa_info
|
||||
row.update(
|
||||
c2pa_issuer=str(c2pa.get("issuer", "")),
|
||||
c2pa_claim_generator=str(c2pa.get("claim_generator", "")),
|
||||
c2pa_actions=str(c2pa.get("actions", "")),
|
||||
c2pa_source_type=str(c2pa.get("source_type", "")),
|
||||
c2pa_validation=str(c2pa.get("c2pa_validation_state", "")),
|
||||
)
|
||||
hits = _marker_hits(identify_metadata_region(evidence.scan))
|
||||
row["markers"] = "|".join(hits)
|
||||
row["candidate_classes"] = "|".join(_candidate_classes(rep, hits))
|
||||
return row
|
||||
row["markers"] = "|".join(_marker_hits_for_path(path))
|
||||
return row
|
||||
|
||||
|
||||
def _scan_batch(args: tuple[tuple[str, str, str], ...]) -> list[dict[str, str]]:
|
||||
"""Scan a small path batch to amortize process-pool dispatch overhead."""
|
||||
return [_scan_one(item) for item in args]
|
||||
|
||||
|
||||
def _read_checkpoint(path: Path) -> dict[str, dict[str, str]]:
|
||||
"""Read complete JSONL rows, tolerating an interrupted final write."""
|
||||
rows: dict[str, dict[str, str]] = {}
|
||||
if not path.exists():
|
||||
return rows
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
for line in stream:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
continue
|
||||
if (
|
||||
isinstance(row, dict)
|
||||
and row.get("report_schema_version") == _REPORT_SCHEMA_VERSION
|
||||
and isinstance(row.get("path"), str)
|
||||
):
|
||||
rows[row["path"]] = {field: str(row.get(field, "")) for field in REPORT_FIELDS}
|
||||
return rows
|
||||
|
||||
|
||||
def _repair_checkpoint(path: Path) -> None:
|
||||
"""Remove an interrupted final JSONL fragment before another append."""
|
||||
repair_jsonl_tail(path)
|
||||
|
||||
|
||||
def _summarize(rows: list[dict[str, str]]) -> None:
|
||||
"""Print bounded aggregate output for a completed or resumed report."""
|
||||
verdicts: Counter[str] = Counter()
|
||||
platforms: Counter[str] = Counter()
|
||||
shapes: Counter[str] = Counter()
|
||||
errors = 0
|
||||
for row in rows:
|
||||
shapes[row["suffix"] or "(none)"] += 1
|
||||
if row["error"]:
|
||||
errors += 1
|
||||
elif row["is_ai"] == "True":
|
||||
verdicts["ai"] += 1
|
||||
platforms[row["platform"] or "?"] += 1
|
||||
else:
|
||||
verdicts["unknown"] += 1
|
||||
|
||||
console.print(f"\n[bold]Verdicts:[/bold] AI {verdicts['ai']} | unknown {verdicts['unknown']} | errors {errors}")
|
||||
console.print("[bold]Input shapes:[/bold] " + " | ".join(f"{name} {n}" for name, n in shapes.most_common()))
|
||||
plat = Table(title="AI platforms", show_header=False)
|
||||
for name, count in platforms.most_common():
|
||||
plat.add_row(str(count), name)
|
||||
console.print(plat)
|
||||
|
||||
candidates = [row for row in rows if row["candidate_classes"]]
|
||||
if not candidates:
|
||||
console.print("\n[green]No review candidates in this run.[/green]")
|
||||
return
|
||||
candidate_counts = Counter(candidate for row in candidates for candidate in row["candidate_classes"].split("|"))
|
||||
gap_tokens = Counter(marker for row in rows for marker in row["markers"].split("|") if marker)
|
||||
console.print(
|
||||
f"\n[bold red]Review candidates[/bold red]: {len(candidates)} file(s) "
|
||||
f"({', '.join(f'{name}={n}' for name, n in candidate_counts.most_common())})"
|
||||
)
|
||||
tok = Table(title="metadata markers seen across the run")
|
||||
tok.add_column("count", justify="right")
|
||||
tok.add_column("marker")
|
||||
for name, count in gap_tokens.most_common():
|
||||
tok.add_row(str(count), name)
|
||||
console.print(tok)
|
||||
for row in candidates[:_DISPLAY_CANDIDATE_LIMIT]:
|
||||
detail = f"; markers={row['markers'].replace('|', ', ')}" if row["markers"] else ""
|
||||
console.print(f" {row['path']} -> {row['candidate_classes'].replace('|', ', ')}{detail}")
|
||||
if (hidden := len(candidates) - _DISPLAY_CANDIDATE_LIMIT) > 0:
|
||||
console.print(f" ... {hidden} more candidate row(s); inspect the CSV for the complete set")
|
||||
|
||||
|
||||
@click.command()
|
||||
@@ -191,7 +316,9 @@ def _files(corpus: Path, since: date | None) -> list[Path]:
|
||||
help="Only scan files under YYYY-MM-DD corpus directories on or after this date.",
|
||||
)
|
||||
@click.option("--limit", type=int, default=0, help="Scan at most N files (0 = all).")
|
||||
def main(corpus: Path, report: Path | None, since, limit: int) -> None: # noqa: ANN001
|
||||
@click.option("--workers", type=click.IntRange(min=1), default=max(1, (os.cpu_count() or 4) - 2), show_default=True)
|
||||
@click.option("--restart", is_flag=True, help="Discard this report's checkpoint and scan every selected file again.")
|
||||
def main(corpus: Path, report: Path | None, since, limit: int, workers: int, restart: bool) -> None: # noqa: ANN001
|
||||
logging.basicConfig(level=logging.WARNING, format="%(message)s")
|
||||
report = report or corpus.parent / "detector_report.csv"
|
||||
|
||||
@@ -199,78 +326,59 @@ def main(corpus: Path, report: Path | None, since, limit: int) -> None: # noqa:
|
||||
files = _files(corpus, since_date)
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
console.print(f"Scanning [bold]{len(files)}[/bold] files under {corpus} ...")
|
||||
|
||||
verdicts: Counter[str] = Counter()
|
||||
platforms: Counter[str] = Counter()
|
||||
shapes: Counter[str] = Counter()
|
||||
rows: list[dict[str, str]] = []
|
||||
errors = 0
|
||||
checkpoint = report.with_suffix(report.suffix + ".progress.jsonl")
|
||||
if restart:
|
||||
checkpoint.unlink(missing_ok=True)
|
||||
report.unlink(missing_ok=True)
|
||||
_repair_checkpoint(checkpoint)
|
||||
lib_version = version("remove-ai-watermarks")
|
||||
selected = [(path, str(path.relative_to(corpus))) for path in files]
|
||||
selected_paths = {relative for _, relative in selected}
|
||||
completed = {relative: row for relative, row in _read_checkpoint(checkpoint).items() if relative in selected_paths}
|
||||
todo = [(path, relative) for path, relative in selected if relative not in completed]
|
||||
console.print(
|
||||
f"Scanning [bold]{len(files)}[/bold] files under {corpus}: "
|
||||
f"[bold]{len(completed)}[/bold] checkpointed, [bold]{len(todo)}[/bold] remaining, workers {workers}"
|
||||
)
|
||||
|
||||
with click.progressbar(files, label="identify") as bar:
|
||||
for p in bar:
|
||||
rel = str(p.relative_to(corpus))
|
||||
shapes[p.suffix.lower() or "(none)"] += 1
|
||||
try:
|
||||
rep = identify(p)
|
||||
except Exception as exc:
|
||||
log.warning("identify failed on %s: %s", rel, exc)
|
||||
errors += 1
|
||||
row = _base_row(rel, p.suffix.lower(), lib_version)
|
||||
row["candidate_classes"] = "identify_error"
|
||||
row["error"] = f"{type(exc).__name__}: {exc}"[:300]
|
||||
hits = _marker_hits_for_path(p)
|
||||
row["markers"] = "|".join(hits)
|
||||
rows.append(row)
|
||||
continue
|
||||
row = _row(rep, path=rel, suffix=p.suffix.lower(), lib_version=lib_version)
|
||||
# identify() has already populated scan_head's bounded cache, so this
|
||||
# reuses the exact metadata region rather than reading the file again.
|
||||
hits = _marker_hits_for_path(p)
|
||||
classes = _candidate_classes(rep, hits)
|
||||
row["markers"] = "|".join(hits)
|
||||
row["candidate_classes"] = "|".join(classes)
|
||||
rows.append(row)
|
||||
if rep.is_ai_generated:
|
||||
verdicts["ai"] += 1
|
||||
platforms[rep.platform or "?"] += 1
|
||||
continue
|
||||
verdicts["unknown"] += 1
|
||||
checkpoint.parent.mkdir(parents=True, exist_ok=True)
|
||||
with checkpoint.open("a", encoding="utf-8") as stream, ProcessPoolExecutor(max_workers=workers) as executor:
|
||||
work = iter((str(path), relative, lib_version) for path, relative in todo)
|
||||
|
||||
def submit_batch() -> Future[list[dict[str, str]]] | None:
|
||||
batch = tuple(itertools.islice(work, _WORKER_BATCH_SIZE))
|
||||
return executor.submit(_scan_batch, batch) if batch else None
|
||||
|
||||
pending = {future for _ in range(workers * 2) if (future := submit_batch()) is not None}
|
||||
processed = 0
|
||||
next_progress = 500
|
||||
while pending:
|
||||
ready, pending = wait(pending, return_when=FIRST_COMPLETED)
|
||||
for future in ready:
|
||||
for row in future.result():
|
||||
completed[row["path"]] = row
|
||||
stream.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
processed += 1
|
||||
if replacement := submit_batch():
|
||||
pending.add(replacement)
|
||||
stream.flush()
|
||||
if processed >= next_progress:
|
||||
console.print(
|
||||
f" {processed}/{len(todo)} new; {len(completed)}/{len(files)} total",
|
||||
highlight=False,
|
||||
)
|
||||
next_progress += 500
|
||||
|
||||
rows = [completed[relative] for _, relative in selected]
|
||||
report.parent.mkdir(parents=True, exist_ok=True)
|
||||
with report.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=REPORT_FIELDS)
|
||||
temporary_report = report.with_suffix(report.suffix + ".tmp")
|
||||
with temporary_report.open("w", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=REPORT_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
temporary_report.replace(report)
|
||||
console.print(f"\nWrote [bold]{len(rows)}[/bold] rows -> {report}")
|
||||
|
||||
console.print(f"\n[bold]Verdicts:[/bold] AI {verdicts['ai']} | unknown {verdicts['unknown']} | errors {errors}")
|
||||
console.print("[bold]Input shapes:[/bold] " + " | ".join(f"{name} {n}" for name, n in shapes.most_common()))
|
||||
plat = Table(title="AI platforms", show_header=False)
|
||||
for name, n in platforms.most_common():
|
||||
plat.add_row(str(n), name)
|
||||
console.print(plat)
|
||||
|
||||
candidates = [row for row in rows if row["candidate_classes"]]
|
||||
if candidates:
|
||||
candidate_counts = Counter(candidate for row in candidates for candidate in row["candidate_classes"].split("|"))
|
||||
gap_tokens = Counter(marker for row in rows for marker in row["markers"].split("|") if marker)
|
||||
console.print(
|
||||
f"\n[bold red]Review candidates[/bold red]: {len(candidates)} file(s) "
|
||||
f"({', '.join(f'{name}={n}' for name, n in candidate_counts.most_common())})"
|
||||
)
|
||||
tok = Table(title="metadata markers seen across the run")
|
||||
tok.add_column("count", justify="right")
|
||||
tok.add_column("marker")
|
||||
for name, n in gap_tokens.most_common():
|
||||
tok.add_row(str(n), name)
|
||||
console.print(tok)
|
||||
for row in candidates:
|
||||
detail = f"; markers={row['markers'].replace('|', ', ')}" if row["markers"] else ""
|
||||
console.print(f" {row['path']} -> {row['candidate_classes'].replace('|', ', ')}{detail}")
|
||||
else:
|
||||
console.print("\n[green]No review candidates in this run.[/green]")
|
||||
_summarize(rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -41,7 +41,10 @@ from remove_ai_watermarks.image_io import imread
|
||||
|
||||
GT = Path(".local-eval/textmark-relaxation/groundtruth.jsonl")
|
||||
SNAP = Path(".local-eval/textmark-relaxation/snapshots")
|
||||
MARKS = ("gemini", "doubao", "jimeng", "samsung", "jimeng_pill")
|
||||
# Ground truth currently adjudicates only a subset, but inventory follows the
|
||||
# registry. A newly registered mark therefore appears with scope zero instead of
|
||||
# silently vanishing from the evaluation harness.
|
||||
MARKS = tuple(wr.mark_keys())
|
||||
|
||||
|
||||
def wilson(k: int, n: int) -> tuple[float, float]:
|
||||
@@ -122,7 +125,7 @@ def report(res: dict, prev: dict | None = None) -> None:
|
||||
lo, hi = wilson(tp, n)
|
||||
delta = ""
|
||||
if prev:
|
||||
pc = prev["per"][m]
|
||||
pc = prev["per"].get(m, {})
|
||||
pn = pc.get("tp", 0) + pc.get("fp", 0)
|
||||
if pn:
|
||||
d = tp / n - pc.get("tp", 0) / pn
|
||||
|
||||
+230
-34
@@ -1,4 +1,4 @@
|
||||
"""Parallel detection pass: list every corpus image carrying a known visible mark.
|
||||
"""Scan a retained corpus for registered and still-uncovered visible marks.
|
||||
|
||||
Why this exists separately from `visible_removal_audit.py`: that audit is single-process,
|
||||
so repeated full-dataset sweeps waste time. Its
|
||||
@@ -18,22 +18,43 @@ CRASH TOLERANCE IS NOT OPTIONAL AT THIS SCALE
|
||||
|
||||
Treat input datasets as sensitive and read-only, and keep output gitignored.
|
||||
|
||||
uv run python scripts/visible_positives.py --jobs 6
|
||||
The JSONL is also the input to visible-mark discovery. Each row records strict and
|
||||
metadata-corroborated detector results plus the metadata cohort independently of
|
||||
the pixels. ``--sheets`` then makes blinded, native-detail top/bottom bands from:
|
||||
|
||||
* quiet TC260 producer cohorts, including producer identities absent from the
|
||||
visible-mark registry;
|
||||
* quiet named-platform cohorts;
|
||||
* an unbiased sample of every other quiet image, so a completely novel mark with
|
||||
no useful metadata is not excluded by construction.
|
||||
|
||||
The sheets are candidates for human adjudication, not evidence that a watermark is
|
||||
present. Their manifest is deliberately separate and must stay closed until the
|
||||
image review finishes.
|
||||
|
||||
uv run python scripts/visible_positives.py --corpus .local-eval/originals --jobs 6
|
||||
uv run python scripts/visible_positives.py --since 2026-09-01 --sheets .local-eval/visible-fresh-sheets
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from _corpus_scan import corpus_files, repair_jsonl_tail
|
||||
from _isolated_image_workers import run_batch
|
||||
|
||||
# The package's own format set. An inlined copy here silently skipped .heif, which
|
||||
@@ -43,20 +64,66 @@ from remove_ai_watermarks._internal.constants import SUPPORTED_FORMATS as _EXTS
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
CORPUS = REPO / ".local-eval" / "originals"
|
||||
OUT = REPO / ".local-eval" / "visible-positives.jsonl"
|
||||
PATHS = REPO / ".local-eval" / "visible-positives.txt"
|
||||
_SHEET_BAND_FRACTION = 0.12
|
||||
_SCHEMA_VERSION = 2
|
||||
|
||||
|
||||
def _one(path: str) -> dict[str, object]:
|
||||
from remove_ai_watermarks.api import _provenance_from_report
|
||||
from remove_ai_watermarks.identify import identify
|
||||
from remove_ai_watermarks.image_io import imread
|
||||
from remove_ai_watermarks.watermark_registry import detect_marks
|
||||
from remove_ai_watermarks.metadata import aigc_label, uscc_of
|
||||
from remove_ai_watermarks.watermark_registry import _provenance_confirms_product, known_marks
|
||||
|
||||
try:
|
||||
img = imread(path)
|
||||
source = Path(path)
|
||||
img = imread(source)
|
||||
if img is None:
|
||||
return {"path": path, "keys": [], "status": "unreadable"}
|
||||
return {"path": path, "keys": [d.key for d in detect_marks(img) if d.detected], "status": "ok"}
|
||||
return _error_record(path, "unreadable")
|
||||
report = identify(source, check_visible=False, check_invisible=False)
|
||||
provenance = _provenance_from_report(report, source)
|
||||
strict = []
|
||||
accepted = []
|
||||
for mark in known_marks():
|
||||
strict_detection, relaxed_detection = mark.detect_both(img)
|
||||
if strict_detection.detected:
|
||||
strict.append(mark.key)
|
||||
detection = (
|
||||
relaxed_detection if _provenance_confirms_product(mark.product, provenance) else strict_detection
|
||||
)
|
||||
if detection.detected:
|
||||
accepted.append(mark.key)
|
||||
label = aigc_label(source) or {}
|
||||
producer = str(label.get("ContentProducer") or "")
|
||||
return {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
"path": path,
|
||||
"keys": accepted,
|
||||
"strict_keys": strict,
|
||||
"provenance": sorted(provenance),
|
||||
"platform": report.platform or "",
|
||||
"producer": producer,
|
||||
"uscc": uscc_of(producer) if producer else "",
|
||||
"status": "ok",
|
||||
"error": "",
|
||||
}
|
||||
except Exception as e:
|
||||
return {"path": path, "keys": [], "status": f"error:{type(e).__name__}"}
|
||||
return _error_record(path, f"error:{type(e).__name__}", str(e))
|
||||
|
||||
|
||||
def _error_record(path: str, status: str, error: str = "") -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
"path": path,
|
||||
"keys": [],
|
||||
"strict_keys": [],
|
||||
"provenance": [],
|
||||
"platform": "",
|
||||
"producer": "",
|
||||
"uscc": "",
|
||||
"status": status,
|
||||
"error": error[:300].replace("\n", " "),
|
||||
}
|
||||
|
||||
|
||||
def _run_batch(batch: list[str], jobs: int, timeout: int) -> list[dict[str, object]]:
|
||||
@@ -64,65 +131,194 @@ def _run_batch(batch: list[str], jobs: int, timeout: int) -> list[dict[str, obje
|
||||
return run_batch(Path(__file__), batch, jobs=jobs, timeout=timeout)
|
||||
|
||||
|
||||
def _files(corpus: Path, since: date | None) -> list[Path]:
|
||||
return corpus_files(corpus, since, suffixes=_EXTS)
|
||||
|
||||
|
||||
def _read_records(path: Path) -> list[dict[str, Any]]:
|
||||
rows: dict[str, dict[str, Any]] = {}
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
for line in stream:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
continue
|
||||
if isinstance(row, dict) and isinstance(row.get("path"), str):
|
||||
rows[row["path"]] = row
|
||||
return list(rows.values())
|
||||
|
||||
|
||||
def _repair_checkpoint(path: Path) -> None:
|
||||
"""Drop a torn tail before append, otherwise the next row joins invalid JSON."""
|
||||
repair_jsonl_tail(path)
|
||||
|
||||
|
||||
def _discovery_candidates(
|
||||
rows: list[dict[str, Any]], *, per_cohort: int, random_quiet: int, seed: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Stratify quiet rows without treating provenance as a pixel-level label."""
|
||||
rng = random.Random(seed) # noqa: S311 -- reproducible review sampling
|
||||
quiet = [row for row in rows if row.get("status") == "ok" and not row.get("keys")]
|
||||
strata: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
|
||||
for row in quiet:
|
||||
if row.get("uscc"):
|
||||
strata[f"tc260:{row['uscc']}"].append(row)
|
||||
if row.get("platform"):
|
||||
strata[f"platform:{row['platform']}"].append(row)
|
||||
|
||||
selected: dict[str, dict[str, Any]] = {}
|
||||
labels: dict[str, set[str]] = collections.defaultdict(set)
|
||||
for label, members in sorted(strata.items()):
|
||||
for row in rng.sample(members, k=min(per_cohort, len(members))):
|
||||
selected[row["path"]] = row
|
||||
labels[row["path"]].add(label)
|
||||
|
||||
for row in rng.sample(quiet, k=min(random_quiet, len(quiet))):
|
||||
selected[row["path"]] = row
|
||||
labels[row["path"]].add("unbiased-quiet")
|
||||
|
||||
return [{**row, "discovery_strata": sorted(labels[path])} for path, row in sorted(selected.items())]
|
||||
|
||||
|
||||
def _bands(path: str) -> tuple[Any, Any] | None:
|
||||
from remove_ai_watermarks.image_io import imread
|
||||
|
||||
image = imread(path)
|
||||
if image is None:
|
||||
return None
|
||||
height = image.shape[0]
|
||||
band_height = max(24, int(height * _SHEET_BAND_FRACTION))
|
||||
return image[:band_height], image[height - band_height :]
|
||||
|
||||
|
||||
def _write_discovery_sheets(rows: list[dict[str, Any]], output: Path) -> int:
|
||||
import csv
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
if output.exists() and any(output.iterdir()):
|
||||
raise ValueError(f"discovery output must be empty: {output}")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
manifest: list[dict[str, Any]] = []
|
||||
written = 0
|
||||
for row in rows:
|
||||
bands = _bands(row["path"])
|
||||
if bands is None:
|
||||
continue
|
||||
top, bottom = bands
|
||||
width = max(top.shape[1], bottom.shape[1])
|
||||
divider = np.full((6, width, 3), 90, np.uint8)
|
||||
header = np.full((28, width, 3), 40, np.uint8)
|
||||
cv2.putText(header, f"{written:05d}", (8, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 1)
|
||||
cv2.imwrite(str(output / f"{written:05d}.png"), np.vstack((header, top, divider, bottom)))
|
||||
manifest.append({"idx": written, **row})
|
||||
written += 1
|
||||
if manifest:
|
||||
fields = list(manifest[0])
|
||||
with (output / "MANIFEST_DO_NOT_OPEN.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(manifest)
|
||||
return written
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--corpus", type=Path, default=CORPUS)
|
||||
ap.add_argument("--since", type=date.fromisoformat, help="include dated corpus directories on or after YYYY-MM-DD")
|
||||
ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2))
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--batch", type=int, default=200)
|
||||
ap.add_argument("--timeout", type=int, default=600, help="seconds per image before terminating its isolated worker")
|
||||
ap.add_argument("--out", type=Path, default=OUT)
|
||||
ap.add_argument("--restart", action="store_true")
|
||||
ap.add_argument("--sheets", type=Path, help="write blinded discovery bands after the scan")
|
||||
ap.add_argument("--per-cohort", type=int, default=12, help="quiet images sampled from each metadata cohort")
|
||||
ap.add_argument("--random-quiet", type=int, default=400, help="unbiased quiet images sampled across the corpus")
|
||||
ap.add_argument("--seed", type=int, default=2026)
|
||||
a = ap.parse_args()
|
||||
|
||||
files = sorted(p for p in glob.glob(str(CORPUS / "*" / "*")) if Path(p).suffix.lower() in _EXTS)
|
||||
files = [str(path) for path in _files(a.corpus, a.since)]
|
||||
if a.limit:
|
||||
files = files[: a.limit]
|
||||
|
||||
done: set[str] = set()
|
||||
if a.restart and a.out.exists():
|
||||
a.out.unlink() # --restart must TRUNCATE; the file is reopened in append mode below
|
||||
_repair_checkpoint(a.out)
|
||||
if a.out.exists() and not a.restart:
|
||||
with open(a.out, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
done.add(json.loads(line)["path"])
|
||||
except Exception: # noqa: S112 -- tolerate a torn last line
|
||||
continue
|
||||
done = {row["path"] for row in _read_records(a.out) if row.get("schema_version") == _SCHEMA_VERSION}
|
||||
todo = [p for p in files if p not in done]
|
||||
print(f"images {len(files)} already done {len(done)} to do {len(todo)} jobs {a.jobs}", flush=True)
|
||||
|
||||
counts: collections.Counter[str] = collections.Counter()
|
||||
status: collections.Counter[str] = collections.Counter()
|
||||
seen = 0
|
||||
a.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(a.out, "a", encoding="utf-8") as fh:
|
||||
for start in range(0, len(todo), a.batch):
|
||||
for rec in _run_batch(todo[start : start + a.batch], a.jobs, a.timeout):
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
status[str(rec["status"])] += 1
|
||||
for k in rec["keys"]: # type: ignore[union-attr]
|
||||
counts[str(k)] += 1
|
||||
seen += 1
|
||||
fh.flush()
|
||||
print(f" {seen}/{len(todo)} {dict(counts)} {dict(status)}", flush=True)
|
||||
print(f" {seen}/{len(todo)}", flush=True)
|
||||
|
||||
# Rebuild the paths file from the FULL record set, not just this run's slice.
|
||||
hits = []
|
||||
with open(a.out, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except Exception: # noqa: S112
|
||||
continue
|
||||
if r.get("keys"):
|
||||
hits.append(r["path"])
|
||||
records = _read_records(a.out)
|
||||
selected_paths = set(files)
|
||||
records = [row for row in records if row["path"] in selected_paths]
|
||||
hits = [row["path"] for row in records if row.get("keys")]
|
||||
counts = collections.Counter(str(key) for row in records for key in row.get("keys", []))
|
||||
strict_counts = collections.Counter(str(key) for row in records for key in row.get("strict_keys", []))
|
||||
status = collections.Counter(str(row.get("status")) for row in records)
|
||||
# Derive the paths file from --out so a trial run with a scratch --out cannot
|
||||
# overwrite the shared list a full sweep produced.
|
||||
paths_out = a.out.with_suffix(".txt")
|
||||
paths_out.write_text("\n".join(sorted(set(hits))) + "\n", encoding="utf-8")
|
||||
paths_tmp = paths_out.with_suffix(paths_out.suffix + ".tmp")
|
||||
paths_tmp.write_text("\n".join(sorted(set(hits))) + ("\n" if hits else ""), encoding="utf-8")
|
||||
paths_tmp.replace(paths_out)
|
||||
print(f"\npositives: {len(set(hits))} images")
|
||||
for k, v in counts.most_common():
|
||||
print(f" {v:6d} {k}")
|
||||
print(f" {v:6d} {k} ({strict_counts[k]} strict)")
|
||||
print(f"statuses: {dict(status)}\npaths -> {paths_out}\nrecords -> {a.out}")
|
||||
if a.sheets:
|
||||
candidates = _discovery_candidates(
|
||||
records,
|
||||
per_cohort=a.per_cohort,
|
||||
random_quiet=a.random_quiet,
|
||||
seed=a.seed,
|
||||
)
|
||||
written = _write_discovery_sheets(candidates, a.sheets)
|
||||
print(f"discovery candidates: {len(candidates)} sheets written: {written} -> {a.sheets}")
|
||||
|
||||
from remove_ai_watermarks.watermark_registry import mark_keys
|
||||
|
||||
git = shutil.which("git")
|
||||
commit = (
|
||||
subprocess.run( # noqa: S603 -- resolved git binary and fixed read-only arguments
|
||||
[git, "rev-parse", "HEAD"],
|
||||
cwd=REPO,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
).stdout.strip()
|
||||
if git
|
||||
else ""
|
||||
)
|
||||
metadata = {
|
||||
"command": [str(Path(__file__).relative_to(REPO)), *sys.argv[1:]],
|
||||
"library_commit": commit or None,
|
||||
"library_version": version("remove-ai-watermarks"),
|
||||
"corpus": str(a.corpus.resolve()),
|
||||
"since": a.since.isoformat() if a.since else None,
|
||||
"selected_images": len(files),
|
||||
"complete_records": len(records),
|
||||
"registry_keys": mark_keys(),
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
}
|
||||
a.out.with_suffix(a.out.suffix + ".run.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -33,7 +33,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
|
||||
_warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*")
|
||||
|
||||
|
||||
__version__ = "0.40.1"
|
||||
__version__ = "0.40.3"
|
||||
|
||||
__all__ = [
|
||||
"BatchItemResult",
|
||||
|
||||
@@ -18,6 +18,7 @@ from remove_ai_watermarks._internal.constants import (
|
||||
C2PA_AI_TOOLS,
|
||||
C2PA_AI_VENDORS,
|
||||
C2PA_CHUNK_TYPE,
|
||||
C2PA_IDENTITY_AI_ORGS,
|
||||
C2PA_ISSUERS,
|
||||
C2PA_SIGNATURES,
|
||||
C2PA_SOFT_BINDING_REGISTRY,
|
||||
@@ -529,7 +530,8 @@ def _structured_manifest_fields(store: dict[str, Any]) -> dict[str, Any]:
|
||||
info["ai_tool"] = ", ".join(tools)
|
||||
if actions:
|
||||
info["actions"] = ", ".join(actions)
|
||||
if claim_generator_asserts_ai:
|
||||
signer_identity_asserts_ai = not C2PA_IDENTITY_AI_ORGS.isdisjoint(issuers)
|
||||
if claim_generator_asserts_ai or signer_identity_asserts_ai:
|
||||
info["c2pa_identity_ai"] = True
|
||||
|
||||
generated = any(
|
||||
|
||||
@@ -32,7 +32,7 @@ C2PA_SIGNATURES = tuple(
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class C2paAiVendor:
|
||||
"""One issuer signature and its normalized product attribution."""
|
||||
"""One C2PA identity and its AI-product or neutral-signer attribution."""
|
||||
|
||||
issuer: bytes
|
||||
org: str
|
||||
@@ -41,6 +41,8 @@ class C2paAiVendor:
|
||||
synthid: bool = False
|
||||
asserts_ai: bool = False
|
||||
synthid_requires_watermark_action: bool = False
|
||||
signer_platform: str | None = None
|
||||
raw_signer_platforms: tuple[tuple[bytes, str], ...] = ()
|
||||
|
||||
|
||||
def _vendor(
|
||||
@@ -52,6 +54,8 @@ def _vendor(
|
||||
synthid: bool = False,
|
||||
asserts_ai: bool = False,
|
||||
synthid_requires_watermark_action: bool = False,
|
||||
signer_platform: str | None = None,
|
||||
raw_signer_platforms: tuple[tuple[bytes, str], ...] = (),
|
||||
) -> C2paAiVendor:
|
||||
token = issuer.encode() if isinstance(issuer, str) else issuer
|
||||
return C2paAiVendor(
|
||||
@@ -62,6 +66,8 @@ def _vendor(
|
||||
synthid=synthid,
|
||||
asserts_ai=asserts_ai,
|
||||
synthid_requires_watermark_action=synthid_requires_watermark_action,
|
||||
signer_platform=signer_platform,
|
||||
raw_signer_platforms=raw_signer_platforms,
|
||||
)
|
||||
|
||||
|
||||
@@ -109,16 +115,75 @@ C2PA_AI_VENDORS: tuple[C2paAiVendor, ...] = (
|
||||
# "Bria Artificial Intelligence"). Found as an unmapped signer on 4 corpus
|
||||
# uploads 2026-08-08 that identify reported as unknown-signer C2PA.
|
||||
_vendor(b"Ideogram", "Ideogram", "Ideogram", "Ideogram", asserts_ai=True),
|
||||
_vendor(b"xAI Grok Imagine", "xAI Grok Imagine", "xAI Grok Imagine", "xAI", asserts_ai=True),
|
||||
_vendor(b"Producer.ai", "Producer.ai", "Producer.ai", "Producer.ai", asserts_ai=True),
|
||||
# This signature names only SPRING's legal entity, not a product. Preserve
|
||||
# that signer identity without guessing which product produced a file.
|
||||
_vendor(b"SPRING (SG) PTE. LTD.", "SPRING (SG) PTE. LTD.", "SPRING (SG) PTE. LTD.", "SPRING"),
|
||||
# These companies also sign ordinary edits and publications. Register the
|
||||
# signer identity for provenance display, but provide no AI platform and do
|
||||
# not let the identity assert an AI verdict.
|
||||
_vendor(
|
||||
b"TikTok Inc.",
|
||||
"TikTok",
|
||||
None,
|
||||
None,
|
||||
signer_platform="TikTok (C2PA signer)",
|
||||
),
|
||||
_vendor(
|
||||
b"Bytedance Pte",
|
||||
"ByteDance",
|
||||
None,
|
||||
None,
|
||||
signer_platform="ByteDance (C2PA signer)",
|
||||
raw_signer_platforms=(
|
||||
(b"Bytedance Pte. Ltd.", "ByteDance (C2PA signer)"),
|
||||
(b"CapCut/", "CapCut (C2PA signer)"),
|
||||
(b"capcut c2pa-rs", "CapCut (C2PA signer)"),
|
||||
),
|
||||
),
|
||||
_vendor(
|
||||
b"Anthropic Claude Content Signing",
|
||||
"Anthropic Claude",
|
||||
None,
|
||||
None,
|
||||
signer_platform="Anthropic Claude (C2PA signer)",
|
||||
),
|
||||
_vendor(
|
||||
b"Samsung Galaxy",
|
||||
"Samsung Galaxy",
|
||||
None,
|
||||
None,
|
||||
signer_platform="Samsung Galaxy (C2PA)",
|
||||
),
|
||||
_vendor(
|
||||
b"com.asus.gallery",
|
||||
"ASUS Gallery",
|
||||
None,
|
||||
None,
|
||||
signer_platform="ASUS Gallery (C2PA signer)",
|
||||
),
|
||||
_vendor(b"Truepic", "Truepic", None, None),
|
||||
)
|
||||
|
||||
C2PA_ISSUERS = {vendor.issuer: vendor.org for vendor in C2PA_AI_VENDORS}
|
||||
C2PA_IDENTITY_AI_ORGS = frozenset(vendor.org for vendor in C2PA_AI_VENDORS if vendor.asserts_ai)
|
||||
C2PA_SIGNER_PLATFORM_BY_ORG = {
|
||||
vendor.org: vendor.signer_platform for vendor in C2PA_AI_VENDORS if vendor.signer_platform is not None
|
||||
}
|
||||
C2PA_SIGNER_PLATFORMS = tuple(
|
||||
pair
|
||||
for vendor in C2PA_AI_VENDORS
|
||||
if vendor.signer_platform is not None
|
||||
for pair in (vendor.raw_signer_platforms or ((vendor.issuer, vendor.signer_platform),))
|
||||
)
|
||||
|
||||
# Product-specific claim generators can sign through a different upstream issuer.
|
||||
# Keep this attribution beside the issuer registry so every C2PA consumer has one
|
||||
# canonical source rather than maintaining a derived product map in identify.py.
|
||||
C2PA_CLAIM_GENERATOR_PLATFORMS: tuple[tuple[str, str], ...] = (
|
||||
("grok imagine", "xAI Grok Imagine"),
|
||||
("suno", "Suno"),
|
||||
("adobe_firefly", "Adobe Firefly"),
|
||||
("firefly", "Adobe Firefly"),
|
||||
("dreamina", "ByteDance Dreamina"),
|
||||
@@ -126,6 +191,8 @@ C2PA_CLAIM_GENERATOR_PLATFORMS: tuple[tuple[str, str], ...] = (
|
||||
("recraft.ai", "Recraft"),
|
||||
("topaz labs image api", "Topaz Labs"),
|
||||
("tiktok ad creative toolbox", "TikTok Ad Creative Toolbox"),
|
||||
("fastvid", "FastVid"),
|
||||
("capcut", "CapCut (C2PA signer)"),
|
||||
)
|
||||
|
||||
C2PA_AI_TOOLS = {
|
||||
@@ -211,6 +278,7 @@ AI_GENERATOR_TOKENS = frozenset(
|
||||
"invokeai",
|
||||
"imagen",
|
||||
"gpt-image",
|
||||
"nano banana",
|
||||
"nightcafe",
|
||||
"ideogram",
|
||||
"leonardo",
|
||||
|
||||
@@ -43,6 +43,7 @@ from remove_ai_watermarks.metadata import (
|
||||
C2PA_UUID,
|
||||
IPTC_AI_FIELD_MARKERS,
|
||||
IPTC_AI_MARKERS,
|
||||
LOCAL_GENERATION_METADATA_KEYS,
|
||||
MAX_TC260_VALUE_BYTES,
|
||||
parse_tc260_aigc_json,
|
||||
)
|
||||
@@ -152,12 +153,12 @@ def iter_file_boxes(
|
||||
pos = box_end
|
||||
|
||||
|
||||
def _tc260_key_indices(
|
||||
def _metadata_key_indices(
|
||||
stream: BinaryIO,
|
||||
payload_off: int,
|
||||
box_end: int,
|
||||
) -> dict[int, tuple[int, int]]:
|
||||
"""Map every exact ``AIGC`` key index to its byte span."""
|
||||
) -> dict[int, tuple[bytes, tuple[int, int]]]:
|
||||
"""Map bounded metadata key indices to their names and byte spans."""
|
||||
if payload_off + 8 > box_end:
|
||||
return {}
|
||||
stream.seek(payload_off)
|
||||
@@ -166,7 +167,7 @@ def _tc260_key_indices(
|
||||
return {}
|
||||
entry_count = struct.unpack(">I", prefix[4:8])[0]
|
||||
pos = payload_off + 8
|
||||
found: dict[int, tuple[int, int]] = {}
|
||||
found: dict[int, tuple[bytes, tuple[int, int]]] = {}
|
||||
for index in range(1, entry_count + 1):
|
||||
if pos + 8 > box_end:
|
||||
return {}
|
||||
@@ -179,10 +180,12 @@ def _tc260_key_indices(
|
||||
if entry_size < 8 or entry_end > box_end:
|
||||
return {}
|
||||
name_start = pos + 8
|
||||
if entry_end - name_start == 4:
|
||||
name_size = entry_end - name_start
|
||||
if 0 < name_size <= 128:
|
||||
stream.seek(name_start)
|
||||
if stream.read(4) == b"AIGC":
|
||||
found[index] = (name_start, entry_end)
|
||||
name = stream.read(name_size)
|
||||
if len(name) == name_size:
|
||||
found[index] = (name, (name_start, entry_end))
|
||||
pos = entry_end
|
||||
return found
|
||||
|
||||
@@ -248,12 +251,23 @@ def _tc260_aigc_regions(
|
||||
iOS export writes, where the JSON sits in a bare ``ilst`` data item with no
|
||||
key name to blank.
|
||||
"""
|
||||
regions: list[tuple[tuple[int, int] | None, int, int, bytes]] = []
|
||||
return [
|
||||
(key_span, value_start, value_end, value)
|
||||
for key_name, key_span, value_start, value_end, value in _metadata_items(stream, file_size)
|
||||
if (key_name is None or key_name == b"AIGC") and parse_tc260_aigc_json(value) is not None
|
||||
]
|
||||
|
||||
|
||||
def _metadata_items(
|
||||
stream: BinaryIO,
|
||||
file_size: int,
|
||||
) -> Iterator[tuple[bytes | None, tuple[int, int] | None, int, int, bytes]]:
|
||||
"""Yield bounded keyed or keyless metadata-list values from a media file."""
|
||||
for _moov_start, moov_end, moov_type, moov_payload in iter_file_boxes(stream, 0, file_size):
|
||||
if moov_type != b"moov":
|
||||
continue
|
||||
for meta_payload, meta_end in _iter_tc260_meta_boxes(stream, moov_payload, moov_end):
|
||||
keys: dict[int, tuple[int, int]] = {}
|
||||
keys: dict[int, tuple[bytes, tuple[int, int]]] = {}
|
||||
ilst_boxes: list[tuple[int, int]] = []
|
||||
keyed = False
|
||||
for _child_start, child_end, child_type, child_payload in _meta_child_boxes(
|
||||
@@ -263,7 +277,7 @@ def _tc260_aigc_regions(
|
||||
):
|
||||
if child_type == b"keys":
|
||||
keyed = True
|
||||
keys.update(_tc260_key_indices(stream, child_payload, child_end))
|
||||
keys.update(_metadata_key_indices(stream, child_payload, child_end))
|
||||
elif child_type == b"ilst":
|
||||
ilst_boxes.append((child_payload, child_end))
|
||||
if not ilst_boxes:
|
||||
@@ -275,8 +289,8 @@ def _tc260_aigc_regions(
|
||||
ilst_end,
|
||||
):
|
||||
index = int.from_bytes(item_type, "big")
|
||||
key_span = keys.get(index)
|
||||
if keyed and key_span is None:
|
||||
key = keys.get(index)
|
||||
if keyed and key is None:
|
||||
# A keyed (ISO) meta box maps items through ``keys``;
|
||||
# an unmapped index is not an AIGC entry, and reading
|
||||
# its value would pull arbitrary metadata (e.g. cover
|
||||
@@ -284,6 +298,7 @@ def _tc260_aigc_regions(
|
||||
# keyless QuickTime list falls through to content
|
||||
# validation below.
|
||||
continue
|
||||
key_name, key_span = key if key is not None else (None, None)
|
||||
for _data_start, data_end, data_type, data_payload in iter_file_boxes(
|
||||
stream,
|
||||
item_payload,
|
||||
@@ -295,9 +310,8 @@ def _tc260_aigc_regions(
|
||||
continue
|
||||
stream.seek(value_start)
|
||||
value = stream.read(value_size)
|
||||
if len(value) == value_size and parse_tc260_aigc_json(value) is not None:
|
||||
regions.append((key_span, value_start, data_end, value))
|
||||
return regions
|
||||
if len(value) == value_size:
|
||||
yield key_name, key_span, value_start, data_end, value
|
||||
|
||||
|
||||
def tc260_aigc_payloads(path: str | Path, *, strict: bool = False) -> tuple[bytes, ...]:
|
||||
@@ -315,6 +329,42 @@ def tc260_aigc_payloads(path: str | Path, *, strict: bool = False) -> tuple[byte
|
||||
return ()
|
||||
|
||||
|
||||
def _ai_metadata_regions(
|
||||
stream: BinaryIO,
|
||||
file_size: int,
|
||||
) -> list[tuple[tuple[int, int], int, int, str, bytes]]:
|
||||
"""Locate keyed generation metadata without reading media payloads."""
|
||||
regions: list[tuple[tuple[int, int], int, int, str, bytes]] = []
|
||||
for name_bytes, key_span, value_start, value_end, value in _metadata_items(stream, file_size):
|
||||
if name_bytes is None or key_span is None:
|
||||
continue
|
||||
try:
|
||||
name = name_bytes.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if name.lower() in LOCAL_GENERATION_METADATA_KEYS:
|
||||
regions.append((key_span, value_start, value_end, name, value))
|
||||
return regions
|
||||
|
||||
|
||||
def ai_metadata_tags(path: str | Path, *, strict: bool = False) -> dict[str, str]:
|
||||
"""Read keyed AI-generation metadata from an ISOBMFF container."""
|
||||
try:
|
||||
with open(path, "rb") as stream:
|
||||
if not is_isobmff(stream.read(8)):
|
||||
return {}
|
||||
stream.seek(0, 2)
|
||||
file_size = stream.tell()
|
||||
return {
|
||||
name: value.decode("utf-8", "replace")
|
||||
for _key_span, _value_start, _value_end, name, value in _ai_metadata_regions(stream, file_size)
|
||||
}
|
||||
except OSError:
|
||||
if strict:
|
||||
raise
|
||||
return {}
|
||||
|
||||
|
||||
def blank_tc260_aigc_tags(data: bytes) -> tuple[bytes, int]:
|
||||
"""Blank native TC260 values in place while preserving every box offset.
|
||||
|
||||
@@ -576,7 +626,9 @@ def strip_isobmff_media_file(
|
||||
max_scan=max_box_scan,
|
||||
)
|
||||
tc260_regions = _tc260_aigc_regions(stream, file_size) if targets is not None else []
|
||||
ai_metadata_regions = _ai_metadata_regions(stream, file_size) if targets is not None else []
|
||||
tc260_key_spans = {region[0] for region in tc260_regions if region[0] is not None}
|
||||
ai_metadata_key_spans = {region[0] for region in ai_metadata_regions}
|
||||
|
||||
with atomic_video_output(output_path) as temporary_path:
|
||||
with source_path.open("rb") as source_stream, temporary_path.open("r+b") as temporary:
|
||||
@@ -591,6 +643,10 @@ def strip_isobmff_media_file(
|
||||
temporary.seek(key_span[0])
|
||||
temporary.write(b"free")
|
||||
_overwrite_range(temporary, value_start, value_end, byte=b" ")
|
||||
for key_span, value_start, value_end, _name, _value in ai_metadata_regions:
|
||||
temporary.seek(key_span[0])
|
||||
temporary.write(b"free" + b" " * (key_span[1] - key_span[0] - 4))
|
||||
_overwrite_range(temporary, value_start, value_end, byte=b" ")
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
shutil.copymode(source_path, temporary_path)
|
||||
@@ -601,7 +657,7 @@ def strip_isobmff_media_file(
|
||||
source_path,
|
||||
)
|
||||
return 0, 0
|
||||
return len(targets), len(tc260_key_spans)
|
||||
return len(targets), len(tc260_key_spans | ai_metadata_key_spans)
|
||||
|
||||
|
||||
def strip_c2pa_boxes(data: bytes) -> tuple[bytes, int]:
|
||||
|
||||
@@ -44,6 +44,8 @@ from remove_ai_watermarks._internal.constants import (
|
||||
C2PA_CLAIM_GENERATOR_PLATFORMS,
|
||||
C2PA_IDENTITY_AI_ORGS,
|
||||
C2PA_ISSUERS,
|
||||
C2PA_SIGNER_PLATFORM_BY_ORG,
|
||||
C2PA_SIGNER_PLATFORMS,
|
||||
)
|
||||
from remove_ai_watermarks._internal.schema import require_schema_version
|
||||
from remove_ai_watermarks.metadata import (
|
||||
@@ -51,6 +53,7 @@ from remove_ai_watermarks.metadata import (
|
||||
AIGC_MARKERS,
|
||||
IPTC_AI_FIELD_MARKERS,
|
||||
IPTC_AI_MARKERS,
|
||||
LOCAL_GENERATION_METADATA_KEYS,
|
||||
aigc_label,
|
||||
aigc_label_from_metadata,
|
||||
c2pa_cloud_manifest_in,
|
||||
@@ -65,6 +68,7 @@ from remove_ai_watermarks.metadata import (
|
||||
samsung_genai_in,
|
||||
scan_head,
|
||||
xai_signature,
|
||||
xai_signature_in_metadata,
|
||||
xai_signature_pair,
|
||||
)
|
||||
from remove_ai_watermarks.watermark_registry import GEMINI_SPARKLE_TRUST_CONF
|
||||
@@ -106,9 +110,7 @@ _ISSUER_PLATFORM: tuple[tuple[str, str], ...] = tuple(
|
||||
|
||||
# PNG-text / EXIF keys that indicate a local diffusion pipeline (vs. a hosted
|
||||
# platform's C2PA). Subset of AI_METADATA_KEYS; excludes the C2PA/Software keys.
|
||||
_LOCAL_GEN_KEYS = frozenset(
|
||||
AI_METADATA_KEYS & {"parameters", "prompt", "negative_prompt", "workflow", "comfyui", "invokeai_metadata", "dream"}
|
||||
)
|
||||
_LOCAL_GEN_KEYS = LOCAL_GENERATION_METADATA_KEYS
|
||||
|
||||
_STRIP_CAVEAT = (
|
||||
"Absence of metadata is not proof the image is clean: C2PA, EXIF, and PNG "
|
||||
@@ -420,10 +422,16 @@ def evidence_from_metadata_record(
|
||||
values_by_key: dict[str, str] = {}
|
||||
for key, value in pairs:
|
||||
if isinstance(value, (bytes, str)):
|
||||
values_by_key.setdefault(key.lower(), _external_text(value))
|
||||
description = values_by_key.get("imagedescription", "")
|
||||
artist = values_by_key.get("artist", "")
|
||||
xai = xai_signature_pair(description, artist)
|
||||
values_by_key.setdefault(key.lower().removeprefix("info:"), _external_text(value))
|
||||
xai = any(
|
||||
xai_signature_pair(values_by_key.get(description_key, ""), values_by_key.get(creator_key, ""))
|
||||
for description_key, creator_key in (
|
||||
("imagedescription", "artist"),
|
||||
("description", "author"),
|
||||
("description", "creator"),
|
||||
("caption-abstract", "by-line"),
|
||||
)
|
||||
) or xai_signature_in_metadata(scan)
|
||||
|
||||
hf_job = next(
|
||||
(
|
||||
@@ -447,7 +455,7 @@ def evidence_from_metadata_record(
|
||||
f"China AIGC label (TC260){f'; producer {producer}' if producer else ''}",
|
||||
)
|
||||
if xai:
|
||||
ai_metadata.setdefault("xai_signature", "xAI/Grok EXIF signature (Artist UUID + Signature blob)")
|
||||
ai_metadata.setdefault("xai_signature", "xAI/Grok signature (UUID + Signature blob)")
|
||||
if iptc_system:
|
||||
ai_metadata.setdefault("ai_system", f"IPTC 2025.1 AI disclosure ({iptc_system})")
|
||||
if hf_job:
|
||||
@@ -625,7 +633,8 @@ def _ai_tools_in(data: bytes) -> list[str]:
|
||||
# Only tokens verified against a real signed file are listed (Leica, Nikon,
|
||||
# Sony, Truepic, Google Pixel); add Canon/Bria as real samples are captured.
|
||||
# Samsung Galaxy is an AI-capable editing device, not a pure-capture camera, so
|
||||
# it lives in `_SIGNER_C2PA_PLATFORM` below (it must not feed the camera clash).
|
||||
# it lives in the neutral `C2PA_SIGNER_PLATFORMS` registry (it must not feed the
|
||||
# camera clash).
|
||||
_DEVICE_C2PA_PLATFORM: tuple[tuple[bytes, str], ...] = (
|
||||
(b"lc_c2pa", "Leica (camera, C2PA capture)"),
|
||||
(b"Leica Camera", "Leica (camera, C2PA capture)"),
|
||||
@@ -640,6 +649,8 @@ _DEVICE_C2PA_PLATFORM: tuple[tuple[bytes, str], ...] = (
|
||||
# Truepic is a C2PA signing authority whose name appears in the trust chain
|
||||
# of unrelated manifests (e.g. OpenAI), so the bare token mis-attributes.
|
||||
(b"Truepic_Lens", "Truepic Lens (verified capture)"),
|
||||
(b"vivo Camera", "vivo (camera, C2PA capture)"),
|
||||
(b"vivo X300 Pro", "vivo (camera, C2PA capture)"),
|
||||
)
|
||||
|
||||
|
||||
@@ -737,15 +748,14 @@ def _device_platform(head: bytes) -> str | None:
|
||||
# (distinct from the EXIF "SM-xxxx" model string on ordinary Samsung photos).
|
||||
# com.asus.gallery -- ASUS Gallery claim_generator (a C2PA-signed edit, no AI
|
||||
# source type or genAIType on the samples, so it never asserts is_ai).
|
||||
_SIGNER_C2PA_PLATFORM: tuple[tuple[bytes, str], ...] = (
|
||||
(b"Samsung Galaxy", "Samsung Galaxy (C2PA)"),
|
||||
(b"com.asus.gallery", "ASUS Gallery (C2PA signer)"),
|
||||
)
|
||||
|
||||
|
||||
def _signer_platform(head: bytes) -> str | None:
|
||||
"""Map a C2PA editing-app / AI-capable-device signer token to a platform."""
|
||||
return _first_token_match(head, _SIGNER_C2PA_PLATFORM)
|
||||
def _signer_platform(head: bytes, issuers: list[str]) -> str | None:
|
||||
"""Map a raw or normalized C2PA signer identity to a neutral platform."""
|
||||
raw = _first_token_match(head, C2PA_SIGNER_PLATFORMS)
|
||||
normalized = next(
|
||||
(C2PA_SIGNER_PLATFORM_BY_ORG[org] for org in issuers if org in C2PA_SIGNER_PLATFORM_BY_ORG),
|
||||
None,
|
||||
)
|
||||
return raw or normalized
|
||||
|
||||
|
||||
def _attribute_platform(issuers: list[str], *, is_ai: bool = True) -> str | None:
|
||||
@@ -1172,7 +1182,6 @@ def _identify_from_evidence(
|
||||
# below keeps the full buffer: their markers are long and distinctive.
|
||||
region = _metadata_region(head)
|
||||
camera_label = _device_platform(region)
|
||||
signer_label = _signer_platform(region)
|
||||
|
||||
# ── C2PA Content Credentials ────────────────────────────────────
|
||||
has_c2pa = bool(info) or c2pa_marker_in(head)
|
||||
@@ -1183,6 +1192,7 @@ def _identify_from_evidence(
|
||||
# by substring made the displayed reason a second, looser rule than the verdict.
|
||||
failed_c2pa_codes = [str(code) for code in cast("list[object]", info.get("c2pa_failed_codes", []))]
|
||||
issuers = [info["issuer"]] if info.get("issuer") else _issuers_in(region)
|
||||
signer_label = _signer_platform(region, issuers)
|
||||
# Full AI generation (trainedAlgorithmicMedia) vs an AI-enhanced real photo
|
||||
# (compositeWithTrainedAlgorithmicMedia). The structured kind is parsed once in
|
||||
# _internal.c2pa._structured_manifest_fields (covers PNG + any container the c2pa-python
|
||||
@@ -1210,15 +1220,18 @@ def _identify_from_evidence(
|
||||
or (", ".join(tools) if (tools := _ai_tools_in(region)) else None)
|
||||
)
|
||||
# Platform: a distinctive device/camera token in the manifest wins (it is the
|
||||
# signer/producer), then an editing-app/AI-device signer (Samsung Galaxy,
|
||||
# ASUS Gallery), with the issuer byte-scan only as fallback. The issuer scan
|
||||
# alone mis-attributed real samples (Leica->Truepic timestamp authority,
|
||||
# Nikon->Adobe namespace, Pixel->Google Gemini) -- the token scans fix that.
|
||||
# signer/producer), then an exact product generator, then an editing-app or
|
||||
# AI-device signer (Samsung Galaxy, ASUS Gallery), with the issuer byte-scan
|
||||
# only as fallback. The issuer scan alone mis-attributed real samples
|
||||
# (Leica->Truepic timestamp authority, Nikon->Adobe namespace, Pixel->Google
|
||||
# Gemini) -- the token scans fix that.
|
||||
platform = (
|
||||
(
|
||||
camera_label
|
||||
# Exact product generators are useful provenance even when the
|
||||
# signed operation is a non-AI edit (for example, CapCut).
|
||||
or _claim_generator_platform(generator)
|
||||
or signer_label
|
||||
or (_claim_generator_platform(generator) if c2pa_is_ai else None)
|
||||
or (_claim_generator_platform(str(info.get("ai_tool"))) if c2pa_is_ai and info.get("ai_tool") else None)
|
||||
or _attribute_platform(issuers, is_ai=c2pa_is_ai)
|
||||
)
|
||||
@@ -1425,13 +1438,12 @@ def _identify_from_evidence(
|
||||
if v := _vendor_of(generator_tag):
|
||||
ai_vendor_claims["exif_generator"] = v
|
||||
|
||||
# ── xAI / Grok EXIF signature scheme (no C2PA/SynthID/IPTC) ──────
|
||||
# Grok's only provenance signal: EXIF ImageDescription "Signature: <base64>"
|
||||
# + a UUID Artist. Distinct from exif_generator (which matches generator
|
||||
# tokens); verified stable across 3 generations. See CLAUDE.md.
|
||||
# ── xAI / Grok signature scheme ───────────────────────────────────
|
||||
# The original EXIF pair can move into equivalent XMP, PNG text, or IPTC
|
||||
# fields during a metadata-preserving transcode. Neither shape alone is enough.
|
||||
if evidence.xai_signature:
|
||||
signals.append(Signal("xai_signature", "EXIF Signature blob + UUID Artist", "high"))
|
||||
watermarks.append("xAI/Grok EXIF signature")
|
||||
signals.append(Signal("xai_signature", "Signature blob + UUID provenance pair", "high"))
|
||||
watermarks.append("xAI/Grok provenance signature")
|
||||
if platform is None:
|
||||
platform = "xAI (Grok / Aurora)"
|
||||
ai_vendor_claims["xai"] = "xAI"
|
||||
|
||||
@@ -59,6 +59,9 @@ AI_METADATA_KEYS: frozenset[str] = frozenset(
|
||||
"Software",
|
||||
]
|
||||
)
|
||||
LOCAL_GENERATION_METADATA_KEYS: frozenset[str] = frozenset(
|
||||
AI_METADATA_KEYS & {"parameters", "prompt", "negative_prompt", "workflow", "comfyui", "invokeai_metadata", "dream"}
|
||||
)
|
||||
|
||||
AI_KEYWORDS: tuple[str, ...] = (
|
||||
"stable_diffusion",
|
||||
@@ -169,6 +172,7 @@ TC260_AIGC_FIELDS: frozenset[str] = frozenset(
|
||||
"ServiceUser",
|
||||
}
|
||||
)
|
||||
_TC260_CANONICAL_FIELDS: dict[str, str] = {field.casefold(): field for field in TC260_AIGC_FIELDS}
|
||||
MAX_TC260_VALUE_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
@@ -197,7 +201,10 @@ def parse_tc260_aigc_json(value: bytes) -> dict[str, str] | None:
|
||||
return None
|
||||
if not isinstance(parsed, dict):
|
||||
return None
|
||||
fields = {str(key): str(item) for key, item in cast("dict[object, object]", parsed).items()}
|
||||
fields = {
|
||||
_TC260_CANONICAL_FIELDS.get(str(key).casefold(), str(key)): str(item)
|
||||
for key, item in cast("dict[object, object]", parsed).items()
|
||||
}
|
||||
return fields if TC260_AIGC_FIELDS & fields.keys() else None
|
||||
|
||||
|
||||
@@ -520,7 +527,7 @@ def has_ai_metadata(image_path: Path) -> bool:
|
||||
# Hugging Face-hosted job marker (hf-job-id PNG text chunk).
|
||||
if huggingface_job(image_path):
|
||||
return True
|
||||
# xAI / Grok: no C2PA/IPTC/XMP -- only the EXIF Signature + UUID-Artist pair.
|
||||
# xAI / Grok signature + UUID pair, including container-native transcodes.
|
||||
return xai_signature(image_path)
|
||||
|
||||
|
||||
@@ -877,11 +884,47 @@ def generator_from_metadata(candidates: Iterable[str], scan: bytes = b"") -> str
|
||||
for value in itertools.chain(candidates, creator_tools):
|
||||
if app_generator := app_generator_from_metadata(value):
|
||||
return app_generator
|
||||
if structured_generator := _structured_generator_from_metadata(value):
|
||||
return structured_generator
|
||||
if any(token in value.lower() for token in AI_GENERATOR_TOKENS):
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _structured_generator_from_metadata(value: str) -> str | None:
|
||||
"""Extract a bounded generator name without returning its surrounding JSON."""
|
||||
from remove_ai_watermarks._internal.constants import AI_GENERATOR_TOKENS
|
||||
|
||||
if len(value) > MAX_TC260_VALUE_BYTES:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(parsed, dict):
|
||||
return None
|
||||
generation_params = cast("dict[object, object]", parsed).get("generationParams")
|
||||
if not isinstance(generation_params, dict):
|
||||
return None
|
||||
model_name = cast("dict[object, object]", generation_params).get("modelName")
|
||||
if not isinstance(model_name, str) or not model_name.strip() or len(model_name) > 200:
|
||||
return None
|
||||
normalized = model_name.strip()
|
||||
return normalized if any(token in normalized.lower() for token in AI_GENERATOR_TOKENS) else None
|
||||
|
||||
|
||||
def exif_bytes_from_image(image: Any) -> bytes | None:
|
||||
"""Return EXIF bytes from either a normal EXIF field or a decoded raw profile."""
|
||||
value = image.info.get("exif")
|
||||
if isinstance(value, bytes) and value:
|
||||
return value
|
||||
with contextlib.suppress(Exception):
|
||||
exif = image.getexif()
|
||||
if exif:
|
||||
return exif.tobytes()
|
||||
return None
|
||||
|
||||
|
||||
_APP_PROVENANCE_PRODUCTS: dict[str, str] = {
|
||||
"doubao": "ByteDance Doubao",
|
||||
"xinghui": "ByteDance Xinghui",
|
||||
@@ -894,12 +937,17 @@ _APP_AIGC_LABEL_RE = re.compile(r'"aigc_label_type"\s*:\s*[12](?=\s*[,}])', re.I
|
||||
_APP_AIGC_TYPE_RE = re.compile(r'"aigc_type"\s*:\s*1(?=\s*[,}])', re.IGNORECASE)
|
||||
|
||||
|
||||
def _normalized_app_metadata(value: str | bytes) -> str:
|
||||
text = value.decode("latin-1", "ignore") if isinstance(value, bytes) else value
|
||||
def _normalized_app_metadata(value: object) -> str:
|
||||
if isinstance(value, bytes):
|
||||
text = value.decode("latin-1", "ignore")
|
||||
elif isinstance(value, str):
|
||||
text = value
|
||||
else:
|
||||
return ""
|
||||
return text.replace('\\"', '"')
|
||||
|
||||
|
||||
def _app_metadata_evidence(value: str | bytes) -> tuple[str | None, str | None]:
|
||||
def _app_metadata_evidence(value: object) -> tuple[str | None, str | None]:
|
||||
"""Return removable product provenance and stronger AI-origin evidence."""
|
||||
normalized = _normalized_app_metadata(value)
|
||||
products = tuple(match.group(1).lower() for match in _APP_PRODUCT_RE.finditer(normalized))
|
||||
@@ -962,7 +1010,7 @@ def exif_generator(image_path: Path) -> str | None:
|
||||
|
||||
with Image.open(image_path) as img:
|
||||
info = img.info
|
||||
exif_bytes = info.get("exif")
|
||||
exif_bytes = exif_bytes_from_image(img)
|
||||
# PNG tEXt/iTXt chunks land in img.info too (same idiom as the other
|
||||
# PNG-text readers in this module); NovelAI stamps Software/Source/Title.
|
||||
for key in ("Software", "Source", "Title", "Description"):
|
||||
@@ -1004,13 +1052,122 @@ def exif_generator(image_path: Path) -> str | None:
|
||||
# pair xAI-specific -- both required keeps the false-positive rate near zero.
|
||||
_XAI_SIGNATURE_RE = re.compile(r"Signature:\s*[A-Za-z0-9+/=]{64,}")
|
||||
_UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE)
|
||||
_RDF_DESCRIPTION_RE = re.compile(rb"<rdf:Description\b[^>]*>.*?</rdf:Description>", re.IGNORECASE | re.DOTALL)
|
||||
_XMP_DESCRIPTION_RE = re.compile(
|
||||
rb"(?:<dc:description\b[^>]*>(.*?)</dc:description>|\bdc:description=[\"']([^\"']*)[\"'])",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_XMP_CREATOR_RE = re.compile(
|
||||
rb"(?:<dc:creator\b[^>]*>(.*?)</dc:creator>|\bdc:creator=[\"']([^\"']*)[\"'])",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def xai_signature_pair(description: str, artist: str) -> bool:
|
||||
"""True if an EXIF (ImageDescription, Artist) pair is xAI/Grok's scheme."""
|
||||
"""True if a description/creator pair has xAI/Grok's two value shapes."""
|
||||
return _XAI_SIGNATURE_RE.match(description) is not None and _UUID_RE.fullmatch(artist) is not None
|
||||
|
||||
|
||||
def _decoded_xai_signature_pair(description: bytes, creator: bytes) -> bool:
|
||||
description = re.sub(rb"<[^>]+>", b"", description).strip()
|
||||
creator = re.sub(rb"<[^>]+>", b"", creator).strip()
|
||||
return xai_signature_pair(
|
||||
description.decode("utf-8", "replace").strip(),
|
||||
creator.decode("utf-8", "replace").strip(),
|
||||
)
|
||||
|
||||
|
||||
def _xmp_xai_signature_pair(data: bytes) -> bool:
|
||||
"""Match the two xAI values only inside the same RDF description."""
|
||||
for block_match in _RDF_DESCRIPTION_RE.finditer(data):
|
||||
block = block_match.group()
|
||||
descriptions = [
|
||||
next(value for value in match.groups() if value is not None)
|
||||
for match in _XMP_DESCRIPTION_RE.finditer(block)
|
||||
]
|
||||
creators = [
|
||||
next(value for value in match.groups() if value is not None) for match in _XMP_CREATOR_RE.finditer(block)
|
||||
]
|
||||
if any(
|
||||
_decoded_xai_signature_pair(description, creator) for description in descriptions for creator in creators
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _iptc_dataset_values(data: bytes, dataset: int) -> list[bytes]:
|
||||
"""Return bounded IPTC application-record values for one dataset number."""
|
||||
values: list[bytes] = []
|
||||
marker = bytes((0x1C, 0x02, dataset))
|
||||
start = 0
|
||||
while (index := data.find(marker, start)) >= 0:
|
||||
length_start = index + len(marker)
|
||||
if length_start + 2 > len(data):
|
||||
break
|
||||
length = int.from_bytes(data[length_start : length_start + 2], "big")
|
||||
value_start = length_start + 2
|
||||
value_end = value_start + length
|
||||
if value_end <= len(data) and length <= 4096:
|
||||
values.append(data[value_start:value_end])
|
||||
start = length_start
|
||||
return values
|
||||
|
||||
|
||||
def _iptc_xai_signature_pair(data: bytes) -> bool:
|
||||
descriptions = _iptc_dataset_values(data, 120) # Caption-Abstract
|
||||
creators = _iptc_dataset_values(data, 80) # By-line
|
||||
# Some metadata exports retain the field names instead of the binary dataset
|
||||
# headers. Keep that observed representation precise by anchoring each value to
|
||||
# its field label.
|
||||
descriptions += re.findall(rb"Caption-Abstract\x00([^\x00]{1,4096})", data, re.IGNORECASE)
|
||||
creators += re.findall(rb"By-line\x00([^\x00]{1,4096})", data, re.IGNORECASE)
|
||||
return any(
|
||||
_decoded_xai_signature_pair(description, creator) for description in descriptions for creator in creators
|
||||
)
|
||||
|
||||
|
||||
def _jpeg_app_payloads(data: bytes) -> list[tuple[int, bytes]]:
|
||||
"""Extract complete XMP/IPTC APP payloads from a JPEG or concatenated regions."""
|
||||
payloads: list[tuple[int, bytes]] = []
|
||||
position = 2 if data.startswith(b"\xff\xd8") else 0
|
||||
while position + 4 <= len(data):
|
||||
if data[position] != 0xFF:
|
||||
break
|
||||
marker = data[position + 1]
|
||||
if marker in (0xDA, 0xD9): # SOS / EOI
|
||||
break
|
||||
if 0xD0 <= marker <= 0xD7 or marker == 0x01:
|
||||
position += 2
|
||||
continue
|
||||
length = int.from_bytes(data[position + 2 : position + 4], "big")
|
||||
end = position + 2 + length
|
||||
if length >= 2 and end <= len(data):
|
||||
if marker in (0xE1, 0xED):
|
||||
payloads.append((marker, data[position + 4 : end]))
|
||||
position = end
|
||||
else:
|
||||
break
|
||||
return payloads
|
||||
|
||||
|
||||
def xai_signature_in_metadata(data: bytes) -> bool:
|
||||
"""True when one metadata record carries xAI's paired provenance fields.
|
||||
|
||||
Editors preserve the same two values under equivalent container-native fields:
|
||||
XMP ``dc:description``/``dc:creator`` and IPTC
|
||||
``Caption-Abstract``/``By-line``. Values must be attached to the confirmed field
|
||||
names inside one APP segment or RDF description; an unrelated XMP UUID must not
|
||||
complete the pair.
|
||||
"""
|
||||
payloads = _jpeg_app_payloads(data)
|
||||
if payloads:
|
||||
return any(
|
||||
_xmp_xai_signature_pair(payload) if marker == 0xE1 else _iptc_xai_signature_pair(payload)
|
||||
for marker, payload in payloads
|
||||
)
|
||||
return _xmp_xai_signature_pair(data) or _iptc_xai_signature_pair(data)
|
||||
|
||||
|
||||
def _exif_text(ifd: dict[int, Any], tag: int) -> str:
|
||||
"""Decode a piexif 0th-IFD byte tag to a stripped string ('' if absent)."""
|
||||
value = ifd.get(tag)
|
||||
@@ -1018,34 +1175,52 @@ def _exif_text(ifd: dict[int, Any], tag: int) -> str:
|
||||
|
||||
|
||||
def _xai_signature_impl(image_path: Path) -> bool:
|
||||
"""Detect xAI / Grok's EXIF provenance signature scheme.
|
||||
"""Detect xAI / Grok's provenance signature scheme.
|
||||
|
||||
Grok image downloads (Aurora model) carry no C2PA, XMP, SynthID, or IPTC --
|
||||
their only provenance signal is a private EXIF pair: ``ImageDescription`` =
|
||||
``"Signature: <base64>"`` together with ``Artist`` = the image UUID. Verified
|
||||
stable across three independent generations (2026-05-26; see CLAUDE.md). The
|
||||
signature is xAI's and is not locally verifiable (no public key); detection
|
||||
keys on this distinctive, low-false-positive shape, not on the signature's
|
||||
validity. It survives only on the *original* JPEG download -- the web-UI
|
||||
image is a re-encoded WebP that drops EXIF.
|
||||
Legacy Grok/Aurora downloads use a private EXIF pair: ``ImageDescription`` =
|
||||
``"Signature: <base64>"`` together with ``Artist`` = the image UUID. Metadata-
|
||||
preserving editors can move that exact pair into XMP, PNG text, or IPTC fields.
|
||||
The signature is not locally verifiable (no public key); detection keys on both
|
||||
distinctive shapes, not on the signature's validity.
|
||||
"""
|
||||
decoded_pair = False
|
||||
try:
|
||||
import piexif
|
||||
from PIL import Image
|
||||
from PIL.IptcImagePlugin import getiptcinfo
|
||||
|
||||
with Image.open(image_path) as img:
|
||||
exif_bytes = img.info.get("exif")
|
||||
if not exif_bytes:
|
||||
return False
|
||||
tags = piexif.load(exif_bytes).get("0th", {})
|
||||
exif_bytes = exif_bytes_from_image(img)
|
||||
descriptions = [img.info.get(key) for key in ("Description", "ImageDescription")]
|
||||
creators = [img.info.get(key) for key in ("Author", "Artist", "Creator")]
|
||||
decoded_pair = any(
|
||||
xai_signature_pair(description, creator)
|
||||
for description in descriptions
|
||||
if isinstance(description, str)
|
||||
for creator in creators
|
||||
if isinstance(creator, str)
|
||||
)
|
||||
if iptc := getiptcinfo(img):
|
||||
caption = iptc.get((2, 120), b"")
|
||||
byline = iptc.get((2, 80), b"")
|
||||
if isinstance(caption, bytes) and isinstance(byline, bytes):
|
||||
decoded_pair = decoded_pair or _decoded_xai_signature_pair(caption, byline)
|
||||
if exif_bytes:
|
||||
tags = piexif.load(exif_bytes).get("0th", {})
|
||||
decoded_pair = decoded_pair or xai_signature_pair(
|
||||
_exif_text(tags, piexif.ImageIFD.ImageDescription),
|
||||
_exif_text(tags, piexif.ImageIFD.Artist),
|
||||
)
|
||||
except Exception as exc: # unopenable format / malformed EXIF
|
||||
logger.debug("xAI-signature EXIF read failed for %s: %s", image_path, exc)
|
||||
if decoded_pair:
|
||||
return True
|
||||
try:
|
||||
return xai_signature_in_metadata(scan_head(image_path))
|
||||
except OSError as exc:
|
||||
logger.debug("xAI-signature metadata scan failed for %s: %s", image_path, exc)
|
||||
return False
|
||||
|
||||
return xai_signature_pair(
|
||||
_exif_text(tags, piexif.ImageIFD.ImageDescription), _exif_text(tags, piexif.ImageIFD.Artist)
|
||||
)
|
||||
|
||||
|
||||
def _is_aigc_exif_value(raw: object) -> bool:
|
||||
"""Whether an EXIF tag value carries a China TC260 AIGC producer/service block.
|
||||
@@ -1063,8 +1238,7 @@ def _is_aigc_exif_value(raw: object) -> bool:
|
||||
return False
|
||||
if b"AIGC" not in raw:
|
||||
return False
|
||||
text = bytes(raw).decode("latin-1", "ignore")
|
||||
return any(field in text for field in TC260_AIGC_FIELDS)
|
||||
return aigc_label_from_metadata(bytes(raw)) is not None
|
||||
|
||||
|
||||
def _ai_exif_targets(loaded: dict[str, Any]) -> list[tuple[str, int, bytes, str]]:
|
||||
@@ -1100,15 +1274,18 @@ def _ai_exif_targets(loaded: dict[str, Any]) -> list[tuple[str, int, bytes, str]
|
||||
if xai_signature_pair(_exif_text(ifd0, piexif.ImageIFD.ImageDescription), _exif_text(ifd0, piexif.ImageIFD.Artist)):
|
||||
add("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription")
|
||||
add("0th", ifd0, piexif.ImageIFD.Artist, "Artist")
|
||||
# (b) known AI generator token in a 0th text tag.
|
||||
for tag, name in (
|
||||
(piexif.ImageIFD.Software, "Software"),
|
||||
(piexif.ImageIFD.Make, "Make"),
|
||||
(piexif.ImageIFD.Artist, "Artist"),
|
||||
(piexif.ImageIFD.ImageDescription, "ImageDescription"),
|
||||
# (b) known AI generator token or structured generator name in a text tag.
|
||||
for ifd_key, ifd, tag, name in (
|
||||
("0th", ifd0, piexif.ImageIFD.Software, "Software"),
|
||||
("0th", ifd0, piexif.ImageIFD.Make, "Make"),
|
||||
("0th", ifd0, piexif.ImageIFD.Artist, "Artist"),
|
||||
("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription"),
|
||||
("Exif", ifde, piexif.ExifIFD.UserComment, "UserComment"),
|
||||
):
|
||||
if any(token in _exif_text(ifd0, tag).lower() for token in AI_GENERATOR_TOKENS):
|
||||
add("0th", ifd0, tag, name)
|
||||
value = ifd.get(tag)
|
||||
text = value.decode("latin-1", "replace") if isinstance(value, bytes) else ""
|
||||
if any(token in text.lower() for token in AI_GENERATOR_TOKENS) or _structured_generator_from_metadata(text):
|
||||
add(ifd_key, ifd, tag, name)
|
||||
# (c) TC260 AIGC block in ImageDescription (0th) or UserComment (Exif sub-IFD).
|
||||
if _is_aigc_exif_value(ifd0.get(piexif.ImageIFD.ImageDescription)):
|
||||
add("0th", ifd0, piexif.ImageIFD.ImageDescription, "ImageDescription")
|
||||
@@ -1171,6 +1348,11 @@ def get_ai_metadata(image_path: Path) -> dict[str, str]:
|
||||
except Exception as exc:
|
||||
logger.debug("PIL could not open %s for AI-metadata scan: %s", image_path, exc)
|
||||
|
||||
from remove_ai_watermarks._internal.isobmff import ai_metadata_tags
|
||||
|
||||
for key, value in ai_metadata_tags(image_path).items():
|
||||
result.setdefault(key, value[:200] + ("…" if len(value) > 200 else ""))
|
||||
|
||||
# C2PA manifest fields from the single canonical parser (_internal/c2pa.py).
|
||||
c2pa = extract_c2pa_info(image_path)
|
||||
for key in (
|
||||
@@ -1221,9 +1403,9 @@ def get_ai_metadata(image_path: Path) -> dict[str, str]:
|
||||
if app_generator:
|
||||
result.setdefault("app_aigc", f"App AIGC disclosure ({app_generator})")
|
||||
|
||||
# xAI / Grok EXIF signature scheme (its only provenance signal).
|
||||
# xAI / Grok signature pair in EXIF or an equivalent container-native field.
|
||||
if xai_signature(image_path):
|
||||
result.setdefault("xai_signature", "xAI/Grok EXIF signature (Artist UUID + Signature blob)")
|
||||
result.setdefault("xai_signature", "xAI/Grok signature (UUID + Signature blob)")
|
||||
|
||||
# IPTC 2025.1 AI-disclosure XMP fields (Iptc4xmpExt:AISystemUsed etc.).
|
||||
if system := iptc_ai_system(image_path):
|
||||
@@ -1304,12 +1486,15 @@ def _jpeg_app_carries_ai(marker: int, payload: bytes) -> bool:
|
||||
or any(m in payload for m in AIGC_MARKERS)
|
||||
or any(m in payload for m in IPTC_AI_MARKERS)
|
||||
or any(m in payload for m in IPTC_AI_FIELD_MARKERS)
|
||||
or xai_signature_in_metadata(payload)
|
||||
)
|
||||
):
|
||||
return True
|
||||
# IPTC "Made with AI" record (APP13).
|
||||
if marker == 0xED and (
|
||||
any(m in payload for m in IPTC_AI_MARKERS) or any(m in payload for m in IPTC_AI_FIELD_MARKERS)
|
||||
any(m in payload for m in IPTC_AI_MARKERS)
|
||||
or any(m in payload for m in IPTC_AI_FIELD_MARKERS)
|
||||
or xai_signature_in_metadata(payload)
|
||||
):
|
||||
return True
|
||||
# A bare / wrapped China TC260 AIGC block (``AIGC{...}`` or ``{"AIGC":{...}}``) glued
|
||||
@@ -1425,6 +1610,7 @@ def _strip_png_metadata_lossless(source_path: Path, output_path: Path, keep_stan
|
||||
data = source_path.read_bytes()
|
||||
if not data.startswith(PNG_SIGNATURE):
|
||||
return False
|
||||
source_has_xai_pair = xai_signature(source_path)
|
||||
out = bytearray(PNG_SIGNATURE)
|
||||
view = memoryview(data)
|
||||
pos, n = 8, len(data)
|
||||
@@ -1447,7 +1633,13 @@ def _strip_png_metadata_lossless(source_path: Path, output_path: Path, keep_stan
|
||||
+ scrubbed
|
||||
+ struct.pack(">I", zlib.crc32(b"eXIf" + scrubbed))
|
||||
)
|
||||
elif not _png_chunk_should_be_dropped(chunk_type, payload, keep_standard, png_text_decode):
|
||||
elif not _png_chunk_should_be_dropped(
|
||||
chunk_type,
|
||||
payload,
|
||||
keep_standard,
|
||||
png_text_decode,
|
||||
xai_pair_present=source_has_xai_pair,
|
||||
):
|
||||
out += view[pos:end]
|
||||
if chunk_type == b"IEND":
|
||||
break
|
||||
@@ -1479,13 +1671,32 @@ def _scrubbed_png_exif(payload: bytes) -> bytes | None:
|
||||
return None
|
||||
|
||||
|
||||
def _keep_standard_text_metadata(key: str, value: object, keep_standard: bool) -> bool:
|
||||
def _xai_text_pair_member(key: str, value: object) -> bool:
|
||||
"""Whether one standard text item is a member of an already-proven xAI pair."""
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
normalized = key.lower().replace("-", "").replace("_", "")
|
||||
if normalized in {"description", "imagedescription", "captionabstract"}:
|
||||
return _XAI_SIGNATURE_RE.search(value) is not None
|
||||
if normalized in {"author", "artist", "creator", "byline"}:
|
||||
return _UUID_RE.fullmatch(value.strip()) is not None
|
||||
return False
|
||||
|
||||
|
||||
def _keep_standard_text_metadata(
|
||||
key: str,
|
||||
value: object,
|
||||
keep_standard: bool,
|
||||
*,
|
||||
xai_pair_present: bool = False,
|
||||
) -> bool:
|
||||
"""Whether a text metadata item is standard and free of AI provenance."""
|
||||
return (
|
||||
keep_standard
|
||||
and key in STANDARD_METADATA_KEYS
|
||||
and not _is_ai_key(key)
|
||||
and not (isinstance(value, str) and (_is_ai_value(value) or _is_aigc_exif_value(value)))
|
||||
and not (xai_pair_present and _xai_text_pair_member(key, value))
|
||||
)
|
||||
|
||||
|
||||
@@ -1494,6 +1705,8 @@ def _png_chunk_should_be_dropped(
|
||||
payload: bytes,
|
||||
keep_standard: bool,
|
||||
decode: Callable[[str, bytes], str],
|
||||
*,
|
||||
xai_pair_present: bool = False,
|
||||
) -> bool:
|
||||
"""True when a PNG chunk must be dropped by the lossless strip.
|
||||
|
||||
@@ -1510,7 +1723,12 @@ def _png_chunk_should_be_dropped(
|
||||
if chunk_type not in (b"tEXt", b"zTXt", b"iTXt"):
|
||||
return False
|
||||
keyword, _, text = decode(chunk_type.decode("ascii"), payload).partition("\x00")
|
||||
return not _keep_standard_text_metadata(keyword, text, keep_standard)
|
||||
return not _keep_standard_text_metadata(
|
||||
keyword,
|
||||
text,
|
||||
keep_standard,
|
||||
xai_pair_present=xai_pair_present,
|
||||
)
|
||||
|
||||
|
||||
# Fallback extension -> PIL save format, used only when the content sniff is
|
||||
@@ -1632,11 +1850,11 @@ def remove_ai_metadata(
|
||||
with open(source_path, "rb") as f:
|
||||
head = f.read(26)
|
||||
if source_path.suffix.lower() in _STREAMING_ISOBMFF_EXTS and is_isobmff(head):
|
||||
stripped, tc260_blanked = strip_isobmff_media_file(source_path, output_path)
|
||||
stripped, native_ai_blanked = strip_isobmff_media_file(source_path, output_path)
|
||||
logger.info(
|
||||
"Stream-blanked %d AI-provenance box(es) and %d native TC260 tag(s) → %s",
|
||||
"Stream-blanked %d AI-provenance box(es) and %d native AI metadata tag(s) → %s",
|
||||
stripped,
|
||||
tc260_blanked,
|
||||
native_ai_blanked,
|
||||
output_path,
|
||||
)
|
||||
return output_path
|
||||
@@ -1714,8 +1932,10 @@ def remove_ai_metadata(
|
||||
return output_path
|
||||
|
||||
# Read image and filter metadata
|
||||
with Image.open(source_path) as img:
|
||||
img = img.copy()
|
||||
with Image.open(source_path) as source_image:
|
||||
source_exif_bytes = exif_bytes_from_image(source_image)
|
||||
source_has_xai_pair = xai_signature(source_path)
|
||||
img = source_image.copy()
|
||||
# Pick the save format. Honor the caller's output extension (so a deliberate
|
||||
# source.png -> output.jpg conversion still works) UNLESS the SOURCE is misnamed
|
||||
# -- a lossless PNG/WebP whose extension lies (served as .jpg). There the output
|
||||
@@ -1759,9 +1979,18 @@ def remove_ai_metadata(
|
||||
continue
|
||||
# NovelAI and TC260 can stamp AI provenance into standard Title/Source/
|
||||
# Description fields. Share this decision with the lossless PNG walker.
|
||||
if _keep_standard_text_metadata(key, value, keep_standard):
|
||||
if _keep_standard_text_metadata(
|
||||
key,
|
||||
value,
|
||||
keep_standard,
|
||||
xai_pair_present=source_has_xai_pair,
|
||||
):
|
||||
kept_meta[key] = str(value) if not isinstance(value, str) else value
|
||||
|
||||
if source_exif_bytes:
|
||||
with contextlib.suppress(Exception):
|
||||
exif_data = piexif.load(source_exif_bytes)
|
||||
|
||||
# Apply cleaned metadata
|
||||
if save_kwargs["format"] == "PNG" and kept_meta:
|
||||
pnginfo = PngInfo()
|
||||
|
||||
@@ -50,6 +50,7 @@ from remove_ai_watermarks._internal.schema import require_schema_version
|
||||
from remove_ai_watermarks.metadata import (
|
||||
QUICK_SCAN_BYTES,
|
||||
SAMSUNG_EDITOR_MARKER,
|
||||
exif_bytes_from_image,
|
||||
exif_text,
|
||||
read_file_tail,
|
||||
)
|
||||
@@ -267,7 +268,10 @@ def _decoder_info(image_path: Path) -> dict[str, Any]:
|
||||
with Image.open(image_path) as img:
|
||||
# PIL types this mapping with a non-string key union (a DPI tuple key
|
||||
# exists), so the keys are normalized here rather than assumed.
|
||||
return {str(key): value for key, value in img.info.items()}
|
||||
info = {str(key): value for key, value in img.info.items()}
|
||||
if "exif" not in info and (exif_bytes := exif_bytes_from_image(img)):
|
||||
info["exif"] = exif_bytes
|
||||
return info
|
||||
except (UnidentifiedImageError, ImportError) as exc: # unsupported optional decoder
|
||||
logger.debug("PIL info unavailable for %s: %s", image_path, exc)
|
||||
return {}
|
||||
@@ -277,8 +281,9 @@ def _exif_pairs(info: dict[str, Any]) -> dict[str, str]:
|
||||
"""The 0th-IFD tags the verdict reads, under their tag NAMES.
|
||||
|
||||
Not a convenience: two probes key on names rather than on the raw bytes already
|
||||
in the regions. ``xai_signature_pair`` wants an (ImageDescription, Artist) pair,
|
||||
and ``_external_exif_generator`` looks for Software / Make / Artist /
|
||||
in the regions. The original xAI shape needs an (ImageDescription, Artist) pair
|
||||
(container-native pairs arrive through ``pil`` or structured regions), and
|
||||
``_external_exif_generator`` looks for Software / Make / Artist /
|
||||
ImageDescription. Ship the bytes alone and both silently return nothing, which
|
||||
is how a collector can silently lose Grok and NovelAI verdicts.
|
||||
"""
|
||||
@@ -392,6 +397,10 @@ def collect_metadata_record(
|
||||
issues.append({"stage": "trailer", "code": "collection-failed"})
|
||||
try:
|
||||
info = _decoder_info(image_path)
|
||||
if container == "isobmff":
|
||||
from remove_ai_watermarks._internal.isobmff import ai_metadata_tags
|
||||
|
||||
info.update(ai_metadata_tags(image_path, strict=True))
|
||||
exif = _exif_pairs(info)
|
||||
except Exception as exc:
|
||||
logger.debug("metadata decoder collection failed for %s: %s", image_path, exc)
|
||||
|
||||
+87
-9
@@ -46,6 +46,12 @@ _SPARKLE_TARGET = "remove_ai_watermarks.gemini_engine.detect_sparkle_confidence"
|
||||
SAMPLES_DIR = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance"
|
||||
|
||||
|
||||
def _write_c2pa_jpeg(tmp_path: Path, name: str, blob: bytes) -> Path:
|
||||
path = tmp_path / name
|
||||
path.write_bytes(b"\xff\xd8\xff\xe1jumbc2pa" + blob + b"\xff\xd9")
|
||||
return path
|
||||
|
||||
|
||||
class TestProvenanceEvidence:
|
||||
def test_exact_ai_claim_generator_can_assert_ai_without_source_type(self, tmp_path: Path):
|
||||
path = tmp_path / "firefly.png"
|
||||
@@ -464,6 +470,8 @@ class TestIdentifyNonPng:
|
||||
@pytest.mark.parametrize(
|
||||
("claim_generator", "platform"),
|
||||
[
|
||||
("Grok Imagine", "xAI Grok Imagine"),
|
||||
("Suno", "Suno"),
|
||||
("Higgsfield AI", "Higgsfield AI"),
|
||||
("recraft.ai", "Recraft"),
|
||||
("Topaz Labs Image API", "Topaz Labs"),
|
||||
@@ -644,13 +652,8 @@ class TestIdentifySamsungGalaxy:
|
||||
so the signer attribution must NOT trip the camera-vs-AI integrity clash.
|
||||
"""
|
||||
|
||||
def _jpeg(self, tmp_path: Path, name: str, blob: bytes) -> Path:
|
||||
path = tmp_path / name
|
||||
path.write_bytes(b"\xff\xd8\xff\xe1jumbc2pa" + blob + b"\xff\xd9")
|
||||
return path
|
||||
|
||||
def test_galaxy_trained_source_is_unverified_ai(self, tmp_path: Path):
|
||||
path = self._jpeg(tmp_path, "s25.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs trainedAlgorithmicMedia")
|
||||
path = _write_c2pa_jpeg(tmp_path, "s25.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs trainedAlgorithmicMedia")
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is True
|
||||
assert r.confidence == "medium"
|
||||
@@ -662,7 +665,7 @@ class TestIdentifySamsungGalaxy:
|
||||
def test_galaxy_genai_only_is_medium_ai(self, tmp_path: Path):
|
||||
# The Galaxy S24 case: no trainedAlgorithmicMedia, genAIType is the only
|
||||
# AI marker -- previously missed, now a medium-confidence verdict.
|
||||
path = self._jpeg(
|
||||
path = _write_c2pa_jpeg(
|
||||
tmp_path, "s24.jpg", b'Samsung Galaxy Galaxy S24 c2pa-rs PhotoEditor_Re_Edit_Data{"genAIType":1}'
|
||||
)
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
@@ -675,7 +678,7 @@ class TestIdentifySamsungGalaxy:
|
||||
def test_asus_gallery_signer_not_ai(self, tmp_path: Path):
|
||||
# ASUS Gallery signs edited photos; no AI source-type or genAIType, so the
|
||||
# platform is attributed but the verdict stays unknown.
|
||||
path = self._jpeg(tmp_path, "asus.jpg", b"/com.asus.gallery/3.8.0.98 c2pa-rs no ai marker")
|
||||
path = _write_c2pa_jpeg(tmp_path, "asus.jpg", b"/com.asus.gallery/3.8.0.98 c2pa-rs no ai marker")
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is None
|
||||
assert r.platform == "ASUS Gallery (C2PA signer)"
|
||||
@@ -685,13 +688,88 @@ class TestIdentifySamsungGalaxy:
|
||||
# A genuine Galaxy phone capture carries Samsung Galaxy C2PA provenance but
|
||||
# NO AI source-type / genAIType. It must stay is_ai=None -- the device cert
|
||||
# is authenticity provenance of a real photo, not an AI-generation signal.
|
||||
path = self._jpeg(tmp_path, "s25_capture.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs no ai marker")
|
||||
path = _write_c2pa_jpeg(tmp_path, "s25_capture.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs no ai marker")
|
||||
r = identify(path, check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is None
|
||||
assert r.platform == "Samsung Galaxy (C2PA)"
|
||||
assert any("C2PA" in w for w in r.watermarks)
|
||||
|
||||
|
||||
class TestIdentifyDiscoveredC2paSigners:
|
||||
@pytest.mark.parametrize(
|
||||
("blob", "platform"),
|
||||
[
|
||||
(b"Anthropic Claude Content Signing", "Anthropic Claude (C2PA signer)"),
|
||||
(b"CapCut/ c2pa-rs", "CapCut (C2PA signer)"),
|
||||
(b"TikTok Inc.", "TikTok (C2PA signer)"),
|
||||
(b"vivo Camera vivo X300 Pro", "vivo (camera, C2PA capture)"),
|
||||
],
|
||||
)
|
||||
def test_non_ai_signer_is_attributed_without_asserting_ai(self, tmp_path: Path, blob: bytes, platform: str):
|
||||
path = _write_c2pa_jpeg(tmp_path, "signed.jpg", blob)
|
||||
|
||||
report = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
assert report.is_ai_generated is None
|
||||
assert report.platform == platform
|
||||
|
||||
def test_structured_non_ai_capcut_generator_is_attributed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
path = tmp_path / "edited.png"
|
||||
from PIL import Image
|
||||
|
||||
Image.new("RGB", (32, 32)).save(path)
|
||||
monkeypatch.setattr(
|
||||
"remove_ai_watermarks.identify.extract_c2pa_info",
|
||||
lambda _path: {
|
||||
"has_c2pa": True,
|
||||
"issuer": "ByteDance",
|
||||
"claim_generator": "CapCut/ c2pa-tool/0.1.0 c2pa-rs/0.31.3",
|
||||
},
|
||||
)
|
||||
|
||||
report = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
assert report.is_ai_generated is None
|
||||
assert report.platform == "CapCut (C2PA signer)"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("issuer", "claim_generator", "platform"),
|
||||
[
|
||||
("xAI Grok Imagine", "c2pa-rs", "xAI Grok Imagine"),
|
||||
("Producer.ai", "producer_ai", "Producer.ai"),
|
||||
("", "FastVid", "FastVid"),
|
||||
("SPRING (SG) PTE. LTD.", "c2pa-rs", "SPRING (SG) PTE. LTD."),
|
||||
],
|
||||
)
|
||||
def test_ai_signer_or_generator_is_attributed(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
issuer: str,
|
||||
claim_generator: str,
|
||||
platform: str,
|
||||
):
|
||||
path = tmp_path / "generated.png"
|
||||
from PIL import Image
|
||||
|
||||
Image.new("RGB", (32, 32)).save(path)
|
||||
monkeypatch.setattr(
|
||||
"remove_ai_watermarks.identify.extract_c2pa_info",
|
||||
lambda _path: {
|
||||
"has_c2pa": True,
|
||||
"issuer": issuer,
|
||||
"claim_generator": claim_generator,
|
||||
"source_type": "trainedAlgorithmicMedia (AI-generated)",
|
||||
"ai_source_kind": "generated",
|
||||
},
|
||||
)
|
||||
|
||||
report = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
assert report.is_ai_generated is True
|
||||
assert report.platform == platform
|
||||
|
||||
|
||||
# ── End-to-end verdicts on real fixtures ────────────────────────────
|
||||
|
||||
|
||||
|
||||
+210
-1
@@ -945,6 +945,30 @@ class TestExifGenerator:
|
||||
def test_clean_image_is_none(self, tmp_clean_png: Path):
|
||||
assert exif_generator(tmp_clean_png) is None
|
||||
|
||||
def test_nested_generation_model_in_webp_user_comment(self, tmp_path: Path):
|
||||
import json
|
||||
|
||||
payload = json.dumps(
|
||||
{
|
||||
"generationParams": {
|
||||
"modelName": "Nano Banana synthetic fixture",
|
||||
"prompt": "private prompt must not become the platform label",
|
||||
}
|
||||
}
|
||||
)
|
||||
exif = piexif.dump({"Exif": {piexif.ExifIFD.UserComment: payload.encode("utf-8")}})
|
||||
source = tmp_path / "generated.webp"
|
||||
Image.new("RGB", (32, 32)).save(source, "WEBP", lossless=True, exif=exif)
|
||||
|
||||
assert exif_generator(source) == "Nano Banana synthetic fixture"
|
||||
assert (
|
||||
identify(source, check_visible=False).platform == "Nano Banana synthetic fixture (embedded generator tag)"
|
||||
)
|
||||
|
||||
output = tmp_path / "clean.webp"
|
||||
remove_ai_metadata(source, output)
|
||||
assert exif_generator(output) is None
|
||||
|
||||
|
||||
_FAKE_SIG = "A" * 120 # 64+ base64 chars; real Grok payloads are 300-1004
|
||||
_FAKE_UUID = "12345678-1234-1234-1234-123456789abc"
|
||||
@@ -970,12 +994,139 @@ def _grok_jpeg(tmp_path: Path, *, signature: str = _FAKE_SIG, artist: str = _FAK
|
||||
return path
|
||||
|
||||
|
||||
def _grok_imagemagick_png(tmp_path: Path) -> Path:
|
||||
"""Write the ImageMagick raw-profile EXIF form seen on converted PNGs."""
|
||||
exif = piexif.dump(
|
||||
{
|
||||
"0th": {
|
||||
piexif.ImageIFD.ImageDescription: f"Signature: {_FAKE_SIG}".encode("latin1"),
|
||||
piexif.ImageIFD.Artist: _FAKE_UUID.encode("latin1"),
|
||||
piexif.ImageIFD.Make: b"Canon",
|
||||
},
|
||||
"Exif": {},
|
||||
"GPS": {},
|
||||
"1st": {},
|
||||
}
|
||||
)
|
||||
profile = (
|
||||
"\nexif\n"
|
||||
+ f"{len(exif):8d}\n"
|
||||
+ "\n".join(exif.hex()[start : start + 72] for start in range(0, len(exif) * 2, 72))
|
||||
)
|
||||
info = PngInfo()
|
||||
info.add_text("Raw profile type exif", profile)
|
||||
path = tmp_path / "grok-imagemagick.png"
|
||||
Image.new("RGB", (64, 64), (70, 80, 90)).save(path, pnginfo=info)
|
||||
return path
|
||||
|
||||
|
||||
class TestXaiSignature:
|
||||
"""xAI / Grok's EXIF Signature + UUID-Artist provenance scheme."""
|
||||
"""xAI / Grok's Signature + UUID provenance scheme across metadata containers."""
|
||||
|
||||
@staticmethod
|
||||
def _png_text_pair(tmp_path: Path) -> Path:
|
||||
info = PngInfo()
|
||||
info.add_text("Description", f"Signature: {_FAKE_SIG}")
|
||||
info.add_text("Author", _FAKE_UUID)
|
||||
path = tmp_path / "grok-text.png"
|
||||
Image.new("RGB", (64, 64), (30, 40, 50)).save(path, pnginfo=info)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _jpeg_app_pair(tmp_path: Path, marker: bytes, payload: bytes, name: str) -> Path:
|
||||
path = tmp_path / name
|
||||
Image.new("RGB", (64, 64), (30, 40, 50)).save(path, "JPEG")
|
||||
data = path.read_bytes()
|
||||
segment = marker + (len(payload) + 2).to_bytes(2, "big") + payload
|
||||
path.write_bytes(data[:2] + segment + data[2:])
|
||||
return path
|
||||
|
||||
def test_signature_plus_uuid_detected(self, tmp_path: Path):
|
||||
assert xai_signature(_grok_jpeg(tmp_path)) is True
|
||||
|
||||
def test_imagemagick_png_raw_exif_profile_detected(self, tmp_path: Path):
|
||||
path = _grok_imagemagick_png(tmp_path)
|
||||
|
||||
with Image.open(path) as image:
|
||||
assert "exif" not in image.info
|
||||
assert image.getexif()
|
||||
|
||||
assert xai_signature(path) is True
|
||||
assert "xai_signature" in get_ai_metadata(path)
|
||||
|
||||
def test_png_description_and_author_pair_detected_and_stripped(self, tmp_path: Path):
|
||||
path = self._png_text_pair(tmp_path)
|
||||
|
||||
assert xai_signature(path) is True
|
||||
assert "xai_signature" in get_ai_metadata(path)
|
||||
|
||||
out = tmp_path / "clean.png"
|
||||
remove_ai_metadata(path, out)
|
||||
|
||||
assert xai_signature(out) is False
|
||||
with Image.open(out) as image:
|
||||
assert "Description" not in image.info
|
||||
assert "Author" not in image.info
|
||||
|
||||
def test_xmp_description_and_creator_pair_detected_and_stripped(self, tmp_path: Path):
|
||||
payload = (
|
||||
b"http://ns.adobe.com/xap/1.0/\x00"
|
||||
b'<rdf:Description xmlns:dc="http://purl.org/dc/elements/1.1/">'
|
||||
b"<dc:description>Signature: "
|
||||
+ _FAKE_SIG.encode()
|
||||
+ b"</dc:description><dc:creator>"
|
||||
+ _FAKE_UUID.encode()
|
||||
+ b"</dc:creator></rdf:Description>"
|
||||
)
|
||||
path = self._jpeg_app_pair(tmp_path, b"\xff\xe1", payload, "grok-xmp.jpg")
|
||||
|
||||
assert xai_signature(path) is True
|
||||
out = tmp_path / "clean-xmp.jpg"
|
||||
remove_ai_metadata(path, out)
|
||||
|
||||
assert xai_signature(out) is False
|
||||
assert _FAKE_SIG.encode() not in out.read_bytes()
|
||||
assert _FAKE_UUID.encode() not in out.read_bytes()
|
||||
|
||||
def test_xmp_signature_does_not_pair_with_an_unrelated_uuid(self, tmp_path: Path):
|
||||
payload = (
|
||||
b"http://ns.adobe.com/xap/1.0/\x00"
|
||||
b'<rdf:Description xmlns:dc="http://purl.org/dc/elements/1.1/" '
|
||||
b'xmp:InstanceID="xmp.iid:'
|
||||
+ _FAKE_UUID.encode()
|
||||
+ b'"><dc:description>Signature: '
|
||||
+ _FAKE_SIG.encode()
|
||||
+ b"</dc:description></rdf:Description>"
|
||||
)
|
||||
path = self._jpeg_app_pair(tmp_path, b"\xff\xe1", payload, "unrelated-xmp-uuid.jpg")
|
||||
|
||||
assert xai_signature(path) is False
|
||||
|
||||
def test_xmp_values_in_different_descriptions_do_not_form_a_pair(self, tmp_path: Path):
|
||||
payload = (
|
||||
b"http://ns.adobe.com/xap/1.0/\x00"
|
||||
b"<rdf:Description><dc:description>Signature: "
|
||||
+ _FAKE_SIG.encode()
|
||||
+ b"</dc:description></rdf:Description>"
|
||||
b"<rdf:Description><dc:creator>" + _FAKE_UUID.encode() + b"</dc:creator></rdf:Description>"
|
||||
)
|
||||
path = self._jpeg_app_pair(tmp_path, b"\xff\xe1", payload, "split-xmp-pair.jpg")
|
||||
|
||||
assert xai_signature(path) is False
|
||||
|
||||
def test_iptc_caption_and_byline_pair_detected_and_stripped(self, tmp_path: Path):
|
||||
payload = b"Photoshop 3.0\x00Caption-Abstract\x00Signature: " + _FAKE_SIG.encode()
|
||||
payload += b"\x00By-line\x00" + _FAKE_UUID.encode()
|
||||
path = self._jpeg_app_pair(tmp_path, b"\xff\xed", payload, "grok-iptc.jpg")
|
||||
|
||||
assert xai_signature(path) is True
|
||||
out = tmp_path / "clean-iptc.jpg"
|
||||
remove_ai_metadata(path, out)
|
||||
|
||||
assert xai_signature(out) is False
|
||||
assert _FAKE_SIG.encode() not in out.read_bytes()
|
||||
assert _FAKE_UUID.encode() not in out.read_bytes()
|
||||
|
||||
def test_real_grok_sample_detected(self):
|
||||
# Real committed Grok download (data/fixtures/provenance/grok-1.jpg); the EXIF
|
||||
# Signature + UUID-Artist pair is the only AI signal it carries.
|
||||
@@ -996,6 +1147,18 @@ class TestXaiSignature:
|
||||
# Incidental short "Signature: ..." text is below the 64-char base64 bar.
|
||||
assert xai_signature(_grok_jpeg(tmp_path, signature="ok")) is False
|
||||
|
||||
def test_png_signature_without_uuid_is_preserved(self, tmp_path: Path):
|
||||
info = PngInfo()
|
||||
info.add_text("Description", f"Signature: {_FAKE_SIG}")
|
||||
path = tmp_path / "signature-only.png"
|
||||
Image.new("RGB", (64, 64)).save(path, pnginfo=info)
|
||||
|
||||
assert xai_signature(path) is False
|
||||
out = tmp_path / "signature-only-clean.png"
|
||||
remove_ai_metadata(path, out)
|
||||
with Image.open(out) as image:
|
||||
assert image.info["Description"] == f"Signature: {_FAKE_SIG}"
|
||||
|
||||
def test_clean_image_is_false(self, tmp_clean_png: Path):
|
||||
assert xai_signature(tmp_clean_png) is False
|
||||
|
||||
@@ -1169,6 +1332,21 @@ class TestRemoveAiExif:
|
||||
assert xai_signature(out) is False
|
||||
assert has_ai_metadata(out) is False
|
||||
|
||||
def test_grok_raw_profile_stripped_on_png_output(self, tmp_path: Path):
|
||||
src = _grok_imagemagick_png(tmp_path)
|
||||
out = tmp_path / "clean.png"
|
||||
|
||||
remove_ai_metadata(src, out)
|
||||
|
||||
assert xai_signature(out) is False
|
||||
with Image.open(out) as image:
|
||||
assert image.getexif().get(piexif.ImageIFD.Make) == "Canon"
|
||||
|
||||
def test_non_text_exif_value_does_not_break_app_metadata_scrub(self):
|
||||
from remove_ai_watermarks.metadata import _app_metadata_evidence
|
||||
|
||||
assert _app_metadata_evidence((1, 2)) == (None, None)
|
||||
|
||||
def test_generator_make_token_stripped(self, tmp_path: Path):
|
||||
# Ideogram's EXIF Make="Ideogram AI" must be scrubbed on removal.
|
||||
exif = piexif.dump({"0th": {piexif.ImageIFD.Make: b"Ideogram AI"}, "Exif": {}, "GPS": {}, "1st": {}})
|
||||
@@ -1391,6 +1569,37 @@ class TestAIGCLabel:
|
||||
assert info["Label"] == "1"
|
||||
assert info["ContentProducer"] == "001191440300708461136T1308L"
|
||||
|
||||
def test_parses_lower_camel_case_tc260_fields(self, tmp_path: Path):
|
||||
import json
|
||||
|
||||
from remove_ai_watermarks.metadata import aigc_label
|
||||
|
||||
p = tmp_path / "aigc-lower-camel.jpg"
|
||||
payload = json.dumps(
|
||||
{
|
||||
"data": {
|
||||
"AIGC": {
|
||||
"label": "1",
|
||||
"contentProducer": "synthetic-producer",
|
||||
"produceId": "synthetic-id",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
exif = {"Exif": {piexif.ExifIFD.UserComment: payload.encode("ascii")}}
|
||||
Image.new("RGB", (32, 32)).save(p, exif=piexif.dump(exif))
|
||||
|
||||
info = aigc_label(p)
|
||||
|
||||
assert info == {
|
||||
"Label": "1",
|
||||
"ContentProducer": "synthetic-producer",
|
||||
"ProduceID": "synthetic-id",
|
||||
}
|
||||
out = tmp_path / "clean-lower-camel.jpg"
|
||||
remove_ai_metadata(p, out)
|
||||
assert aigc_label(out) is None
|
||||
|
||||
def test_has_ai_metadata_detects_raw_json_exif_form(self, tmp_path: Path):
|
||||
assert has_ai_metadata(self._aigc_exif_jpeg(tmp_path))
|
||||
|
||||
|
||||
@@ -211,6 +211,43 @@ class TestC2PA:
|
||||
assert info["ai_tool"] == "Dreamina"
|
||||
assert info["c2pa_identity_ai"] is True
|
||||
|
||||
def test_structured_xai_signer_identity_is_normalized(self):
|
||||
store = {
|
||||
"active_manifest": "active",
|
||||
"manifests": {
|
||||
"active": {
|
||||
"signature_info": {
|
||||
"issuer": "Self-signed ephemeral certificate (Content Authenticity SDK) -- LOCAL USE ONLY",
|
||||
"common_name": "xAI Grok Imagine",
|
||||
},
|
||||
"assertions": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
info = c2pa_info_from_manifest_store(store)
|
||||
|
||||
assert info["issuer"] == "xAI Grok Imagine"
|
||||
assert info["c2pa_identity_ai"] is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("signature", "issuer"),
|
||||
[
|
||||
({"issuer": "Bytedance Pte. Ltd."}, "ByteDance"),
|
||||
({"issuer": "TikTok Inc."}, "TikTok"),
|
||||
],
|
||||
)
|
||||
def test_structured_publisher_identity_does_not_assert_ai(self, signature, issuer):
|
||||
store = {
|
||||
"active_manifest": "active",
|
||||
"manifests": {"active": {"signature_info": signature, "assertions": []}},
|
||||
}
|
||||
|
||||
info = c2pa_info_from_manifest_store(store)
|
||||
|
||||
assert info["issuer"] == issuer
|
||||
assert "c2pa_identity_ai" not in info
|
||||
|
||||
def test_structured_invismark_exposes_algorithm_and_watermark_id(self):
|
||||
watermark_id = "83424621-03cb-40e3-9808-a9fae837156d"
|
||||
store = {
|
||||
|
||||
@@ -15,6 +15,7 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from remove_ai_watermarks.identify import (
|
||||
PROVENANCE_REPORT_SCHEMA_VERSION,
|
||||
@@ -157,6 +158,65 @@ class TestPlacementsTheRecordCouldDrop:
|
||||
assert "xai_signature" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_an_imagemagick_raw_exif_profile_survives(self, tmp_path: Path):
|
||||
"""Pillow decodes this PNG EXIF profile through getexif, not img.info."""
|
||||
import piexif
|
||||
|
||||
exif = piexif.dump(
|
||||
{
|
||||
"0th": {
|
||||
piexif.ImageIFD.ImageDescription: b"Signature: " + b"A" * 80,
|
||||
piexif.ImageIFD.Artist: b"3f2504e0-4f89-11d3-9a0c-0305e82c3301",
|
||||
}
|
||||
}
|
||||
)
|
||||
profile = (
|
||||
"\nexif\n"
|
||||
+ f"{len(exif):8d}\n"
|
||||
+ "\n".join(exif.hex()[start : start + 72] for start in range(0, len(exif) * 2, 72))
|
||||
)
|
||||
info = PngInfo()
|
||||
info.add_text("Raw profile type exif", profile)
|
||||
path = tmp_path / "grok-converted.png"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "PNG", pnginfo=info)
|
||||
|
||||
assert "xai_signature" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_a_container_native_xai_pair_survives(self, tmp_path: Path):
|
||||
"""The portable record must preserve xAI pairs moved out of EXIF."""
|
||||
info = PngInfo()
|
||||
info.add_text("Description", "Signature: " + "A" * 80)
|
||||
info.add_text("Author", "3f2504e0-4f89-11d3-9a0c-0305e82c3301")
|
||||
path = tmp_path / "grok-text.png"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "PNG", pnginfo=info)
|
||||
|
||||
assert "xai_signature" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_isobmff_generation_tags_survive(self, tmp_path: Path):
|
||||
def box(kind: bytes, payload: bytes) -> bytes:
|
||||
return (8 + len(payload)).to_bytes(4, "big") + kind + payload
|
||||
|
||||
def key(name: bytes) -> bytes:
|
||||
return box(b"mdta", name)
|
||||
|
||||
def value(index: int, payload: bytes) -> bytes:
|
||||
return box(index.to_bytes(4, "big"), box(b"data", b"\x00" * 8 + payload))
|
||||
|
||||
keys = box(b"keys", b"\x00" * 4 + (2).to_bytes(4, "big") + key(b"workflow") + key(b"title"))
|
||||
ilst = box(b"ilst", value(1, b'{"app":"ComfyUI"}') + value(2, b"standard title"))
|
||||
meta = box(b"meta", b"\x00" * 4 + keys + ilst)
|
||||
path = tmp_path / "workflow.mp4"
|
||||
path.write_bytes(
|
||||
b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom"
|
||||
+ box(b"mdat", b"synthetic-video")
|
||||
+ box(b"moov", box(b"udta", meta))
|
||||
)
|
||||
|
||||
assert "gen_params" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
|
||||
class TestRecordShape:
|
||||
def test_the_record_survives_json(self, tmp_path: Path):
|
||||
|
||||
@@ -175,6 +175,11 @@ def test_space_pin_is_dispatched_before_the_factory_rebuild() -> None:
|
||||
assert "pin-library.yml" in script
|
||||
assert "RAIW_HF_SPACE_TOKEN secret is not set" in script
|
||||
assert "restart_space" in script
|
||||
# Nested YAML indent inside python -c crashed 0.40.1's waiter with
|
||||
# IndentationError before it could see the dispatched pin run.
|
||||
waiter = re.search(r"python3 -c \"(.*?)\"", script)
|
||||
assert waiter is not None
|
||||
assert waiter.group(1).lstrip() == waiter.group(1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix" or shutil.which("bash") is None, reason="Executes an Ubuntu Bash workflow")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
@@ -11,6 +12,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import corpus_gap_scan as gap
|
||||
import sidecar_regression as regression
|
||||
import visible_eval
|
||||
import visible_positives as visible
|
||||
|
||||
|
||||
def _report(**overrides):
|
||||
@@ -25,7 +28,16 @@ def _report(**overrides):
|
||||
|
||||
|
||||
def test_gap_markers_are_case_insensitive():
|
||||
assert "c2pa" in gap._marker_hits(b"UPPERCASE C2PA MANIFEST")
|
||||
assert "ComfyUI" in gap._marker_hits(b"COMFYUI WORKFLOW")
|
||||
|
||||
|
||||
def test_gap_markers_require_structural_provenance_evidence():
|
||||
noise = b"c2pa digitalSourceType Samsung Galaxy Doubao PhotoEditor_Re_Edit Signature: " + b"A" * 80
|
||||
assert gap._marker_hits(noise) == []
|
||||
|
||||
tc260 = b'{"AIGC":{"label":"1","contentProducer":"synthetic"}}'
|
||||
assert "TC260 AIGC" in gap._marker_hits(tc260)
|
||||
assert "Samsung genAIType" in gap._marker_hits(b'PhotoEditor_Re_Edit_Data{"genAIType":1}')
|
||||
|
||||
|
||||
def test_gap_candidates_cover_more_than_blind_unknowns():
|
||||
@@ -45,6 +57,140 @@ def test_since_filter_uses_corpus_date_directories(tmp_path):
|
||||
assert gap._files(tmp_path, date(2026, 9, 10)) == [current]
|
||||
|
||||
|
||||
def test_gap_checkpoint_keeps_complete_rows_and_ignores_a_torn_tail(tmp_path):
|
||||
checkpoint = tmp_path / "report.csv.progress.jsonl"
|
||||
row = gap._base_row("2026-09-10/example.png", ".png", "1.2.3")
|
||||
checkpoint.write_text(json.dumps(row) + '\n{"path":"torn', encoding="utf-8")
|
||||
|
||||
assert gap._read_checkpoint(checkpoint) == {row["path"]: row}
|
||||
|
||||
gap._repair_checkpoint(checkpoint)
|
||||
second = gap._base_row("2026-09-10/second.png", ".png", "1.2.3")
|
||||
with checkpoint.open("a", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(second) + "\n")
|
||||
|
||||
assert gap._read_checkpoint(checkpoint) == {row["path"]: row, second["path"]: second}
|
||||
|
||||
|
||||
def test_visible_checkpoint_is_repaired_before_resume(tmp_path):
|
||||
checkpoint = tmp_path / "visible.jsonl"
|
||||
first = {"path": "/corpus/2026-09-10/first.png", "keys": [], "status": "ok"}
|
||||
checkpoint.write_text(json.dumps(first) + '\n{"path":"torn', encoding="utf-8")
|
||||
|
||||
visible._repair_checkpoint(checkpoint)
|
||||
with checkpoint.open("a", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps({"path": "/corpus/2026-09-10/second.png", "keys": [], "status": "ok"}) + "\n")
|
||||
|
||||
assert [row["path"] for row in visible._read_records(checkpoint)] == [
|
||||
"/corpus/2026-09-10/first.png",
|
||||
"/corpus/2026-09-10/second.png",
|
||||
]
|
||||
|
||||
|
||||
def test_visible_since_filter_uses_corpus_date_directories(tmp_path):
|
||||
old = tmp_path / "2026-09-01" / "old.png"
|
||||
current = tmp_path / "2026-09-10" / "current.png"
|
||||
nested = tmp_path / "2026-09-10" / "nested" / "current.webp"
|
||||
undated = tmp_path / "loose.png"
|
||||
for path in (old, current, nested, undated):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.touch()
|
||||
|
||||
assert visible._files(tmp_path, date(2026, 9, 10)) == [current, nested]
|
||||
|
||||
|
||||
def test_visible_discovery_candidates_include_unmapped_provenance_and_unbiased_quiet_rows():
|
||||
rows = [
|
||||
{"path": "known.png", "keys": ["qwen"], "status": "ok", "uscc": "unmapped-a"},
|
||||
{"path": "a.png", "keys": [], "status": "ok", "uscc": "unmapped-a"},
|
||||
{"path": "b.png", "keys": [], "status": "ok", "platform": "Google Gemini"},
|
||||
{"path": "c.png", "keys": [], "status": "ok"},
|
||||
{"path": "bad.png", "keys": [], "status": "unreadable", "uscc": "unmapped-b"},
|
||||
]
|
||||
|
||||
selected = visible._discovery_candidates(rows, per_cohort=2, random_quiet=10, seed=7)
|
||||
|
||||
assert {row["path"] for row in selected} == {"a.png", "b.png", "c.png"}
|
||||
assert next(row for row in selected if row["path"] == "a.png")["discovery_strata"] == [
|
||||
"tc260:unmapped-a",
|
||||
"unbiased-quiet",
|
||||
]
|
||||
|
||||
|
||||
def test_visible_evaluation_inventory_follows_the_registry():
|
||||
from remove_ai_watermarks.watermark_registry import mark_keys
|
||||
|
||||
assert tuple(mark_keys()) == visible_eval.MARKS
|
||||
|
||||
|
||||
def test_gap_worker_error_is_a_complete_resumable_row(monkeypatch, tmp_path):
|
||||
path = tmp_path / "bad.bin"
|
||||
path.write_bytes(b"not an image")
|
||||
calls = []
|
||||
|
||||
def fail(_path):
|
||||
calls.append(_path)
|
||||
raise ValueError("broken\ninput")
|
||||
|
||||
monkeypatch.setattr(gap, "extract_provenance_evidence", fail)
|
||||
row = gap._scan_one((str(path), "bad.bin", "1.2.3"))
|
||||
|
||||
assert set(row) == set(gap.REPORT_FIELDS)
|
||||
assert row["candidate_classes"] == "identify_error"
|
||||
assert row["error"] == "ValueError: broken input"
|
||||
assert calls == [path]
|
||||
|
||||
|
||||
def test_gap_worker_records_c2pa_review_fields(monkeypatch, tmp_path):
|
||||
path = tmp_path / "signed.png"
|
||||
path.touch()
|
||||
evidence = SimpleNamespace(
|
||||
c2pa_info={
|
||||
"issuer": "Example signer",
|
||||
"claim_generator": "Example generator",
|
||||
"actions": "created",
|
||||
"source_type": "trainedAlgorithmicMedia (AI-generated)",
|
||||
"c2pa_validation_state": "Valid",
|
||||
},
|
||||
scan=b"c2pa",
|
||||
)
|
||||
monkeypatch.setattr(gap, "extract_provenance_evidence", lambda _path: evidence)
|
||||
monkeypatch.setattr(
|
||||
gap,
|
||||
"identify_from_evidence",
|
||||
lambda actual: (
|
||||
SimpleNamespace(
|
||||
is_ai_generated=True,
|
||||
platform="Example",
|
||||
confidence="high",
|
||||
watermarks=["C2PA"],
|
||||
signals=[SimpleNamespace(name="c2pa")],
|
||||
integrity_clashes=[],
|
||||
)
|
||||
if actual is evidence
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
row = gap._scan_one((str(path), "signed.png", "1.2.3"))
|
||||
|
||||
assert row["c2pa_issuer"] == "Example signer"
|
||||
assert row["c2pa_claim_generator"] == "Example generator"
|
||||
assert row["c2pa_actions"] == "created"
|
||||
assert row["c2pa_source_type"] == "trainedAlgorithmicMedia (AI-generated)"
|
||||
assert row["c2pa_validation"] == "Valid"
|
||||
|
||||
|
||||
def test_gap_summary_does_not_report_a_negative_hidden_count(capsys):
|
||||
row = gap._base_row("candidate.png", ".png", "1.2.3")
|
||||
row["candidate_classes"] = "blind_marker"
|
||||
|
||||
gap._summarize([row])
|
||||
|
||||
output = capsys.readouterr()
|
||||
assert "more candidate row" not in output.out + output.err
|
||||
|
||||
|
||||
def test_sidecar_regression_prefers_stable_signal_names():
|
||||
sidecar = {"signals": ["visible_qwen"], "watermarks": ["wording may change"]}
|
||||
assert regression.sidecar_families(sidecar) == {"visible_qwen"}
|
||||
|
||||
@@ -65,6 +65,30 @@ def _video_with_tc260(path: Path, *, media_payload: bytes = _VIDEO_PAYLOAD) -> P
|
||||
return path
|
||||
|
||||
|
||||
def _video_with_comfyui_metadata(path: Path, *, media_payload: bytes = _VIDEO_PAYLOAD) -> Path:
|
||||
workflow = b'{"nodes":[{"type":"KSampler"}],"app":"ComfyUI"}'
|
||||
prompt = b'{"1":{"class_type":"KSampler"}}'
|
||||
keys = _box(
|
||||
b"keys",
|
||||
b"\x00\x00\x00\x00"
|
||||
+ (4).to_bytes(4, "big")
|
||||
+ _metadata_key(b"workflow")
|
||||
+ _metadata_key(b"prompt")
|
||||
+ _metadata_key(b"title")
|
||||
+ _metadata_key(b"software"),
|
||||
)
|
||||
ilst = _box(
|
||||
b"ilst",
|
||||
_metadata_value(1, workflow)
|
||||
+ _metadata_value(2, prompt)
|
||||
+ _metadata_value(3, b"standard title")
|
||||
+ _metadata_value(4, b"ordinary editor"),
|
||||
)
|
||||
meta = _box(b"meta", b"\x00\x00\x00\x00" + keys + ilst)
|
||||
path.write_bytes(_MP4_FTYP + _box(b"mdat", media_payload) + _box(b"moov", _box(b"udta", meta)))
|
||||
return path
|
||||
|
||||
|
||||
def _hdlr(handler: bytes) -> bytes:
|
||||
# version/flags + pre_defined + handler_type + reserved[3]; the TC260 walker
|
||||
# never parses it, but a real-shaped hdlr keeps the fixture honest.
|
||||
@@ -564,6 +588,24 @@ class TestVideoDependencies:
|
||||
|
||||
|
||||
class TestVideoMetadataApi:
|
||||
def test_inspects_and_removes_comfyui_mp4_metadata(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.video import inspect_video_metadata, remove_video_metadata
|
||||
|
||||
source = _video_with_comfyui_metadata(tmp_path / "source.mp4")
|
||||
output = tmp_path / "clean.mp4"
|
||||
|
||||
report = inspect_video_metadata(source)
|
||||
assert set(report.markers) >= {"workflow", "prompt"}
|
||||
|
||||
result = remove_video_metadata(source, output)
|
||||
cleaned = output.read_bytes()
|
||||
assert result.remaining == {}
|
||||
assert len(cleaned) == source.stat().st_size
|
||||
assert _VIDEO_PAYLOAD in cleaned
|
||||
assert b"standard title" in cleaned
|
||||
assert b"ordinary editor" in cleaned
|
||||
assert b"ComfyUI" not in cleaned
|
||||
|
||||
def test_top_level_api_is_lazy_exported(self):
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
|
||||
@@ -800,7 +800,7 @@ name = "cuda-bindings"
|
||||
version = "13.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cuda-pathfinder", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "cuda-pathfinder" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" },
|
||||
@@ -833,43 +833,43 @@ wheels = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
cublas = [
|
||||
{ name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cudart = [
|
||||
{ name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cufft = [
|
||||
{ name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cufile = [
|
||||
{ name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cupti = [
|
||||
{ name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
curand = [
|
||||
{ name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cusolver = [
|
||||
{ name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
cusparse = [
|
||||
{ name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvjitlink = [
|
||||
{ name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvrtc = [
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
nvtx = [
|
||||
{ name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
|
||||
{ name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1066,7 +1066,7 @@ wheels = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
http = [
|
||||
{ name = "aiohttp", marker = "python_full_version < '3.13'" },
|
||||
{ name = "aiohttp" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1198,8 +1198,8 @@ name = "httpcore2"
|
||||
version = "2.12.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "h11", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "truststore", marker = "sys_platform != 'emscripten'" },
|
||||
{ name = "h11" },
|
||||
{ name = "truststore" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" }
|
||||
wheels = [
|
||||
@@ -1461,15 +1461,15 @@ name = "lightning"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pytorch-lightning", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pyyaml", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchmetrics", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "fsspec", extra = ["http"] },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pytorch-lightning" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchmetrics" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/39/b8/fbf1469d917c3ed3b22e00ab3a852fb9a5f85b9dc7f7060eedc3449098e7/lightning-2.6.6.tar.gz", hash = "sha256:1c6329e241f8a85f29fedead89c1ee138b36caa08de60c260c6a7eaee81c8295", size = 659902, upload-time = "2026-09-10T09:41:03.009Z" }
|
||||
wheels = [
|
||||
@@ -1481,8 +1481,8 @@ name = "lightning-utilities"
|
||||
version = "0.15.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f1/45/7fa8f56b17dc0f0a41ec70dd307ecd6787254483549843bef4c30ab5adce/lightning_utilities-0.15.3.tar.gz", hash = "sha256:792ae0204c79f6859721ac7f386c237a33b0ed06ba775009cb894e010a842033", size = 33553, upload-time = "2026-02-22T14:48:53.348Z" }
|
||||
wheels = [
|
||||
@@ -1896,7 +1896,7 @@ name = "nvidia-cublas"
|
||||
version = "13.1.1.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cuda-nvrtc" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
|
||||
@@ -1935,7 +1935,7 @@ name = "nvidia-cudnn-cu13"
|
||||
version = "9.24.0.43"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cublas" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" },
|
||||
@@ -1947,7 +1947,7 @@ name = "nvidia-cufft"
|
||||
version = "12.0.0.61"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
|
||||
@@ -1977,9 +1977,9 @@ name = "nvidia-cusolver"
|
||||
version = "12.0.4.66"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cusparse", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-cublas" },
|
||||
{ name = "nvidia-cusparse" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
|
||||
@@ -1991,7 +1991,7 @@ name = "nvidia-cusparse"
|
||||
version = "12.6.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
|
||||
@@ -2146,7 +2146,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" }
|
||||
wheels = [
|
||||
@@ -2175,7 +2175,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ed/e8/c2959c0f969ea4ae96b55ad10c108c0410de4e8b3550d9941a4cfdd52703/opencv_python-4.14.0.94.tar.gz", hash = "sha256:afadc76a38a07713a4c2b4702da917cf0ed489be57e2a40a34e670189731de7d", size = 96400225, upload-time = "2026-07-28T19:23:09.546Z" }
|
||||
wheels = [
|
||||
@@ -2206,7 +2206,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" }
|
||||
wheels = [
|
||||
@@ -2235,7 +2235,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/67/8a/51ff1c6287be0b6cd3070cf095875da4ff7d6eda38507e6ada5608786a78/opencv_python_headless-4.14.0.94.tar.gz", hash = "sha256:4afa2ea1214453648be88259f035712454faa9039b686de7753569ba8eec1577", size = 96405664, upload-time = "2026-07-28T19:23:48.758Z" }
|
||||
wheels = [
|
||||
@@ -3179,14 +3179,14 @@ name = "pytorch-lightning"
|
||||
version = "2.6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fsspec", extra = ["http"], marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "pyyaml", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchmetrics", marker = "python_full_version < '3.13'" },
|
||||
{ name = "tqdm", marker = "python_full_version < '3.13'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
{ name = "fsspec", extra = ["http"] },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchmetrics" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/97/e4/5025cb42311b4f8cc6d1b12cd81399ca433159f8e3831ab229c5a6c5177e/pytorch_lightning-2.6.6.tar.gz", hash = "sha256:52c52b20522dd3d3feb3bb3aae2dbe3f291e8e836d4111a74cfd72e9df018834", size = 662507, upload-time = "2026-09-10T09:41:16.684Z" }
|
||||
wheels = [
|
||||
@@ -3205,7 +3205,7 @@ resolution-markers = [
|
||||
"(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5a/75/50581633d199812205ea8cdd0f6d52f12a624886b74bf1486335b67f01ff/pywavelets-1.9.0.tar.gz", hash = "sha256:148d12203377772bea452a59211d98649c8ee4a05eff019a9021853a36babdc8", size = 3938340, upload-time = "2025-08-04T16:20:04.978Z" }
|
||||
wheels = [
|
||||
@@ -3281,7 +3281,7 @@ resolution-markers = [
|
||||
"(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/25/84/a9e80e33eb114e78bd2f1829a90d72d8d4d660109c721a8aa2f0d58d7592/pywavelets-1.10.0.tar.gz", hash = "sha256:f3cb8640225a8f3fdcbc3f6f3bd112c78927c9407cbfa65a5ff303990a6a3e32", size = 3948163, upload-time = "2026-09-07T17:15:15.637Z" }
|
||||
@@ -3449,7 +3449,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "remove-ai-watermarks"
|
||||
version = "0.40.1"
|
||||
version = "0.40.3"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "c2pa-python" },
|
||||
@@ -4067,10 +4067,10 @@ name = "torchmetrics"
|
||||
version = "1.9.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "lightning-utilities", marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "packaging", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning-utilities" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
{ name = "packaging" },
|
||||
{ name = "torch" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/34/39b8b749333db56c0585d7a11fa62a283c087bb1dfc897d69fb8cedbefb1/torchmetrics-1.9.0.tar.gz", hash = "sha256:a488609948600df52d3db4fcdab02e62aab2a85ef34da67037dc3e65b8512faa", size = 581765, upload-time = "2026-03-09T17:41:22.443Z" }
|
||||
wheels = [
|
||||
@@ -4165,13 +4165,13 @@ name = "trustmark"
|
||||
version = "0.9.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "einops", marker = "python_full_version < '3.13'" },
|
||||
{ name = "lightning", marker = "python_full_version < '3.13'" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
|
||||
{ name = "omegaconf", marker = "python_full_version < '3.13'" },
|
||||
{ name = "six", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torch", marker = "python_full_version < '3.13'" },
|
||||
{ name = "torchvision", marker = "python_full_version < '3.13'" },
|
||||
{ name = "einops" },
|
||||
{ name = "lightning" },
|
||||
{ name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } },
|
||||
{ name = "omegaconf" },
|
||||
{ name = "six" },
|
||||
{ name = "torch" },
|
||||
{ name = "torchvision" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/53/41/32e84596af94e8696c55060d55eb9eed9d99722a673d12212a1c0c4ef1d2/trustmark-0.9.2.tar.gz", hash = "sha256:bcaa3bc38ead1c0558df81932672f34e306efaed31389da57dbed100e236b28a", size = 63780, upload-time = "2026-09-08T10:58:11.601Z" }
|
||||
|
||||
|
||||
Reference in new issue
Block a user