From 25a1acc53b8ba7b2679a1b1284892d171671188c Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Sat, 30 May 2026 11:44:53 -0700 Subject: [PATCH] Detect TC260 AIGC label in JPEG EXIF and late/attribute PNG XMP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A corpus audit surfaced China TC260 AIGC-labeled images that `identify` missed. Three detection gaps in `aigc_label`, all fixed: - raw-JSON `{"AIGC":{...}}` in JPEG EXIF (UserComment): brace-matched from the scan head with `json.raw_decode`, gated on a TC260 field like the PNG-chunk path. (Doubao-class output via that export surface.) - XMP attribute form `TC260:AIGC="{...}"` (PicWish): folded into the element regex as a second alternation. - TC260 XMP packet appended after a large `IDAT`, past the 1 MB scan window: `scan_head` now appends late PNG metadata chunks via `_png_late_metadata`, mirroring the existing ISOBMFF late-box scan. Adds `scripts/corpus_gap_scan.py`: runs `identify` over a corpus, writes the per-file report CSV, and flags `unknown` files that carry a known marker in their metadata region (the audit that found these gaps). Scanning only the metadata region — not the whole file — avoids the random short-token collisions inside compressed PNG/JPEG streams. On the local corpus this lifts 3 files from `unknown` to AI (China AIGC) and leaves zero false gap candidates. Synthetic piexif/PngInfo fixtures cover all three forms. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 4 +- README.md | 6 +- scripts/corpus_gap_scan.py | 215 +++++++++++++++++++++++++++ src/remove_ai_watermarks/metadata.py | 109 +++++++++++--- tests/test_metadata.py | 76 ++++++++++ 5 files changed, 388 insertions(+), 22 deletions(-) create mode 100644 scripts/corpus_gap_scan.py diff --git a/CLAUDE.md b/CLAUDE.md index 2352269..f85ebd8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ You are a **principal Python engineer** maintaining a CLI tool and library for r - `noai/c2pa.py` — PNG chunk parser; use `extract_c2pa_chunk(path)` to get raw caBX payload, `has_c2pa_metadata(path)` to detect. Do not reimplement chunk parsing. `extract_c2pa_info(path)` sets `synthid_watermark`/`synthid_vendors` when the manifest is signed by a SynthID-using vendor, and `soft_binding`/`soft_binding_vendors` when a `c2pa.soft-binding` `alg` names a forensic-watermark vendor (`soft_binding_vendors_in(buffer)` is the shared byte-scan, used by both the PNG parser and the non-PNG binary path). - `noai/constants.py` — PNG_SIGNATURE, C2PA_CHUNK_TYPE, C2PA_SIGNATURES, C2PA_ISSUERS, `SYNTHID_C2PA_ISSUERS` (issuers that pair SynthID with C2PA: Google, OpenAI), and `C2PA_SOFT_BINDINGS` (soft-binding `alg` prefix → forensic-watermark vendor: Adobe TrustMark, Digimarc, Imatag, Steg.AI, Microsoft, ...). Add a new issuer/binding here, not inline. -- `metadata.py` — `scan_head(path, size=1MB)` is the shared input for every C2PA/AIGC/IPTC byte scan: first `size` bytes plus, for ISOBMFF, the late provenance-box payloads from `isobmff.scan_c2pa_region` (catches a manifest after a large `mdat`); behavior-neutral (`f.read(size)`) for non-ISOBMFF. Use it instead of `open().read(1MB)` for any new marker scan. `synthid_source(path)` returns the vendor name(s) if the C2PA manifest implies a SynthID pixel watermark, else None. Format-agnostic: PNG via the caBX parser, JPEG/WebP/AVIF/HEIF/JXL via a binary scan (C2PA marker + SynthID issuer + AI-source marker). `get_ai_metadata` surfaces the verdict, and `metadata --check` prints it as a callout. Both `get_ai_metadata` and `has_ai_metadata` guard the PIL open with `except Exception` (HEIC/unknown formats raise non-OSError) and fall through to the binary scan. `xai_signature(path)` detects xAI/Grok's EXIF-only scheme (`ImageDescription` = `Signature: ` + UUID `Artist`); it feeds `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify`. `iptc_ai_system(path)` detects the IPTC Photo Metadata 2025.1 AI-disclosure XMP properties (`IPTC_AI_FIELD_MARKERS` = `AISystemUsed`/`AISystemVersionUsed`/`AIPromptInformation`/`AIPromptWriterName`) and returns the `AISystemUsed` generator name (or `"fields present"`). `remove_ai_metadata` routes **ISOBMFF video** (`.mp4`/`.mov`/`.m4v`) through the same `isobmff.strip_c2pa_boxes` as AVIF/HEIF (MP4 is ISOBMFF), and `_scrub_ai_exif` removes the xAI signature + AI-generator EXIF tags on JPEG output. +- `metadata.py` — `scan_head(path, size=1MB)` is the shared input for every C2PA/AIGC/IPTC byte scan: first `size` bytes plus the payloads of any provenance metadata found beyond that window — for ISOBMFF, the late provenance boxes from `isobmff.scan_c2pa_region` (catches a manifest after a large `mdat`); for **PNG**, the late `tEXt`/`iTXt`/`zTXt`/`eXIf`/`iCCP` chunks from `_png_late_metadata` (catches an XMP/EXIF packet appended after a large `IDAT`, e.g. a TC260 AIGC label at ~2.7 MB). Behavior-neutral (`f.read(size)`) for non-ISOBMFF inputs and for any file that fits within `size`. Use it instead of `open().read(1MB)` for any new marker scan. `synthid_source(path)` returns the vendor name(s) if the C2PA manifest implies a SynthID pixel watermark, else None. Format-agnostic: PNG via the caBX parser, JPEG/WebP/AVIF/HEIF/JXL via a binary scan (C2PA marker + SynthID issuer + AI-source marker). `get_ai_metadata` surfaces the verdict, and `metadata --check` prints it as a callout. Both `get_ai_metadata` and `has_ai_metadata` guard the PIL open with `except Exception` (HEIC/unknown formats raise non-OSError) and fall through to the binary scan. `xai_signature(path)` detects xAI/Grok's EXIF-only scheme (`ImageDescription` = `Signature: ` + UUID `Artist`); it feeds `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify`. `iptc_ai_system(path)` detects the IPTC Photo Metadata 2025.1 AI-disclosure XMP properties (`IPTC_AI_FIELD_MARKERS` = `AISystemUsed`/`AISystemVersionUsed`/`AIPromptInformation`/`AIPromptWriterName`) and returns the `AISystemUsed` generator name (or `"fields present"`). `remove_ai_metadata` routes **ISOBMFF video** (`.mp4`/`.mov`/`.m4v`) through the same `isobmff.strip_c2pa_boxes` as AVIF/HEIF (MP4 is ISOBMFF), and `_scrub_ai_exif` removes the xAI signature + AI-generator EXIF tags on JPEG output. - `identify.py` — `identify(path)` aggregates every locally-readable signal (C2PA issuer→platform, C2PA soft-binding forensic-watermark vendor, IPTC "Made with AI" + IPTC 2025.1 `AISystemUsed`, embedded SD/ComfyUI params, SynthID proxy, xAI/Grok EXIF signature via `metadata.xai_signature`, the China TC260 AIGC label via `metadata.aigc_label`, the HuggingFace `hf-job-id` job marker via `metadata.huggingface_job`, the Samsung Galaxy AI editing marker via `metadata.samsung_genai`, visible Gemini sparkle, open invisible watermark, Adobe TrustMark via `trustmark_detector`) into one `ProvenanceReport`. `is_ai_generated` is True or None (never asserted False — stripped metadata is not proof of clean origin). The `hf_job`, visible-sparkle, and Samsung `samsung_genai` signals are **medium** confidence: each lifts an otherwise-Unknown verdict to a tentative AI (`hf_only` / `visible_only` / `samsung_only`, parallel branches) but is excluded from the high-confidence `ai_from_metadata` set, so none overrides a hard metadata signal. Visible-sparkle is promoted only at confidence ≥ `_SPARKLE_THRESHOLD` (0.5; corpus-tuned to separate Gemini sparkles ≥0.56 from non-sparkle ≤0.49). The cv2 dependency lives in `gemini_engine.detect_sparkle_confidence`, not here. **C2PA platform attribution is device-token-first, issuer-scan fallback** (`_device_platform` scans manifest bytes for `_DEVICE_C2PA_PLATFORM` tokens, then `_attribute_platform`/`_ISSUER_PLATFORM`). **Why, verified on real signed files 2026-05-26:** the old issuer-only byte-scan matched ANY issuer substring anywhere, so multi-entity manifests mis-attributed -- Leica→"Truepic" (a signing authority in the trust chain), Nikon→"Adobe Firefly" (XMP-toolkit "Adobe" + the sample's "Adobe_MAX" name), Pixel→"Google (Gemini)" ("Google LLC" cert org), Truepic→"Google". A distinctive device token wins instead. **Token distinctiveness is load-bearing:** bare `b"Truepic"` mis-fires (it appears in unrelated trust chains -- it mis-attributed the OpenAI `chatgpt-1.png` fixture), so the token is the specific `b"Truepic_Lens"` from the Lens SDK claim generator; likewise `b"Pixel Camera"` (cert CN) not bare `b"Pixel"`. `_DEVICE_C2PA_PLATFORM` lists ONLY tokens **verified against a real C2PA file**: Leica (`lc_c2pa`/`Leica Camera`), Nikon (`NIKON`), Pixel (`Pixel Camera` -- from a real Pixel 10 Pro file attached to c2pa-rs issue #1609/#1554), Sony (`sony.sig`/`sony.cert` -- Sony's own C2PA assertion namespace, verified on a real Sony PXW-Z300 file; NOT bare "Sony" which is a common EXIF Make), Truepic (`Truepic_Lens`). Canon/Bria have **no public direct-download C2PA sample** (checked exhaustively: GitHub issue/PR attachments, contentcredentials gallery, HF datasets -- all upload-to-verify or token-gated; Canon's only public file was a self-signed hobbyist CR3, not factory), so they stay unmapped until a real file is captured (same fixture discipline as Grok/Doubao). The Sony sample is video (MP4) -- our ISOBMFF C2PA path detects it; Sony Alpha stills likely share the `sony.*` namespace but are not separately verified. **Samsung Galaxy + ASUS Gallery live in a separate `_SIGNER_C2PA_PLATFORM` (scanned after `_device_platform`, before the issuer fallback), NOT in `_DEVICE_C2PA_PLATFORM`** — verified on real signed files 2026-05-29. Reason: a Galaxy phone stamps BOTH its device cert AND a `trainedAlgorithmicMedia`/genAIType AI marker on a Generative-Edit image, so treating it as a "genuine camera capture" would false-fire integrity-clash rule 2 on every Galaxy AI edit. The signer tokens (`b"Samsung Galaxy"` cert org — distinct from the EXIF `SM-xxxx` model string on ordinary Samsung photos; `b"com.asus.gallery"` claim generator) only resolve the platform label; the AI verdict still comes from the source-type / genAIType. ASUS Gallery is a C2PA-signed edit with no AI marker, so it attributes the platform without asserting `is_ai`. **Samsung's `genAIType` (in the proprietary `PhotoEditor_Re_Edit_Data` JSON) is an undocumented Galaxy-AI editing marker** (`metadata.samsung_genai`, gated on the `PhotoEditor_Re_Edit_Data` container; non-zero value = AI tool used, values {1,5} observed): medium-confidence because the field has no public spec (verified 2026-05-29: absent from C2PA spec + Samsung docs), but it co-occurred with `trainedAlgorithmicMedia` in 3/3 verified files that record a source-type and was the SOLE AI marker on a Galaxy S24 file that omits the source type. Camera C2PA marks capture authenticity, not AI (Pixel carries `computationalCapture`, not `trainedAlgorithmicMedia`), so these never set `is_ai` -- that stays driven by digital-source-type. `c2pa.cbor_text_after` (now public) is best-effort for the `generator` detail string only and can be None when the manifest keys it `claim_generator_info` (Pixel). **Issuer→generator mapping is `is_ai`-gated** (`_attribute_platform(issuers, is_ai=c2pa_is_ai)`): a specific AI-generator platform is named only when the digital-source-type is `trainedAlgorithmicMedia`; on a non-AI source an issuer substring is treated as incidental (an "Adobe XMP" toolkit string in an *unmapped* Canon/Sony capture would otherwise mislabel it "Adobe Firefly"), so it degrades to the neutral "C2PA signer: X" label. Real Firefly/OpenAI/Google output carries the AI source-type, so it is unaffected (verified: chatgpt-1.png→OpenAI, firefly-1.png→Adobe Firefly still attribute). `_attribute_platform` defaults `is_ai=True` so the mapping stays unit-testable in isolation. Add capture-camera tokens to `_DEVICE_C2PA_PLATFORM`, editing-app/AI-device signer tokens to `_SIGNER_C2PA_PLATFORM`, generator/issuer platforms to `_ISSUER_PLATFORM`, not inline. For non-PNG containers (JPEG/WebP/AVIF/HEIF/JXL) the caBX parser returns nothing, so issuer (`_issuers_in`) and generator (`_ai_tools_in`, reusing `C2PA_AI_TOOLS`) are recovered by binary-scanning the first MB. EXIF `Software` / `Make` / `Artist` / `ImageDescription` and XMP `CreatorTool` generator tags are read by `metadata.exif_generator` (PIL+piexif for any format PIL opens incl. AVIF, plus a container-agnostic XMP raw-byte scan that also covers HEIF/JXL), matched against `AI_GENERATOR_TOKENS` so ordinary editors (plain "Adobe Photoshop") and real-camera `Make` ("Apple"/"Canon") are not flagged. **Ideogram tags its output with EXIF `Make="Ideogram AI"`** (verified on a real download 2026-05-24) — that's why `Make` is read. **Integrity-clash detection** (`_integrity_clashes`, surfaced as `ProvenanceReport.integrity_clashes`, printed in red by `identify` and serialized to `--json`): contradictions between independent generator stamps are a laundering/spoofing tell. Two rules: (1) two or more distinct AI-origin vendors named by independent signals (e.g. C2PA OpenAI + EXIF `Make="Ideogram AI"`), and (2) a camera-capture C2PA device (`_DEVICE_C2PA_PLATFORM`) coexisting with any AI-generation marker. Vendor normalization is `_vendor_of` over `_AI_VENDOR_TOKENS` (so a C2PA "Google (Gemini)" issuer and a SynthID-Google proxy agree, while different vendors clash). **High-precision by design:** only hard generator stamps feed it (C2PA-issuer when source is AI, SynthID, EXIF/XMP generator, IPTC `AISystemUsed`, xAI, AIGC); the fuzzy visible sparkle and the open invisible watermark are **excluded** (the latter can be a by-product of our own SDXL removal pass). The c2pa vendor is classified from the issuer attribution / generator, NOT the resolved `platform` (a camera label like "Google Pixel" would mis-normalize to "Google"). All real single-origin fixtures (chatgpt/firefly/doubao/grok/mj) verified to produce **zero** clashes (false-positive guard in `test_identify.py::TestRealSamplesHaveNoClash`). - `watermark_registry.py` — **single catalog of known visible watermarks**, the unified "find known marks in their usual places, recognize, remove" entry. **Reverse-alpha only by policy**: a mark is listed only once a real alpha map has been captured for it, and removal inverts that map (`original = (wm - a*logo)/(1-a)`, exact recovery) — no inpaint/heuristic removal here (arbitrary-region inpainting lives in `region_eraser`/`erase`). Each `KnownMark` ties a key to {usual `location`, `in_auto` flag, `recovery` (="reverse-alpha"), a `detect` adapter → uniform `MarkDetection`, a `remove` adapter}. Entries today: `gemini` (bottom-right sparkle) and `doubao` (bottom-right "豆包AI生成"). `detect_marks` scans all; `best_auto_mark` picks the highest-confidence detection. **Cross-engine confidences aren't directly comparable**, so the gemini adapter applies the corpus-validated 0.5 sparkle threshold (`_GEMINI_AUTO_MIN_CONF`) for its `detected` flag — otherwise the gemini engine's loose internal threshold weakly fires (~0.36) on the Doubao text and hijacks `auto`. `cli.cmd_visible` is registry-driven: `--mark auto` → `best_auto_mark`, `--mark ` → that mark; `--mark` choices come from `mark_keys()`. `_doubao_remove` applies reverse-alpha only when the mark is detected AND `reverse_alpha_available` (resolution in the alpha band); outside that, removal is **skipped** (not inpainted). Add a new visible mark = one `KnownMark` entry + its engine (with a captured alpha map); do not re-add per-mark `if` branches in the CLI. - `gemini_engine.py` — visible Gemini-sparkle remover/detector (cv2/numpy, no GPU). `detect_sparkle_confidence(path)` is the file-level entry point used by `identify.py`. @@ -63,7 +63,7 @@ Who embeds what, and whether it is locally detectable (so we know which gaps are - **C2PA / IPTC (covered by the issuer/marker scan):** OpenAI, Google, Adobe Firefly, Microsoft (Designer + **Bing Image Creator** — collected 2026-05-24; Bing now runs Microsoft's own **MAI-Image** model, signs C2PA as "Microsoft", NOT OpenAI/DALL-E), and **Stability AI** (collected from Brand Studio / DreamStudio successor; signs C2PA as "Stability AI Ltd", no SynthID, no imwatermark on its current Stable Image model — issuer added to `C2PA_ISSUERS`). Still unsampled: Canva (its downloads are re-encoded design *exports* that strip C2PA, so a Canva "positive" is inconclusive — skipped), Getty, Shutterstock. Midjourney embeds NO C2PA and no invisible watermark (our `mj-*` sample carried only the IPTC tag). **Samsung Galaxy AI** (Generative Edit / Sketch to Image / Portrait Studio on Galaxy S23 FE / S24 / S25, One UI 7+) signs C2PA as "Samsung Galaxy" with the standard `trainedAlgorithmicMedia` source type AND a proprietary `genAIType` marker; verified on real signed files 2026-05-29 (the standard scan catches the source type; `genAIType` additionally catches a Galaxy S24 file that omits it). **ASUS Gallery** also signs edited photos as C2PA (`com.asus.gallery`) but with no AI source type — a signer, not an AI marker. **Black Forest Labs (FLUX)** API output signs C2PA: `claim_generator_info "Black Forest Labs API"` + a `c2pa.ai_generated_content` assertion + `trainedAlgorithmicMedia` (issuer `b"Black Forest Labs"` added to `C2PA_ISSUERS`, platform "Black Forest Labs (FLUX)"). **ByteDance Volcano Engine (Volcengine)** — the cloud behind Doubao / Jimeng — signs its AI image output with a cert from `certificate_center@volcengine.com` + `trainedAlgorithmicMedia` (issuer `b"volcengine"` → "ByteDance (Volcano Engine)", platform "ByteDance (Doubao / Jimeng / Volcano Engine)"); note this is the C2PA-signed surface, distinct from the XMP/PNG TC260 `AIGC` label Doubao also uses. All three verified on real signed files 2026-05-29. - **EXIF/XMP generator tag (caught by `exif_generator`):** **Ideogram** writes EXIF `Make="Ideogram AI"` (collected 2026-05-24 — no C2PA, no SynthID, no imwatermark; the Make tag is the only signal). - **xAI / Grok — its own EXIF signature scheme, NOT C2PA (DETECTED by `metadata.xai_signature`, built 2026-05-26).** Grok JPEG downloads (Aurora model) carry **no C2PA, no XMP, no SynthID, no IPTC** — only EXIF `Artist` = a UUID and EXIF `ImageDescription` = `Signature: ` (a crypto signature, unverifiable locally without xAI's public key). This empirically kills the earlier unverified "xAI signs C2PA as xAI" lead — xAI is not even a C2PA member. `exif_generator` misses it (neither field holds an `AI_GENERATOR_TOKENS` token), so a dedicated detector `xai_signature(path)` matches the pair (`ImageDescription ~ ^Signature: [A-Za-z0-9+/=]{64,}` AND UUID `Artist`); wired into `has_ai_metadata`, `get_ai_metadata` (key `xai_signature`), and `identify` (signal `xai_signature`, platform "xAI (Grok / Aurora)"). **Format confirmed stable across n=3 genuine generations:** exactly three EXIF tags (`Artist`, `ExifOffset`, `ImageDescription`), `Signature:` prefix constant, base64 payload 300-1004 chars. Two capture facts: (a) the `Artist` UUID **equals the public image id** in the asset URL (`https://imagine-public.x.ai/imagine-public/images/.jpg`), so it is NOT a private per-user secret — only the `Signature` blob is; (b) the Grok web-UI image is a re-encoded **WebP with no signature** — the EXIF survives only in the *original* JPEG (download button or that public tokenless URL), which is why screenshots / re-encodes are metadata-stripped. A real fixture `data/samples/grok-1.jpg` plus **synthetic** JPEG fixtures (fake UUID + fake `Signature:` blob) cover the detector; never add a real Grok image carrying private content (the repo is public). **Stripped on removal too:** `remove_ai_metadata` now calls `_scrub_ai_exif` on the JPEG EXIF, which deletes the xAI Signature+UUID-Artist pair **and** any `Software`/`Make`/`Artist`/`ImageDescription` tag holding an `AI_GENERATOR_TOKENS` token (so Ideogram's `Make="Ideogram AI"` is scrubbed too), while keeping genuine camera/editor EXIF. The shared `_is_xai_signature_pair` helper (module-level compiled regexes) is the single source of truth for the pattern, used by both `xai_signature` and `_scrub_ai_exif`. (AVIF/HEIF/JXL still strip only C2PA boxes via `isobmff`, not EXIF — unchanged.) -- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). **Doubao** (ByteDance) uses it (verified on the real #13 sample 2026-05-25; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **two serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `` block (container-agnostic raw-byte scan, any JSON object accepted) **and** a raw-JSON PNG `AIGC` tEXt chunk — Doubao also writes the label this way, with no namespaced marker at all (confirmed on the corpus 2026-05-28, `ContentProducer="doubao"`). The PNG-chunk path is gated on at least one TC260 field (`_TC260_FIELDS`) so a generic `AIGC` key cannot false-positive. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated). +- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). **Doubao** (ByteDance) uses it (verified on the real #13 sample 2026-05-25; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **three serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `{...}` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on the corpus 2026-05-30) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on the corpus 2026-05-28, `ContentProducer="doubao"`), **and** a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on the corpus 2026-05-30, `ContentProducer="001191440300708461136T1308L"`). Both generic forms (the PNG chunk and the bare `{"AIGC":...}` object) are gated on at least one TC260 field (`_TC260_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated). - **HuggingFace-hosted job (caught by `metadata.huggingface_job`, surfaced by `identify` as the `hf_job` signal, MEDIUM confidence):** HuggingFace Jobs / Spaces stamp generated PNGs with an `hf-job-id` tEXt chunk holding the job UUID (3 on the corpus 2026-05-28, no other signal). It marks the *hosting job*, not a model — most commonly diffusion output — so it lifts an Unknown verdict to a tentative AI via `hf_only` (parallel to the visible sparkle) but never overrides a hard metadata signal; `_HF_JOB_CAVEAT` states the limit (job, not model; not proof of AI pixels). Stripped on removal (the PNG save whitelist keeps only `STANDARD_METADATA_KEYS`, so `hf-job-id` and the `AIGC` chunk are both dropped). The exact writer is not authoritatively documented (HF Jobs are generic GPU jobs), hence medium not high. - **No detectable signal on download (correctly reported `unknown`):** **Recraft** (PNG export is a re-encoded design export — strips everything), **Krea hosting FLUX 2** (no imwatermark despite FLUX — the host omits the encoder, same as Stability's hosted SDXL), and Midjourney (embeds nothing). Lesson: the imwatermark detector only fires on *pristine* output from a pipeline that runs the encoder (diffusers default, official BFL), not from re-hosts (Krea/Stability) or re-encoded exports (Recraft/Canva). - **Invisible but NOT locally detectable (proprietary, API/oracle only — same wall as SynthID):** Amazon Titan Image Generator + Nova Canvas (Bedrock `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), NVIDIA Cosmos (SynthID video). No local detector possible; treat like SynthID. diff --git a/README.md b/README.md index 8080038..6745db8 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ If this tool saves you time, consider [sponsoring its development](https://githu - **Smart Face Protection** — automatic extraction and blending of human faces to prevent AI distortion - **Batch processing** — process entire directories - **Detection** — three-stage NCC watermark detection with confidence scoring -- **Provenance detection (`identify`)** — aggregate C2PA issuer, the C2PA soft-binding forensic-watermark vendor (Adobe TrustMark, Digimarc, Imatag, ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, embedded SD/ComfyUI params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the China TC260 AIGC label (XMP or PNG chunk), the HuggingFace `hf-job-id` job marker, the SynthID metadata proxy, the visible sparkle, the open SD/SDXL/FLUX invisible watermark, and (with the `trustmark` extra) the open Adobe TrustMark watermark into one origin-platform + watermark-inventory verdict (`--json` for machine output) +- **Provenance detection (`identify`)** — aggregate C2PA issuer, the C2PA soft-binding forensic-watermark vendor (Adobe TrustMark, Digimarc, Imatag, ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, embedded SD/ComfyUI params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the China TC260 AIGC label (XMP, PNG chunk, or EXIF), the HuggingFace `hf-job-id` job marker, the SynthID metadata proxy, the visible sparkle, the open SD/SDXL/FLUX invisible watermark, and (with the `trustmark` extra) the open Adobe TrustMark watermark into one origin-platform + watermark-inventory verdict (`--json` for machine output) ## Examples @@ -49,7 +49,7 @@ If this tool saves you time, consider [sponsoring its development](https://githu | **xAI Grok (Aurora)** | — | — | ✅ EXIF signature scheme (no C2PA): `Signature:` blob + UUID `Artist` | Detected (`identify`); metadata strip | | **Midjourney** | — | — | ✅ EXIF + XMP (prompt, model, seed) | Metadata strip | | **Meta AI** | — | — | ✅ IPTC "Made with AI" (digitalSourceType) | Metadata strip (removes the label) | -| **Doubao** (ByteDance) / China AIGC generators | ✅ "豆包AI生成" text strip (bottom-right) | — | ✅ TC260 AIGC label (`` XMP **or** `AIGC` PNG chunk) **+ C2PA** signed by ByteDance Volcano Engine (`volcengine`) | Exact reverse-alpha (captured α map): pixel-exact at native width, NCC-aligned at other resolutions, + metadata strip | +| **Doubao** (ByteDance) / China AIGC generators | ✅ "豆包AI生成" text strip (bottom-right) | — | ✅ TC260 AIGC label (`` XMP, `AIGC` PNG chunk, or EXIF JSON) **+ C2PA** signed by ByteDance Volcano Engine (`volcengine`) | Exact reverse-alpha (captured α map): pixel-exact at native width, NCC-aligned at other resolutions, + metadata strip | | **Samsung Galaxy AI** (Generative Edit, Sketch to Image, ...) | — | — | ✅ C2PA (signer "Samsung Galaxy") + `trainedAlgorithmicMedia` / proprietary `genAIType` marker | Detected (`identify`) + metadata strip | | **Black Forest Labs** (FLUX API) | — | — | ✅ C2PA (`Black Forest Labs API` + `c2pa.ai_generated_content` + `trainedAlgorithmicMedia`) | Metadata strip | | **StableSignature** (Meta) | — | ✅ In-model watermark | — | Diffusion regeneration | @@ -57,7 +57,7 @@ If this tool saves you time, consider [sponsoring its development](https://githu > Visible overlays are used by Google Gemini / Nano Banana (sparkle logo) and by Doubao / China AIGC generators (the mandated "...AI生成" corner text). Both are removed deterministically on CPU. Other services rely on invisible watermarks and/or metadata; our diffusion-based regeneration works against any invisible watermark in pixel or frequency domain. For a visible mark from any other source (any position, any colour), use the universal `erase --region` command. -> **Detection:** `remove-ai-watermarks identify ` reports the origin platform and watermark inventory for all the signals above — C2PA issuer, the C2PA soft-binding forensic-watermark vendor (TrustMark / Digimarc / Imatag / ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, the China TC260 AIGC label (XMP or PNG chunk), the HuggingFace `hf-job-id` job marker, embedded generation params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the SynthID metadata proxy, the visible sparkle, and (with the `[detect]` / `[trustmark]` extras) the open SD/SDXL/FLUX and Adobe TrustMark invisible watermarks. SynthID and the proprietary soft-binding watermarks (Digimarc etc.) have no local decoder, so they are reported by metadata proxy / vendor name only. +> **Detection:** `remove-ai-watermarks identify ` reports the origin platform and watermark inventory for all the signals above — C2PA issuer, the C2PA soft-binding forensic-watermark vendor (TrustMark / Digimarc / Imatag / ...), IPTC "Made with AI" plus the IPTC 2025.1 `AISystemUsed` field, the China TC260 AIGC label (XMP, PNG chunk, or EXIF), the HuggingFace `hf-job-id` job marker, embedded generation params, EXIF/XMP generator tags, the xAI/Grok EXIF signature, the SynthID metadata proxy, the visible sparkle, and (with the `[detect]` / `[trustmark]` extras) the open SD/SDXL/FLUX and Adobe TrustMark invisible watermarks. SynthID and the proprietary soft-binding watermarks (Digimarc etc.) have no local decoder, so they are reported by metadata proxy / vendor name only. ## How it works diff --git a/scripts/corpus_gap_scan.py b/scripts/corpus_gap_scan.py new file mode 100644 index 0000000..7e96097 --- /dev/null +++ b/scripts/corpus_gap_scan.py @@ -0,0 +1,215 @@ +"""Audit a local image corpus against the library's own ``identify`` detector. + +Two jobs in one pass: + +1. **Report** -- run ``identify`` over every image and write one CSV row per file + (verdict, platform, confidence, watermarks, signals, integrity clashes). +2. **Gap audit** -- for every ``unknown``-verdict file, scan only its *metadata + region* (PNG text/eXIf chunks, JPEG APPn segments before SOS, or the file + head for other containers) for known provenance markers. A marker found there + on a file the detector calls ``unknown`` is a concrete lib gap: a serialization + or generator we do not yet parse. Scanning the metadata region -- not the whole + file -- is deliberate: short tokens collide randomly inside compressed PNG + ``IDAT`` / JPEG scan data, which produced false "xAI/Flux/AIGC" hits when the + first audit naively scanned the first megabyte. + +This is how new detector gaps get found (it is what surfaced the JPEG-EXIF +``{"AIGC":{...}}`` form). Re-run after collecting a fresh corpus batch. + +Usage: + uv run python scripts/corpus_gap_scan.py --corpus data/spaces/originals + uv run python scripts/corpus_gap_scan.py --corpus data/spaces/originals \\ + --report data/spaces/detector_report.csv +""" + +from __future__ import annotations + +import csv +import logging +from collections import Counter +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from remove_ai_watermarks.identify import identify +from remove_ai_watermarks.metadata import _png_late_metadata + +log = logging.getLogger(__name__) +console = Console() + +# Distinctive, multi-byte provenance markers worth flagging when they appear in a +# file the detector calls `unknown`. Kept long enough that a random collision in a +# (non-scanned) compressed stream is implausible; the metadata-region restriction +# below is the primary guard, this list is the second. Group: C2PA/JUMBF infra, +# AI source-type / labeling schemes, and distinctive generator name strings. +MARKERS: tuple[bytes, ...] = ( + # C2PA / JUMBF infrastructure and AI source-type / labeling schemes. + b"c2pa", + b"jumbf", + b"contentauth", + b"trainedAlgorithmicMedia", + b"digitalSourceType", + b'"AIGC"', + b"", + b"TC260:AIGC", + b"tc260.org.cn", + b"AISystemUsed", + b"SynthID", + b"hf-job-id", + b"genAIType", + b"PhotoEditor_Re_Edit", + b"Signature:", + # Distinctive multi-word generator strings only. Bare single words (Luma, + # Gemini, Sora, ...) are omitted: they collide with unrelated metadata prose + # (e.g. "Luma" in Lightroom's EnhanceDenoiseLumaAmount), defeating precision. + b"Midjourney", + b"Stable Diffusion", + b"StableDiffusion", + b"ComfyUI", + b"Automatic1111", + b"DALL-E", + b"Ideogram AI", + b"Adobe Firefly", + b"Black Forest", + b"volcengine", + b"Doubao", + b"\xe8\xb1\x86\xe5\x8c\x85", + b"Nano Banana", + b"Stability AI", + b"Samsung Galaxy", +) + + +def _metadata_region(path: Path) -> bytes: + """Return only the bytes where provenance metadata can live, never the + compressed pixel stream (which produces random short-token collisions).""" + try: + head = path.read_bytes() + except OSError: + return b"" + if head[:8] == b"\x89PNG\r\n\x1a\n": + # All ancillary metadata chunks (window=0), via the library's own walker. + return _png_late_metadata(path, 0) + if head[:2] == b"\xff\xd8": # JPEG: APPn segments up to Start-Of-Scan + out = bytearray() + p = 2 + n = len(head) + while p + 4 <= n and head[p] == 0xFF: + marker = head[p + 1] + if marker == 0xDA: # SOS -> compressed scan data follows + break + seg_len = (head[p + 2] << 8) | head[p + 3] + out += head[p + 4 : p + 2 + seg_len] + p += 2 + seg_len + return bytes(out) + return head[:65536] # webp/avif/heif/jxl: metadata sits near the head + + +def _row(rep) -> dict[str, str]: # noqa: ANN001 (ProvenanceReport) + return { + "path": "", # filled by caller (relative) + "is_ai": str(rep.is_ai_generated), + "platform": rep.platform or "", + "confidence": rep.confidence, + "watermarks": "|".join(rep.watermarks), + "signals": "|".join(s.name for s in rep.signals), + "integrity_clashes": "|".join(rep.integrity_clashes), + } + + +@click.command() +@click.option( + "--corpus", + type=click.Path(exists=True, file_okay=False, path_type=Path), + default=Path("data/spaces/originals"), + show_default=True, + help="Directory of images to scan (recursively).", +) +@click.option( + "--report", + type=click.Path(path_type=Path), + default=None, + help="Write the per-file CSV here (default: /../detector_report.csv).", +) +@click.option("--limit", type=int, default=0, help="Scan at most N files (0 = all).") +def main(corpus: Path, report: Path | None, limit: int) -> None: + logging.basicConfig(level=logging.WARNING, format="%(message)s") + report = report or corpus.parent / "detector_report.csv" + + files = sorted(p for p in corpus.rglob("*") if p.is_file()) + if limit: + files = files[:limit] + console.print(f"Scanning [bold]{len(files)}[/bold] files under {corpus} ...") + + verdicts: Counter[str] = Counter() + platforms: Counter[str] = Counter() + gap_tokens: Counter[str] = Counter() + gaps: list[tuple[str, list[str]]] = [] + rows: list[dict[str, str]] = [] + errors = 0 + + with click.progressbar(files, label="identify") as bar: + for p in bar: + rel = str(p.relative_to(corpus)) + try: + rep = identify(p) + except Exception as exc: + log.warning("identify failed on %s: %s", rel, exc) + errors += 1 + continue + row = _row(rep) + row["path"] = rel + rows.append(row) + if rep.is_ai_generated: + verdicts["ai"] += 1 + platforms[rep.platform or "?"] += 1 + continue + verdicts["unknown"] += 1 + # A gap candidate is a file identify is *blind* to (no signal at all) + # yet whose metadata carries a known marker. A file that produced a + # signal but no AI verdict (e.g. an ASUS Gallery C2PA signer, which we + # attribute but do not call AI) is handled correctly -- not a gap. + if rep.signals: + continue + region = _metadata_region(p) + hits = sorted({m.decode("latin-1", "replace") for m in MARKERS if m in region}) + if hits: + gaps.append((rel, hits)) + gap_tokens.update(hits) + + with report.open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=["path", "is_ai", "platform", "confidence", "watermarks", "signals", "integrity_clashes"], + ) + writer.writeheader() + writer.writerows(rows) + console.print(f"\nWrote [bold]{len(rows)}[/bold] rows -> {report}") + + console.print(f"\n[bold]Verdicts:[/bold] AI {verdicts['ai']} | unknown {verdicts['unknown']} | errors {errors}") + plat = Table(title="AI platforms", show_header=False) + for name, n in platforms.most_common(): + plat.add_row(str(n), name) + console.print(plat) + + if gaps: + console.print( + f"\n[bold red]Gap candidates[/bold red]: {len(gaps)} unknown files carry a known " + f"marker in their metadata region (potential undetected serialization/generator):" + ) + tok = Table(title="markers seen in unknown files") + tok.add_column("count", justify="right") + tok.add_column("marker") + for name, n in gap_tokens.most_common(): + tok.add_row(str(n), name) + console.print(tok) + for rel, hits in gaps: + console.print(f" {rel} -> {', '.join(hits)}") + else: + console.print("\n[green]No gap candidates: every unknown file is metadata-free.[/green]") + + +if __name__ == "__main__": + main() diff --git a/src/remove_ai_watermarks/metadata.py b/src/remove_ai_watermarks/metadata.py index 83ffe16..b6379b6 100644 --- a/src/remove_ai_watermarks/metadata.py +++ b/src/remove_ai_watermarks/metadata.py @@ -11,6 +11,7 @@ from __future__ import annotations import contextlib import logging import re +import struct from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -169,15 +170,58 @@ def _is_ai_key(key: str) -> bool: return any(kw in key_lower for kw in AI_KEYWORDS) +# PNG ancillary chunks that can carry provenance metadata (XMP, EXIF, text). +# Never IDAT -- that is the compressed pixel stream. +_PNG_META_CHUNKS: frozenset[bytes] = frozenset({b"tEXt", b"iTXt", b"zTXt", b"eXIf", b"iCCP"}) + + +def _png_late_metadata(image_path: Path, window: int) -> bytes: + """Payloads of PNG metadata chunks that start *beyond* the first ``window`` + bytes, found by seeking past the (large) ``IDAT`` pixel stream. + + A PNG encoder may append the XMP/EXIF packet after the image data, so a + fixed first-``size`` read misses it (e.g. a TC260 AIGC label in an XMP + ``iTXt`` chunk at ~2.7 MB). This is the PNG analogue of the ISOBMFF + late-box scan in :func:`scan_head`. Returns only chunks past ``window`` so + bytes already in the head are not duplicated; empty when there are none. + """ + out = bytearray() + try: + with open(image_path, "rb") as f: + if f.read(8) != b"\x89PNG\r\n\x1a\n": + return b"" + pos = 8 + while True: + f.seek(pos) + header = f.read(8) + if len(header) < 8: + break + (length,) = struct.unpack(">I", header[:4]) + chunk_type = header[4:8] + if chunk_type == b"IEND": + break + data_start = pos + 8 + if chunk_type in _PNG_META_CHUNKS and data_start >= window: + f.seek(data_start) + out += f.read(length) + pos = data_start + length + 4 # data + CRC + except OSError as exc: + logger.debug("PNG late-metadata scan failed on %s: %s", image_path, exc) + return b"" + return bytes(out) + + def scan_head(image_path: Path, size: int = 1024 * 1024) -> bytes: - """First ``size`` bytes of the file, plus -- for ISOBMFF containers -- the - payloads of any provenance (``uuid`` / ``jumb``) boxes found beyond that - window by seeking past large boxes like ``mdat``. + """First ``size`` bytes of the file, plus the payloads of any provenance + metadata found beyond that window: ISOBMFF ``uuid`` / ``jumb`` boxes (seeking + past large boxes like ``mdat``) and PNG ``tEXt`` / ``iTXt`` / ``eXIf`` chunks + (seeking past ``IDAT``). This is the shared input for every C2PA / AIGC / IPTC byte scan. The - ISOBMFF extension catches a manifest placed AFTER the media data in a - streaming / non-faststart MP4, which a fixed first-MB read would miss. For - non-ISOBMFF inputs it is exactly ``f.read(size)`` -- behavior-neutral. + extensions catch a manifest or XMP packet placed AFTER the media data -- a + non-faststart MP4 manifest, or a PNG XMP packet appended after the pixels -- + which a fixed first-MB read would miss. For other inputs, and for files that + fit within ``size``, it is exactly ``f.read(size)`` -- behavior-neutral. """ with open(image_path, "rb") as f: head = f.read(size) @@ -188,6 +232,10 @@ def scan_head(image_path: Path, size: int = 1024 * 1024) -> bytes: region = isobmff.scan_c2pa_region(image_path) if region: head += region + elif head[:8] == b"\x89PNG\r\n\x1a\n" and len(head) == size: + # len(head) == size means the file is at least `size` bytes, so metadata + # chunks may lie beyond the window; otherwise the whole PNG is in `head`. + head += _png_late_metadata(image_path, size) return head @@ -252,17 +300,21 @@ def has_ai_metadata(image_path: Path) -> bool: def aigc_label(image_path: Path) -> dict[str, str] | None: """Parse a China TC260 AI-labeling block, if present. - Two serializations are recognized: + Three serializations are recognized: - a PNG ``tEXt``/``iTXt`` chunk keyed ``AIGC`` carrying the raw JSON object - (as written by Doubao / ByteDance), read via PIL; and + (as written by Doubao / ByteDance), read via PIL; - an XMP ``{...}`` block (HTML-entity encoded text), - found by a container-agnostic raw-byte scan (PNG/JPEG/WebP alike). + found by a container-agnostic raw-byte scan (PNG/JPEG/WebP alike); and + - a raw-JSON ``{"AIGC":{...}}`` block with no namespace, as embedded in JPEG + EXIF (UserComment) by some China-served generators, brace-matched from the + scan head. Returns the decoded JSON (e.g. ``{"Label": "1", "ContentProducer": ...}``) - or None. The PNG-chunk key ``AIGC`` is generic, so a JSON object there is - accepted only if it carries at least one known TC260 field (``_TC260_FIELDS``); - the namespaced XMP element is unambiguous, so any JSON object is accepted. + or None. The generic forms (the PNG-chunk key ``AIGC`` and the bare + ``{"AIGC":...}`` object) are accepted only if they carry at least one known + TC260 field (``_TC260_FIELDS``); the namespaced XMP element is unambiguous, + so any JSON object is accepted. """ import html import json @@ -293,12 +345,35 @@ def aigc_label(image_path: Path) -> dict[str, str] | None: if isinstance(value, str) and (result := _parse(value, require_tc260_field=True)): return result - # XMP {...} block (namespaced element, unambiguous). + # XMP TC260:AIGC, namespaced (unambiguous) in either serialization RDF allows: + # an element {...} or an attribute TC260:AIGC="{...}" + # (the attribute form is what PicWish writes). Both are HTML-entity encoded. data = scan_head(image_path) - match = re.search(rb"(.*?)", data, re.DOTALL) - if not match: - return None - return _parse(html.unescape(match.group(1).decode("utf-8", "replace")), require_tc260_field=False) + match = re.search( + rb'(.*?)|TC260:AIGC\s*=\s*"(.*?)"', + data, + re.DOTALL, + ) + if match: + body = match.group(1) if match.group(1) is not None else match.group(2) + return _parse(html.unescape(body.decode("utf-8", "replace")), require_tc260_field=False) + + # Raw-JSON {"AIGC":{...}} block (no namespace), as written into JPEG EXIF + # (UserComment) by some China-served generators -- the PNG-chunk and XMP + # paths above both miss it. The bytes pre-check keeps the common (no-AIGC) + # path off the full-buffer decode; raw_decode then brace-matches the inner + # object (respecting nested braces / quoted strings) and `_parse` applies the + # same dict coercion + TC260-field gate as the generic PNG-chunk path. + if b'"AIGC"' in data: + text = data.decode("latin-1") + brace = text.find("{", text.find('"AIGC"') + len('"AIGC"')) + if brace != -1: + try: + _, end = json.JSONDecoder().raw_decode(text, brace) + except ValueError: + return None + return _parse(text[brace:end], require_tc260_field=True) + return None def huggingface_job(image_path: Path) -> str | None: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 436ab3b..d04279d 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -705,6 +705,82 @@ class TestAIGCLabel: assert aigc_label(out) is None assert not has_ai_metadata(out) + def _aigc_exif_jpeg(self, tmp_path: Path, producer: str = "001191440300708461136T1308L") -> Path: + """Some China-served generators embed the raw-JSON ``{"AIGC":{...}}`` + block in JPEG EXIF (UserComment) -- no PNG chunk, no namespaced XMP.""" + import json + + import piexif + + p = tmp_path / "aigc_exif.jpg" + Image.new("RGB", (32, 32)).save(p) + payload = json.dumps({"AIGC": {"Label": "1", "ContentProducer": producer, "ProduceID": "abc123"}}) + exif = {"Exif": {piexif.ExifIFD.UserComment: payload.encode("ascii")}} + piexif.insert(piexif.dump(exif), str(p)) + return p + + def test_parses_raw_json_exif_form(self, tmp_path: Path): + from remove_ai_watermarks.metadata import aigc_label + + info = aigc_label(self._aigc_exif_jpeg(tmp_path)) + assert info is not None + assert info["Label"] == "1" + assert info["ContentProducer"] == "001191440300708461136T1308L" + + def test_has_ai_metadata_detects_raw_json_exif_form(self, tmp_path: Path): + assert has_ai_metadata(self._aigc_exif_jpeg(tmp_path)) + + def test_raw_json_without_tc260_field_ignored(self, tmp_path: Path): + """A bare ``{"AIGC":{...}}`` object with no TC260 field must not fire.""" + import json + + import piexif + + from remove_ai_watermarks.metadata import aigc_label + + p = tmp_path / "unrelated.jpg" + Image.new("RGB", (32, 32)).save(p) + payload = json.dumps({"AIGC": {"unrelated": "value"}}) + exif = {"Exif": {piexif.ExifIFD.UserComment: payload.encode("ascii")}} + piexif.insert(piexif.dump(exif), str(p)) + assert aigc_label(p) is None + + def _aigc_attr_png(self, tmp_path: Path, producer: str = "picwish") -> Path: + """PicWish writes the TC260 label as an XMP *attribute* + (``TC260:AIGC="{...}"``), not the nested element form.""" + p = tmp_path / "picwish.png" + Image.new("RGB", (32, 32)).save(p) + xmp = ( + '' + ) + with open(p, "ab") as f: + f.write(xmp.encode()) + return p + + def test_parses_xmp_attribute_form(self, tmp_path: Path): + from remove_ai_watermarks.metadata import aigc_label + + info = aigc_label(self._aigc_attr_png(tmp_path)) + assert info is not None + assert info["ContentProducer"] == "picwish" + + def test_scan_head_collects_png_metadata_past_window(self, tmp_path: Path): + """A PNG metadata chunk beyond the read window is still reachable -- the + regression for a TC260 XMP packet appended after a large IDAT.""" + import json + + from remove_ai_watermarks.metadata import _png_late_metadata, scan_head + + p = tmp_path / "late.png" + pnginfo = PngInfo() + pnginfo.add_text("AIGC", json.dumps({"Label": "1", "ContentProducer": "doubao"})) + Image.new("RGB", (16, 16)).save(p, pnginfo=pnginfo) + # window = 8 (just the signature) makes the text chunk "late". + assert b"ContentProducer" in _png_late_metadata(p, 8) + assert b"ContentProducer" in scan_head(p, 8) + class TestHuggingFaceJob: """HuggingFace-hosted job marker (``hf-job-id`` PNG text chunk)."""