diff --git a/data/README.md b/data/README.md index 4a4667a..a76aadd 100644 --- a/data/README.md +++ b/data/README.md @@ -6,6 +6,7 @@ Tracked data is organized by purpose: data/ fixtures/ provenance/ Real format and provenance fixtures used by tests + (source records live in fixtures/README.md) calibration/ / Minimal controlled inputs needed to rebuild detector assets synthid/ @@ -31,7 +32,9 @@ data/ commands, hashes, and oracle verdicts instead of committing another corpus copy. A small curated before-and-after example may live in `docs/images/` when it is part of the public documentation. -6. Runtime detector assets belong in `src/remove_ai_watermarks/assets/`. +6. Keep third-party fixture license notices in `licenses/`, outside directories + that tests enumerate as media inputs. +7. Runtime detector assets belong in `src/remove_ai_watermarks/assets/`. Unregistered research candidates belong in `scripts/assets/visible-mark-candidates/` so they are not shipped in the wheel. diff --git a/data/fixtures/README.md b/data/fixtures/README.md new file mode 100644 index 0000000..c71559f --- /dev/null +++ b/data/fixtures/README.md @@ -0,0 +1,15 @@ +# Provenance fixtures + +These files exercise real container, metadata, and pixel-provenance formats. +They are test inputs, not evaluation corpora. + +## Adobe TrustMark + +`adobe-trustmark-p.png` is Adobe's official `images/ghost_P.png` Variant P +example from [`adobe/trustmark`](https://github.com/adobe/trustmark) at commit +`0ed40cbe8188f664fd9cbbeacd969807de27440a`. + +- Source: `https://raw.githubusercontent.com/adobe/trustmark/0ed40cbe8188f664fd9cbbeacd969807de27440a/images/ghost_P.png` +- SHA-256: `e58c5825ed7e5d9fb04710ea541b61bd55879cad65554c0d46260aa24b3d0755` +- Expected signal: TrustMark Variant P, schema 1 +- License: MIT, reproduced in `../licenses/adobe-trustmark-MIT.txt` diff --git a/data/fixtures/provenance/adobe-trustmark-p.png b/data/fixtures/provenance/adobe-trustmark-p.png new file mode 100644 index 0000000..8aabf5f Binary files /dev/null and b/data/fixtures/provenance/adobe-trustmark-p.png differ diff --git a/data/licenses/adobe-trustmark-MIT.txt b/data/licenses/adobe-trustmark-MIT.txt new file mode 100644 index 0000000..aa449b5 --- /dev/null +++ b/data/licenses/adobe-trustmark-MIT.txt @@ -0,0 +1,25 @@ +Copyright 2023 Adobe +All Rights Reserved. + +NOTICE: Adobe permits you to use, modify, and distribute this file in +accordance with the terms of the license agreement accompanying it. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/code-provenance.md b/docs/code-provenance.md index 0d7e98a..d26ba2c 100644 --- a/docs/code-provenance.md +++ b/docs/code-provenance.md @@ -7,3 +7,10 @@ This page records notices required by source dependencies and licensed derivativ - The DWT-DCT implementation derives from ShieldMnt's [`invisible-watermark`](https://github.com/ShieldMnt/invisible-watermark), licensed under MIT. Its notice ships in `src/remove_ai_watermarks/licenses/invisible-watermark-MIT.txt`. + +## Licensed test fixtures + +- `data/fixtures/provenance/adobe-trustmark-p.png` is Adobe's official + TrustMark Variant P example, licensed under MIT. Its source commit, digest, + and reproduced license are recorded beside the fixture in + `data/fixtures/README.md` and `data/licenses/adobe-trustmark-MIT.txt`. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 4d48ace..2596ea9 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -344,8 +344,11 @@ Diffusion, SDXL, and FLUX workflows. That decoder is sensitive to the carrier and transformations. A negative result is not a universal negative. The `trustmark` extra adds Adobe TrustMark decoding. The implementation retains -an additional JPEG re-encode gate because isolated decoder hits can otherwise -be content noise. +an additional JPEG re-encode gate and requires the binary payload and schema to +remain identical because isolated decoder hits can otherwise be content noise. +It accepts Variant P schemas 0-2. Variant Q requires a different model, and +schema 3 is rejected at the measured precision threshold. The calibration +history is in [module internals](module-internals.md#metadata-and-provenance). External AI versus real image classifiers are out of scope. The project identifies concrete local provenance signals instead of shipping a generic diff --git a/docs/module-internals.md b/docs/module-internals.md index 4ceb242..69c7784 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -564,7 +564,13 @@ human-made verdict. `ai_source_kind` distinguishes fully generated content from AI-enhanced composites when the source metadata provides that distinction. TrustMark is reported as a watermark signal but does not by itself assert AI -origin because it can also protect human-authored content. +origin because it can also protect human-authored content. The decoder requests +binary mode, matching Adobe's Durable Content Credentials example, then requires +the same payload and schema after a quality-95 JPEG round-trip. Only Variant P +schemas 0-2 count as positives. Schema 3 is below the precision threshold: all +38 measured historical false-positive candidates used it, and six retained the +same false payload after re-encoding. The official Adobe Variant P schema-1 +fixture in `data/fixtures/provenance/` is the positive regression control. Regression coverage: diff --git a/docs/supported-signals.md b/docs/supported-signals.md index 752fb40..dd0a85a 100644 --- a/docs/supported-signals.md +++ b/docs/supported-signals.md @@ -80,7 +80,9 @@ The inspection and stripping code handles signals in these groups: - Samsung AI editing markers; - Hugging Face job metadata; - open Stable Diffusion style DWT-DCT watermarks with the `detect` extra; -- Adobe TrustMark with the `trustmark` extra. +- Adobe TrustMark Variant P schemas 0-2 with the `trustmark` extra. Variant Q + needs a different model, while schema 3 is deliberately rejected because it + produced persistent false positives on unrelated generators. `identify` combines detected signals into a `ProvenanceReport`. It reports unknown when evidence is absent. It never treats missing metadata as proof that @@ -154,7 +156,7 @@ not a universal clean verdict. | OpenAI image generators | None registered | Diffusion regeneration for supported invisible signals | C2PA and generator provenance | | Stable Diffusion and SDXL | None registered | Diffusion regeneration; optional open decoder | Embedded parameters and text metadata | | FLUX | None registered | Diffusion regeneration; optional open decoder | C2PA for supported sources | -| Adobe Firefly | None registered | No proprietary local decoder | C2PA; optional TrustMark decoder | +| Adobe Firefly | None registered | Optional TrustMark Variant P decoder | C2PA | | Midjourney | None registered | No registered pixel decoder | EXIF, XMP, and IPTC signals | | ByteDance generators | Doubao and Jimeng marks | No registered pixel decoder | TC260 AIGC, supported C2PA, and exact app-export AIGC disclosures | | Qwen | Qwen mark | No registered pixel decoder | TC260 AIGC | diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index d765028..d59ced1 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -450,10 +450,11 @@ class ProvenanceReport: ai_source_kind: str | None = None # True when the AI verdict rests on a metadata or embedded-invisible signal # (C2PA AI issuer / SynthID provenance, IPTC, AIGC, local gen params, EXIF/xAI, or - # an open DWT-DCT / TrustMark decode) -- as opposed to a visible mark or a - # weak medium-confidence hint (hf-job, Samsung genAIType). It is exactly the - # set of signals an invisible/diffusion scrub targets: a visible-only or - # no-signal image has it False. Equivalent to ``confidence == "high"``; + # an open DWT-DCT decode) -- as opposed to a visible mark, provenance-only + # TrustMark, or a weak medium-confidence hint (hf-job, Samsung genAIType). This + # is exactly the set of signals an invisible/diffusion scrub targets: a + # visible-only or no-signal image has it False. Equivalent to + # ``confidence == "high"``; # surfaced as a field so callers gate on intent, not on the string. ai_from_metadata: bool = False watermarks: list[str] = field(default_factory=list[str]) @@ -1431,13 +1432,15 @@ def has_invisible_target(image_path: Path) -> bool: """True when a locally-detectable invisible/metadata AI signal is present. The decision gate for the diffusion scrub (``invisible`` / ``all`` / ``batch``): - regenerating pixels removes an invisible watermark (SynthID, open DWT-DCT, - TrustMark) but degrades a real photo, so it must not run when there is nothing - to remove. Runs :func:`identify` with ``check_visible=False`` -- a visible mark - is handled by the separate visible pass and is NOT a diffusion target -- and - ``check_invisible=True`` so an open watermark counts. Returns + regenerating pixels removes an AI-specific invisible watermark (SynthID, + open DWT-DCT) but degrades a real photo, so it must not run when there is + nothing to remove. Runs :func:`identify` with ``check_visible=False`` -- a + visible mark is handled by the separate visible pass and is NOT a diffusion + target -- and ``check_invisible=True`` so an open watermark counts. Returns ``report.ai_from_metadata`` (C2PA AI issuer / SynthID provenance, IPTC, AIGC, local - gen params, EXIF/xAI, open DWT-DCT / TrustMark). + gen params, EXIF/xAI, or open DWT-DCT). TrustMark alone does not trigger the + scrub because it also protects human-authored work and therefore is not an AI + signal by itself. IMPORTANT -- this cannot prove a pixel SynthID is absent: SynthID is detectable only through its C2PA proxy, so a metadata-stripped AI image reads as no signal diff --git a/src/remove_ai_watermarks/trustmark_detector.py b/src/remove_ai_watermarks/trustmark_detector.py index a846430..375c11d 100644 --- a/src/remove_ai_watermarks/trustmark_detector.py +++ b/src/remove_ai_watermarks/trustmark_detector.py @@ -32,6 +32,9 @@ logger = logging.getLogger(__name__) # Adobe ships Variant P in production (com.adobe.trustmark.P). _MODEL_TYPE = "P" +# Schema 3 (BCH_3) is below the detector's measured precision threshold; the +# calibration history is canonical in docs/module-internals.md. +_SUPPORTED_SCHEMAS = frozenset({0, 1, 2}) # Lazily constructed singleton -- model load + first-use download is expensive. # Guarded by a lock so concurrent callers don't double-construct/double-download. _tm: Any = None @@ -57,8 +60,9 @@ def _decoder() -> Any: # JPEG quality for the false-positive durability gate (see detect_trustmark). -# Deliberately mild: a genuine TrustMark survives far harsher, while every -# observed false positive collapsed even at this quality. +# Deliberately mild: a genuine TrustMark survives far harsher. The round-trip +# still needs payload and schema validation because content-correlated false +# positives can survive this compression level. _REENCODE_QUALITY = 95 @@ -79,8 +83,8 @@ def detect_trustmark(image_path: Path) -> str | None: cannot carry Adobe's watermark, and decoded a random-bytes secret). A genuine TrustMark is a *durable* soft binding engineered to survive re-encoding (that is its entire purpose once C2PA is stripped), so we re-decode after a mild - JPEG round-trip and require the same schema both times. Every observed false - positive collapsed under this gate. + JPEG round-trip and require the same binary payload and schema both times. + Only the calibrated schemas 0-2 count as high-precision positives. """ if not is_available(): return None @@ -90,10 +94,17 @@ def detect_trustmark(image_path: Path) -> str | None: with Image.open(image_path) as img: cover = img.convert("RGB") decoder = _decoder() - _wm_secret, wm_present, wm_schema = decoder.decode(cover) + wm_secret, wm_present, wm_schema = decoder.decode(cover, "binary") if not wm_present: return None - if not _survives_reencode(decoder, cover, wm_schema): + if wm_schema not in _SUPPORTED_SCHEMAS: + logger.debug( + "TrustMark decode for %s used weak schema %s; treating as false positive", + image_path, + wm_schema, + ) + return None + if not _survives_reencode(decoder, cover, wm_secret, wm_schema): logger.debug("TrustMark decode for %s did not survive re-encode; treating as false positive", image_path) return None except Exception as exc: # model download / decode failure / unreadable image @@ -102,10 +113,8 @@ def detect_trustmark(image_path: Path) -> str | None: return f"Adobe TrustMark (variant {_MODEL_TYPE}, schema {wm_schema})" -def _survives_reencode(decoder: Any, cover: Any, schema: int) -> bool: - """True if the watermark re-decodes with the same schema after a mild JPEG - round-trip -- the durability a genuine TrustMark guarantees, which a BCH - false positive (content noise) does not.""" +def _survives_reencode(decoder: Any, cover: Any, payload: str, schema: int) -> bool: + """True if the same watermark re-decodes after a mild JPEG round-trip.""" import io from PIL import Image @@ -114,5 +123,5 @@ def _survives_reencode(decoder: Any, cover: Any, schema: int) -> bool: cover.save(buffer, "JPEG", quality=_REENCODE_QUALITY) buffer.seek(0) with Image.open(buffer) as reencoded: - _secret, present, reencoded_schema = decoder.decode(reencoded.convert("RGB")) - return bool(present) and reencoded_schema == schema + reencoded_payload, present, reencoded_schema = decoder.decode(reencoded.convert("RGB"), "binary") + return bool(present) and reencoded_schema == schema and reencoded_payload == payload diff --git a/tests/test_trustmark_detector.py b/tests/test_trustmark_detector.py index f2fec3d..9c6a106 100644 --- a/tests/test_trustmark_detector.py +++ b/tests/test_trustmark_detector.py @@ -8,15 +8,19 @@ absent/error behaviour: detect must return None, never raise. from __future__ import annotations -from typing import TYPE_CHECKING +import hashlib +from pathlib import Path import pytest from remove_ai_watermarks import trustmark_detector +from remove_ai_watermarks.identify import identify from remove_ai_watermarks.trustmark_detector import detect_trustmark, is_available -if TYPE_CHECKING: - from pathlib import Path +_OFFICIAL_FIXTURE = ( + Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance" / "adobe-trustmark-p.png" +) +_OFFICIAL_FIXTURE_SHA256 = "e58c5825ed7e5d9fb04710ea541b61bd55879cad65554c0d46260aa24b3d0755" class _FakeDecoder: @@ -24,11 +28,12 @@ class _FakeDecoder: ``(secret, present, schema)`` tuples -- the first for the original image, the second for the re-encoded copy used by the false-positive durability gate.""" - def __init__(self, *results: tuple[bytes, bool, int]): + def __init__(self, *results: tuple[str, bool, int]): self._results = list(results) self.calls = 0 - def decode(self, _img: object) -> tuple[bytes, bool, int]: + def decode(self, _img: object, mode: str = "binary") -> tuple[str, bool, int]: + assert mode == "binary" result = self._results[min(self.calls, len(self._results) - 1)] self.calls += 1 return result @@ -64,7 +69,7 @@ class TestFalsePositiveGate: monkeypatch.setattr(trustmark_detector, "_decoder", lambda: decoder) def test_durable_watermark_survives_and_is_reported(self, monkeypatch, tmp_clean_png: Path): - decoder = _FakeDecoder((b"secret", True, 2), (b"secret", True, 2)) + decoder = _FakeDecoder(("secret", True, 2), ("secret", True, 2)) self._patch_decoder(monkeypatch, decoder) result = detect_trustmark(tmp_clean_png) assert result == "Adobe TrustMark (variant P, schema 2)" @@ -72,18 +77,46 @@ class TestFalsePositiveGate: def test_false_positive_collapsing_on_reencode_is_dropped(self, monkeypatch, tmp_clean_png: Path): # Present on the original, absent after re-encode -> content-noise FP. - decoder = _FakeDecoder((b"\x00\x01", True, 3), (b"", False, -1)) + decoder = _FakeDecoder(("01", True, 2), ("", False, -1)) self._patch_decoder(monkeypatch, decoder) assert detect_trustmark(tmp_clean_png) is None def test_schema_drift_on_reencode_is_dropped(self, monkeypatch, tmp_clean_png: Path): # Present both times but the schema changes -> not a stable watermark. - decoder = _FakeDecoder((b"\x00", True, 2), (b"\x00", True, 3)) + decoder = _FakeDecoder(("01", True, 2), ("01", True, 3)) self._patch_decoder(monkeypatch, decoder) assert detect_trustmark(tmp_clean_png) is None + def test_payload_drift_on_reencode_is_dropped(self, monkeypatch, tmp_clean_png: Path): + decoder = _FakeDecoder(("first", True, 2), ("second", True, 2)) + self._patch_decoder(monkeypatch, decoder) + assert detect_trustmark(tmp_clean_png) is None + + def test_weak_schema_three_is_dropped(self, monkeypatch, tmp_clean_png: Path): + decoder = _FakeDecoder(("secret", True, 3)) + self._patch_decoder(monkeypatch, decoder) + assert detect_trustmark(tmp_clean_png) is None + assert decoder.calls == 1 + def test_absent_skips_reencode(self, monkeypatch, tmp_clean_png: Path): - decoder = _FakeDecoder((b"", False, -1)) + decoder = _FakeDecoder(("", False, -1)) self._patch_decoder(monkeypatch, decoder) assert detect_trustmark(tmp_clean_png) is None assert decoder.calls == 1 # no second decode when the first is absent + + +@pytest.mark.skipif(not is_available(), reason="trustmark not installed") +def test_official_adobe_variant_p_fixture(): + assert detect_trustmark(_OFFICIAL_FIXTURE) == "Adobe TrustMark (variant P, schema 1)" + + +@pytest.mark.skipif(not is_available(), reason="trustmark not installed") +def test_official_adobe_variant_p_fixture_is_provenance_not_ai(): + report = identify(_OFFICIAL_FIXTURE, check_visible=False) + assert report.is_ai_generated is None + assert report.ai_from_metadata is False + assert any(signal.name == "trustmark" for signal in report.signals) + + +def test_official_adobe_variant_p_fixture_digest(): + assert hashlib.sha256(_OFFICIAL_FIXTURE.read_bytes()).hexdigest() == _OFFICIAL_FIXTURE_SHA256