mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-09-04 19:26:34 +02:00
feat(metadata): detect China TC260 AIGC PNG chunk and HuggingFace hf-job-id
aigc_label now reads the TC260 label from a raw-JSON `AIGC` PNG tEXt chunk (as Doubao/ByteDance write it, with no namespaced XMP marker) in addition to the `<TC260:AIGC>` XMP block, via a shared _parse helper gated on a TC260 field so a generic AIGC key cannot false-positive. New huggingface_job() reads the hf-job-id PNG chunk; identify surfaces it as a medium-confidence hf_job signal (parallel to the visible sparkle, never overriding a hard metadata verdict). Both wired into has_ai_metadata/get_ai_metadata; the PNG save whitelist already strips them on removal. Found by auditing 646 corpus originals: 28 AIGC and 3 hf-job files the library previously reported as Unknown. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
0eec3001bb
commit
223cbcf171
@@ -201,6 +201,78 @@ class TestIdentifyLocalParams:
|
||||
assert r.signals == []
|
||||
|
||||
|
||||
# ── China TC260 AIGC label as a PNG text chunk (Doubao) ─────────────
|
||||
|
||||
|
||||
class TestIdentifyAigcPngChunk:
|
||||
"""The raw-JSON ``AIGC`` PNG chunk (no namespaced XMP marker) is a high-
|
||||
confidence AI verdict, same as the XMP form."""
|
||||
|
||||
def _aigc_chunk_png(self, tmp_path: Path) -> Path:
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
p = tmp_path / "doubao_chunk.png"
|
||||
pnginfo = PngInfo()
|
||||
pnginfo.add_text("AIGC", json.dumps({"Label": "1", "ContentProducer": "doubao"}))
|
||||
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
|
||||
return p
|
||||
|
||||
def test_png_chunk_detected_high(self, tmp_path: Path):
|
||||
r = identify(self._aigc_chunk_png(tmp_path), check_visible=False)
|
||||
assert r.is_ai_generated is True
|
||||
assert r.confidence == "high"
|
||||
assert r.platform is not None
|
||||
assert "AIGC" in r.platform
|
||||
signal = next(s for s in r.signals if s.name == "aigc")
|
||||
assert "doubao" in signal.detail
|
||||
|
||||
|
||||
# ── HuggingFace-hosted job marker (medium confidence) ───────────────
|
||||
|
||||
|
||||
class TestIdentifyHuggingFaceJob:
|
||||
"""The hf-job-id chunk lifts an otherwise-Unknown verdict to a tentative
|
||||
(medium) AI, never overriding a high-confidence metadata signal."""
|
||||
|
||||
def _hf_png(self, tmp_path: Path) -> Path:
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
p = tmp_path / "hfjob.png"
|
||||
pnginfo = PngInfo()
|
||||
pnginfo.add_text("hf-job-id", "ec8380a6-2091-423a-b835-209420f99ee1")
|
||||
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
|
||||
return p
|
||||
|
||||
def test_hf_job_promotes_to_medium(self, tmp_path: Path):
|
||||
r = identify(self._hf_png(tmp_path), check_visible=False)
|
||||
assert r.is_ai_generated is True
|
||||
assert r.confidence == "medium"
|
||||
assert r.platform is not None
|
||||
assert "HuggingFace" in r.platform
|
||||
signal = next(s for s in r.signals if s.name == "hf_job")
|
||||
assert signal.confidence == "medium"
|
||||
|
||||
def test_hf_job_caveat_present(self, tmp_path: Path):
|
||||
r = identify(self._hf_png(tmp_path), check_visible=False)
|
||||
assert any("hf-job-id" in c for c in r.caveats)
|
||||
|
||||
def test_metadata_keeps_high_even_with_hf_job(self, tmp_png_with_ai_metadata: Path):
|
||||
# A high-confidence metadata verdict is not downgraded by an hf-job hit.
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
img = Image.open(tmp_png_with_ai_metadata)
|
||||
pnginfo = PngInfo()
|
||||
for k, v in img.text.items():
|
||||
pnginfo.add_text(k, v)
|
||||
pnginfo.add_text("hf-job-id", "ec8380a6-2091-423a-b835-209420f99ee1")
|
||||
img.save(tmp_png_with_ai_metadata, pnginfo=pnginfo)
|
||||
r = identify(tmp_png_with_ai_metadata, check_visible=False)
|
||||
assert r.confidence == "high"
|
||||
|
||||
|
||||
# ── Visible-sparkle fallback (mocked detector) ──────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -554,6 +554,88 @@ class TestAIGCLabel:
|
||||
assert "aigc_label" in meta
|
||||
assert "TC260" in meta["aigc_label"]
|
||||
|
||||
def _aigc_chunk_png(self, tmp_path: Path, producer: str = "doubao") -> Path:
|
||||
"""Doubao writes the TC260 object as a PNG ``tEXt`` chunk keyed ``AIGC``
|
||||
with raw JSON (no XMP, no namespaced marker)."""
|
||||
import json
|
||||
|
||||
p = tmp_path / "doubao_chunk.png"
|
||||
pnginfo = PngInfo()
|
||||
pnginfo.add_text(
|
||||
"AIGC",
|
||||
json.dumps({"Label": "1", "ContentProducer": producer, "ProduceID": "abc123"}),
|
||||
)
|
||||
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
|
||||
return p
|
||||
|
||||
def test_parses_png_text_chunk_form(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import aigc_label
|
||||
|
||||
info = aigc_label(self._aigc_chunk_png(tmp_path))
|
||||
assert info is not None
|
||||
assert info["Label"] == "1"
|
||||
assert info["ContentProducer"] == "doubao"
|
||||
|
||||
def test_png_chunk_without_tc260_field_ignored(self, tmp_path: Path):
|
||||
"""A generic ``AIGC`` chunk with no TC260 field must not false-positive."""
|
||||
import json
|
||||
|
||||
from remove_ai_watermarks.metadata import aigc_label
|
||||
|
||||
p = tmp_path / "unrelated.png"
|
||||
pnginfo = PngInfo()
|
||||
pnginfo.add_text("AIGC", json.dumps({"unrelated": "value"}))
|
||||
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
|
||||
assert aigc_label(p) is None
|
||||
|
||||
def test_has_ai_metadata_detects_png_chunk_form(self, tmp_path: Path):
|
||||
assert has_ai_metadata(self._aigc_chunk_png(tmp_path))
|
||||
|
||||
def test_remove_strips_png_chunk_form(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import aigc_label, remove_ai_metadata
|
||||
|
||||
out = tmp_path / "clean.png"
|
||||
remove_ai_metadata(self._aigc_chunk_png(tmp_path), out)
|
||||
assert aigc_label(out) is None
|
||||
assert not has_ai_metadata(out)
|
||||
|
||||
|
||||
class TestHuggingFaceJob:
|
||||
"""HuggingFace-hosted job marker (``hf-job-id`` PNG text chunk)."""
|
||||
|
||||
def _hf_png(self, tmp_path: Path, job_id: str = "ec8380a6-2091-423a-b835-209420f99ee1") -> Path:
|
||||
p = tmp_path / "hfjob.png"
|
||||
pnginfo = PngInfo()
|
||||
pnginfo.add_text("hf-job-id", job_id)
|
||||
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
|
||||
return p
|
||||
|
||||
def test_returns_job_id(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import huggingface_job
|
||||
|
||||
assert huggingface_job(self._hf_png(tmp_path)) == "ec8380a6-2091-423a-b835-209420f99ee1"
|
||||
|
||||
def test_none_when_absent(self, tmp_clean_png):
|
||||
from remove_ai_watermarks.metadata import huggingface_job
|
||||
|
||||
assert huggingface_job(tmp_clean_png) is None
|
||||
|
||||
def test_has_ai_metadata_detects_hf_job(self, tmp_path: Path):
|
||||
assert has_ai_metadata(self._hf_png(tmp_path))
|
||||
|
||||
def test_get_ai_metadata_surfaces_hf_job(self, tmp_path: Path):
|
||||
meta = get_ai_metadata(self._hf_png(tmp_path))
|
||||
assert "huggingface_job" in meta
|
||||
assert "ec8380a6" in meta["huggingface_job"]
|
||||
|
||||
def test_remove_strips_hf_job(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata import huggingface_job, remove_ai_metadata
|
||||
|
||||
out = tmp_path / "clean.png"
|
||||
remove_ai_metadata(self._hf_png(tmp_path), out)
|
||||
assert huggingface_job(out) is None
|
||||
assert not has_ai_metadata(out)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not (SAMPLES_DIR / "doubao-1.png").exists(), reason="doubao sample not present")
|
||||
class TestAIGCRealSample:
|
||||
|
||||
Reference in New Issue
Block a user