mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-08-09 23:50:40 +02:00
Merge remote-tracking branch 'origin/main' into research/video-synthid-quality-groundwork
This commit is contained in:
@@ -1,126 +0,0 @@
|
||||
"""Tests for the standalone structural AI-generation scorer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import ai_score
|
||||
|
||||
|
||||
def _complete_record() -> dict[str, Any]:
|
||||
return {
|
||||
"noise": {"noise_std": 1.0, "noise_kurtosis": 2.0},
|
||||
"fft": {
|
||||
"cfa_peak": 3.0,
|
||||
"cfa_peaks": [2.5, 3.0],
|
||||
"fft_band_energy": list(range(8)),
|
||||
},
|
||||
"ela": {"ela_mean": 4.0, "ela_p95": 5.0},
|
||||
"gradient": {"laplacian_var": 6.0, "gradient_hist": list(range(10))},
|
||||
"color": {
|
||||
"saturation_mean": 0.5,
|
||||
"value_mean": 0.75,
|
||||
"color_hist_4x4x4": list(range(64)),
|
||||
},
|
||||
"dct": {
|
||||
"benford_mad": 0.1,
|
||||
"dct_ac_hist": [[1] * 21 for _ in range(8)],
|
||||
},
|
||||
"jpeg_forensics": {
|
||||
"subsampling": "4:4:4",
|
||||
"progressive": True,
|
||||
"quant_tables": {
|
||||
"0": list(range(1, 65)),
|
||||
"1": list(range(65, 129)),
|
||||
},
|
||||
"huffman_tables_hex": ["00ff", "abcd12"],
|
||||
"scan_count": 10,
|
||||
"restart_interval": 4,
|
||||
"precision_bits": 8,
|
||||
"adobe_transform": 1,
|
||||
"jfif": {"version": "1.1"},
|
||||
},
|
||||
"pil": {"width": 2000, "height": 1000},
|
||||
"content_format": "jpeg",
|
||||
}
|
||||
|
||||
|
||||
def test_v1_feature_schema_is_fixed_for_sparse_records() -> None:
|
||||
assert len(ai_score.feature_names("v1")) == 97
|
||||
assert len(ai_score.features_of({}, schema="v1")) == 97
|
||||
|
||||
|
||||
def test_v2_feature_schema_includes_existing_forensic_data() -> None:
|
||||
record = _complete_record()
|
||||
|
||||
names = ai_score.feature_names("v2")
|
||||
values = ai_score.features_of(record, schema="v2")
|
||||
by_name = dict(zip(names, values, strict=True))
|
||||
|
||||
assert len(names) == len(values) == 406
|
||||
assert by_name["cfa_peak_0"] == 2.5
|
||||
assert by_name["cfa_peak_1"] == 3.0
|
||||
assert by_name["dct_ac_0_0"] == 1 / 21
|
||||
assert by_name["dct_ac_7_20"] == 1 / 21
|
||||
assert by_name["jpeg_quant_0_0"] == 1.0
|
||||
assert by_name["jpeg_quant_1_63"] == 128.0
|
||||
assert by_name["jpeg_quant_table_count"] == 2.0
|
||||
assert by_name["jpeg_huffman_table_count"] == 2.0
|
||||
assert by_name["jpeg_huffman_total_bytes"] == 5.0
|
||||
assert by_name["jpeg_scan_count"] == 10.0
|
||||
assert by_name["jpeg_jfif_present"] == 1.0
|
||||
assert by_name["format_webp"] == 0.0
|
||||
assert by_name["format_isobmff"] == 0.0
|
||||
assert by_name["format_other"] == 0.0
|
||||
|
||||
|
||||
def test_v2_feature_schema_is_fixed_when_forensics_are_missing() -> None:
|
||||
names = ai_score.feature_names("v2")
|
||||
values = ai_score.features_of({}, schema="v2")
|
||||
|
||||
assert len(names) == len(values) == 406
|
||||
assert np.isnan(values[names.index("dct_ac_0_0")])
|
||||
assert np.isnan(values[names.index("jpeg_quant_0_0")])
|
||||
assert np.isnan(values[names.index("jpeg_scan_count")])
|
||||
|
||||
|
||||
def test_grouped_stratified_split_keeps_hashes_on_one_side() -> None:
|
||||
labels = np.asarray([1, 1, 1, 0, 0, 0, 1, 0])
|
||||
hashes = np.asarray(["a", "a", "b", "c", "c", "d", "e", "f"])
|
||||
|
||||
train, test = ai_score.grouped_stratified_split(labels, hashes, test_size=0.5, random_state=7)
|
||||
|
||||
assert set(hashes[train]).isdisjoint(set(hashes[test]))
|
||||
assert set(labels[train]) == {0, 1}
|
||||
assert set(labels[test]) == {0, 1}
|
||||
assert sorted(np.concatenate([train, test]).tolist()) == list(range(len(labels)))
|
||||
|
||||
|
||||
def test_grouped_stratified_split_rejects_conflicting_labels() -> None:
|
||||
labels = np.asarray([0, 1, 0, 1])
|
||||
hashes = np.asarray(["same", "same", "negative", "positive"])
|
||||
|
||||
with np.testing.assert_raises_regex(ValueError, "conflicting labels"):
|
||||
ai_score.grouped_stratified_split(labels, hashes)
|
||||
|
||||
|
||||
def test_temporal_holdout_excludes_hashes_seen_during_training() -> None:
|
||||
dates = np.asarray(["2026-01-01", "2026-01-01", "2026-01-02", "2026-01-03", "2026-01-04", "2026-01-04"])
|
||||
hashes = np.asarray(["repeated", "old", "middle", "new-a", "repeated", "new-b"])
|
||||
|
||||
train, test, cutoff = ai_score.temporal_holdout_split(dates, hashes, train_fraction=0.5)
|
||||
|
||||
assert cutoff == "2026-01-03"
|
||||
assert set(hashes[train]).isdisjoint(set(hashes[test]))
|
||||
assert set(hashes[test]) == {"new-a", "new-b"}
|
||||
|
||||
|
||||
def test_legacy_model_bundle_defaults_to_v1_schema() -> None:
|
||||
assert ai_score.model_schema({}) == "v1"
|
||||
assert ai_score.model_schema({"feature_schema": "v2"}) == "v2"
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Tests for the metadata-only forensic collector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import zlib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import piexif
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from remove_ai_watermarks.forensic_metadata import (
|
||||
FORENSIC_METADATA_RECORD_TYPE,
|
||||
FORENSIC_METADATA_SCHEMA_VERSION,
|
||||
SUPPORTED_EXTENSIONS,
|
||||
_b64,
|
||||
_decode_exif_value,
|
||||
_jpeg_forensics_bytes,
|
||||
_png_text_decode,
|
||||
_safe_str,
|
||||
apple_live_photo_id,
|
||||
collect_forensic_metadata,
|
||||
read_full_exif,
|
||||
read_isobmff_inventory,
|
||||
read_isobmff_provenance_path,
|
||||
read_jpeg_segments,
|
||||
read_pil_info,
|
||||
read_png_chunks,
|
||||
read_png_late_metadata_path,
|
||||
read_webp_chunks,
|
||||
sha256_of,
|
||||
sniff_format,
|
||||
xattr_quarantine,
|
||||
xattr_where_from,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _jpeg(path: Path) -> Path:
|
||||
Image.new("RGB", (48, 32), (20, 80, 160)).save(path, "JPEG", quality=87)
|
||||
return path
|
||||
|
||||
|
||||
def _png_chunk(chunk_type: bytes, payload: bytes) -> bytes:
|
||||
crc = zlib.crc32(chunk_type + payload).to_bytes(4, "big")
|
||||
return len(payload).to_bytes(4, "big") + chunk_type + payload + crc
|
||||
|
||||
|
||||
def test_supported_extensions_are_media_not_documents():
|
||||
assert {".jpg", ".png", ".webp", ".heic", ".mp4"}.issubset(SUPPORTED_EXTENSIONS)
|
||||
assert ".pdf" not in SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def test_json_helpers_and_format_sniffer():
|
||||
class BadString:
|
||||
def __str__(self):
|
||||
raise RuntimeError("no string")
|
||||
|
||||
assert _safe_str("ok") == "ok"
|
||||
assert "BadString" in _safe_str(BadString())
|
||||
assert _b64(b"abc") == base64.b64encode(b"abc").decode("ascii")
|
||||
assert _b64(b"x" * 20, cap=4) == "eHh4eA==...TRUNCATED(20 bytes total)"
|
||||
assert _decode_exif_value(b"ascii") == "ascii"
|
||||
assert _decode_exif_value(b"\xff").startswith("hex:")
|
||||
assert _decode_exif_value((1, b"two")) == [1, "two"]
|
||||
assert sniff_format(b"\x89PNG\r\n\x1a\n") == "png"
|
||||
assert sniff_format(b"\xff\xd8\xff\xe0") == "jpeg"
|
||||
assert sniff_format(b"RIFF....WEBP") == "webp"
|
||||
assert sniff_format(b"....ftypheic").startswith("isobmff:")
|
||||
assert sniff_format(b"unknown").startswith("unknown:")
|
||||
|
||||
|
||||
def test_png_text_and_container_metadata_are_preserved(tmp_path: Path):
|
||||
info = PngInfo()
|
||||
info.add_text("parameters", "Steps: 20, Model: SDXL", zip=True)
|
||||
path = tmp_path / "workflow.png"
|
||||
Image.new("RGB", (32, 32)).save(path, pnginfo=info)
|
||||
trailer = b'<TC260:AIGC>{"Label":"1"}</TC260:AIGC>'
|
||||
path.write_bytes(path.read_bytes() + trailer)
|
||||
|
||||
record = collect_forensic_metadata(path)
|
||||
|
||||
assert record["schema_version"] == FORENSIC_METADATA_SCHEMA_VERSION == 1
|
||||
assert record["record_type"] == FORENSIC_METADATA_RECORD_TYPE == "forensic_metadata"
|
||||
assert record["content_format"] == "png"
|
||||
assert any(chunk.get("type") == "zTXt" for chunk in record["png_chunks"])
|
||||
assert record["png_post_iend_bytes"] == len(trailer)
|
||||
assert base64.b64decode(record["png_post_iend_base64"]) == trailer
|
||||
assert "Steps: 20" in json.dumps(record)
|
||||
assert json.loads(json.dumps(record, allow_nan=False)) == record
|
||||
|
||||
|
||||
def test_png_text_decoders_and_direct_chunk_reader(tmp_path: Path):
|
||||
assert "hello" in _png_text_decode("tEXt", b"key\x00hello")
|
||||
compressed = b"prompt\x00\x00" + zlib.compress(b"workflow")
|
||||
assert "workflow" in _png_text_decode("zTXt", compressed)
|
||||
assert "value" in _png_text_decode("iTXt", b"key\x00\x00\x00\x00\x00value")
|
||||
|
||||
path = tmp_path / "plain.png"
|
||||
Image.new("RGB", (8, 8)).save(path)
|
||||
chunks, trailer = read_png_chunks(path.read_bytes())
|
||||
|
||||
assert chunks[0]["type"] == "IHDR"
|
||||
assert trailer == b""
|
||||
|
||||
|
||||
def test_jpeg_exif_segments_encoder_and_trailer(tmp_path: Path):
|
||||
path = tmp_path / "camera.jpg"
|
||||
exif = piexif.dump(
|
||||
{
|
||||
"0th": {
|
||||
piexif.ImageIFD.Make: b"Camera Corp",
|
||||
piexif.ImageIFD.Software: b"Camera Firmware",
|
||||
},
|
||||
"Exif": {},
|
||||
"GPS": {},
|
||||
"1st": {},
|
||||
}
|
||||
)
|
||||
Image.new("RGB", (64, 48)).save(path, "JPEG", exif=exif, quality=82)
|
||||
trailer = b'PhotoEditor_Re_Edit_Data{"genAIType":1}'
|
||||
path.write_bytes(path.read_bytes() + trailer)
|
||||
|
||||
record = collect_forensic_metadata(path)
|
||||
|
||||
assert record["exif"]["0th"]["Make"] == "Camera Corp"
|
||||
assert record["jpeg"]["post_eoi_bytes"] == len(trailer)
|
||||
assert base64.b64decode(record["jpeg"]["post_eoi_base64"]) == trailer
|
||||
assert record["jpeg_forensics"]["quant_tables"]
|
||||
assert sha256_of(path.read_bytes()) == record["sha256"]
|
||||
|
||||
|
||||
def test_direct_exif_pil_and_jpeg_readers(tmp_path: Path):
|
||||
path = _jpeg(tmp_path / "plain.jpg")
|
||||
|
||||
exif, thumbnail = read_full_exif(path)
|
||||
pil, iptc, exif_blob = read_pil_info(path)
|
||||
segments = read_jpeg_segments(path.read_bytes())
|
||||
|
||||
assert isinstance(exif, dict)
|
||||
assert thumbnail is None
|
||||
assert pil["width"] == 48
|
||||
assert pil["height"] == 32
|
||||
assert isinstance(iptc, dict)
|
||||
assert exif_blob is None or isinstance(exif_blob, bytes)
|
||||
assert isinstance(segments["segments"], list)
|
||||
assert _jpeg_forensics_bytes(path.read_bytes())["quant_tables"]
|
||||
assert _jpeg_forensics_bytes(b"not a jpeg") == {}
|
||||
|
||||
|
||||
def test_webp_inventory_keeps_metadata_but_not_frame_pixels(tmp_path: Path):
|
||||
path = tmp_path / "image.webp"
|
||||
xmp = b"<x:xmpmeta>metadata</x:xmpmeta>"
|
||||
Image.new("RGB", (32, 32), (30, 40, 50)).save(path, "WEBP", xmp=xmp)
|
||||
|
||||
chunks = read_webp_chunks(path.read_bytes())
|
||||
|
||||
xmp_chunk = next(chunk for chunk in chunks if chunk["type"] == "XMP ")
|
||||
assert xmp_chunk["text"] == xmp.decode()
|
||||
assert all("base64" not in chunk for chunk in chunks if chunk["type"] in {"VP8 ", "VP8L", "ANMF"})
|
||||
|
||||
|
||||
def test_isobmff_inventory_and_streaming_provenance(tmp_path: Path):
|
||||
path = tmp_path / "signed.mp4"
|
||||
ftyp = b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom"
|
||||
payload = b"jumb c2pa trainedAlgorithmicMedia"
|
||||
uuid_box = (8 + len(payload)).to_bytes(4, "big") + b"uuid" + payload
|
||||
path.write_bytes(ftyp + b"\x00\x00\x00\x08mdat" + uuid_box)
|
||||
|
||||
inventory = read_isobmff_inventory(path.read_bytes())
|
||||
streamed = read_isobmff_provenance_path(path)
|
||||
|
||||
assert "ftyp" in inventory["boxes"]
|
||||
assert base64.b64decode(inventory["provenance_boxes"][0]["base64"]) == payload
|
||||
assert base64.b64decode(streamed["provenance_boxes"][0]["base64"]) == payload
|
||||
|
||||
|
||||
def test_oversized_path_keeps_bounded_windows_and_late_png_metadata(tmp_path: Path, monkeypatch):
|
||||
path = tmp_path / "late.png"
|
||||
Image.new("RGB", (16, 16)).save(path)
|
||||
source = path.read_bytes()
|
||||
iend = source.rfind(b"\x00\x00\x00\x00IEND")
|
||||
padding = _png_chunk(b"vpAg", b"\x00" * ((1 << 20) + 1))
|
||||
metadata = b'AIGC\x00{"Label":"1"}'
|
||||
path.write_bytes(source[:iend] + padding + _png_chunk(b"tEXt", metadata) + source[iend:])
|
||||
monkeypatch.setattr("remove_ai_watermarks.forensic_metadata._MAX_FULL_READ", 1)
|
||||
|
||||
record = collect_forensic_metadata(path)
|
||||
|
||||
assert record["oversized"]["head_scanned_bytes"] == path.stat().st_size
|
||||
assert base64.b64decode(record["raw_metadata_windows"]["head_base64"])
|
||||
assert base64.b64decode(record["png_late_metadata_chunks"][0]["base64"]) == metadata
|
||||
|
||||
|
||||
def test_collection_registers_optional_heif_and_missing_file_raises(tmp_path: Path, monkeypatch):
|
||||
registered = False
|
||||
|
||||
def mark_registered():
|
||||
nonlocal registered
|
||||
registered = True
|
||||
|
||||
monkeypatch.setattr("remove_ai_watermarks.image_io._register_heif", mark_registered)
|
||||
collect_forensic_metadata(_jpeg(tmp_path / "plain.jpg"))
|
||||
|
||||
assert registered is True
|
||||
with pytest.raises(FileNotFoundError):
|
||||
collect_forensic_metadata(tmp_path / "missing.jpg")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("schema_version", [2, True, 1.0])
|
||||
def test_collection_rejects_unsupported_output_schema_before_reading(tmp_path: Path, schema_version: object):
|
||||
with pytest.raises(ValueError, match="Unsupported forensic metadata schema"):
|
||||
collect_forensic_metadata(
|
||||
tmp_path / "missing.jpg",
|
||||
schema_version=schema_version, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
def test_xattrs_and_live_photo_probe_are_safe_on_plain_file(tmp_path: Path):
|
||||
path = _jpeg(tmp_path / "plain.jpg")
|
||||
|
||||
assert xattr_where_from(path) == [] or isinstance(xattr_where_from(path), list)
|
||||
assert xattr_quarantine(path) is None or isinstance(xattr_quarantine(path), str)
|
||||
assert apple_live_photo_id(path.read_bytes()) is None
|
||||
|
||||
|
||||
def test_late_png_reader_soft_fails_on_non_png(tmp_path: Path):
|
||||
path = tmp_path / "plain.bin"
|
||||
path.write_bytes(b"not png")
|
||||
|
||||
assert read_png_late_metadata_path(path) == []
|
||||
@@ -112,6 +112,36 @@ class TestProvenanceEvidence:
|
||||
|
||||
assert evidence.exif_generator == "NovelAI"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"record",
|
||||
[
|
||||
{"name": "trainedAlgorithmicMedia.jpg"},
|
||||
{"sha256": "jumb-c2pa-OpenAI-trainedAlgorithmicMedia"},
|
||||
{"pil": {"trainedAlgorithmicMedia": "plain"}},
|
||||
{"signals": {"provenance": {"is_ai_generated": True}}},
|
||||
{"pixel": {"error": "trainedAlgorithmicMedia"}},
|
||||
],
|
||||
)
|
||||
def test_external_diagnostics_and_arbitrary_keys_are_not_evidence(self, tmp_path: Path, record: dict):
|
||||
report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "plain.jpg"))
|
||||
|
||||
assert report.is_ai_generated is None
|
||||
assert report.signals == []
|
||||
|
||||
def test_external_metadata_value_is_evidence(self, tmp_path: Path):
|
||||
record = {
|
||||
"exif": {
|
||||
"0th": {
|
||||
"ImageDescription": "digitalSourceType=trainedAlgorithmicMedia",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "generated.jpg"))
|
||||
|
||||
assert report.is_ai_generated is True
|
||||
assert report.ai_source_kind == "generated"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filename",
|
||||
[
|
||||
@@ -443,6 +473,18 @@ class TestIdentifyRealSamples:
|
||||
r = identify(p, check_visible=False, check_invisible=False)
|
||||
assert r.is_ai_generated is True
|
||||
assert r.platform == "Apple Photos (Clean Up AI edit)"
|
||||
assert r.ai_source_kind == "enhanced"
|
||||
|
||||
def test_standalone_iptc_composite_synthetic_is_enhanced(self, tmp_path: Path):
|
||||
p = tmp_path / "composite.jpg"
|
||||
p.write_bytes(
|
||||
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="compositeSynthetic"></x:xmpmeta>\xff\xd9'
|
||||
)
|
||||
|
||||
r = identify(p, check_visible=False, check_invisible=False)
|
||||
|
||||
assert r.is_ai_generated is True
|
||||
assert r.ai_source_kind == "enhanced"
|
||||
|
||||
def test_flux_bfl_c2pa_png(self):
|
||||
# flux-1.png: real Black Forest Labs FLUX.2 Playground output (signed C2PA).
|
||||
@@ -1353,3 +1395,106 @@ class TestSharedPixelDecode:
|
||||
report = identify(self.SAMPLE, check_visible=True, check_invisible=False)
|
||||
assert not any(s.name.startswith("visible_") for s in report.signals)
|
||||
assert report.is_ai_generated is True # the C2PA verdict survives the decode failure
|
||||
|
||||
|
||||
class TestSynthIdProxyIsDecidedInTheVerdict:
|
||||
"""The SynthID byte scan belongs to the verdict, not to extraction.
|
||||
|
||||
Extraction has two implementations -- one reading a file, one reading a portable
|
||||
record -- so a rule that lives in only one of them is a rule the other silently
|
||||
lacks. This one did: 74 corpus images reported SynthID through ``identify`` and
|
||||
not through the record."""
|
||||
|
||||
# A JUMBF-wrapped manifest from a SynthID-pairing signer on AI-generated content:
|
||||
# the exact shape `synthid_source`'s byte scan is gated on. Spliced into a real
|
||||
# JPEG as a well-formed APP11 segment, because a malformed one is skipped by the
|
||||
# record's structural walk and the test would compare two different inputs.
|
||||
MANIFEST = b"jumb c2pa Google LLC trainedAlgorithmicMedia"
|
||||
|
||||
def _jpeg_with_manifest(self, path: Path) -> Path:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
data = path.read_bytes()
|
||||
segment = b"\xff\xeb" + (len(self.MANIFEST) + 2).to_bytes(2, "big") + self.MANIFEST
|
||||
path.write_bytes(data[:2] + segment + data[2:])
|
||||
return path
|
||||
|
||||
def test_both_paths_infer_it_from_the_same_bytes(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.identify import identify_metadata_record
|
||||
from remove_ai_watermarks.metadata_record import collect_metadata_record
|
||||
|
||||
path = self._jpeg_with_manifest(tmp_path / "gemini.jpg")
|
||||
|
||||
via_file = identify(path, check_visible=False, check_invisible=False)
|
||||
via_record = identify_metadata_record(collect_metadata_record(path), path=path)
|
||||
|
||||
assert any("SynthID" in mark for mark in via_file.watermarks)
|
||||
assert via_record.watermarks == via_file.watermarks
|
||||
|
||||
def test_it_needs_a_manifest_and_an_ai_source_type(self, tmp_path: Path):
|
||||
"""The vendor name alone is not evidence: an ordinary photo mentioning
|
||||
"Google LLC" in EXIF must not acquire a SynthID verdict."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
path = tmp_path / "photo.jpg"
|
||||
Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
data = path.read_bytes()
|
||||
note = b"Google LLC Pixel"
|
||||
path.write_bytes(data[:2] + b"\xff\xeb" + (len(note) + 2).to_bytes(2, "big") + note + data[2:])
|
||||
|
||||
report = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
assert not any("SynthID" in mark for mark in report.watermarks)
|
||||
|
||||
|
||||
class TestRegistryScansSkipTheCodedPixels:
|
||||
"""The vendor registries match short raw substrings -- the shortest are four and
|
||||
five bytes. Over a megabyte of compressed pixel data such a sequence turns up by
|
||||
chance: `Bria` matched inside the entropy-coded scan of 4 of 14,707 corpus JPEGs,
|
||||
and that entry asserts AI, so a chance match can declare an image AI-generated.
|
||||
`c2pa_marker_in` already refuses a bare `c2pa` substring for the same reason."""
|
||||
|
||||
def _jpeg(self, path: Path, *, in_segment: bytes = b"", in_scan: bytes = b"") -> Path:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
data = path.read_bytes()
|
||||
if in_segment:
|
||||
payload = b"jumb c2pa trainedAlgorithmicMedia " + in_segment
|
||||
data = data[:2] + b"\xff\xeb" + (len(payload) + 2).to_bytes(2, "big") + payload + data[2:]
|
||||
if in_scan:
|
||||
# After SOS, i.e. inside the entropy-coded scan the walk skips.
|
||||
sos = data.index(b"\xff\xda")
|
||||
data = data[: sos + 16] + in_scan + data[sos + 16 :]
|
||||
path.write_bytes(data)
|
||||
return path
|
||||
|
||||
def test_a_token_in_a_marker_segment_is_attributed(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.identify import _issuers_in, _metadata_region
|
||||
from remove_ai_watermarks.metadata import scan_head
|
||||
|
||||
path = self._jpeg(tmp_path / "signed.jpg", in_segment=b"Bria")
|
||||
|
||||
assert _issuers_in(_metadata_region(scan_head(path))) == ["Bria Artificial Intelligence"]
|
||||
|
||||
def test_a_token_in_the_coded_scan_is_not(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.identify import _issuers_in, _metadata_region
|
||||
from remove_ai_watermarks.metadata import scan_head
|
||||
|
||||
path = self._jpeg(tmp_path / "chance.jpg", in_segment=b"OpenAI", in_scan=b"Bria")
|
||||
region = _metadata_region(scan_head(path))
|
||||
|
||||
assert _issuers_in(region) == ["OpenAI"]
|
||||
|
||||
def test_a_container_that_does_not_parse_is_left_whole(self, tmp_path: Path):
|
||||
"""Cutting a buffer the walk did not understand would drop real evidence to
|
||||
avoid a chance match, which is the wrong way round."""
|
||||
from remove_ai_watermarks.identify import _metadata_region
|
||||
|
||||
blob = b"\xff\xd8" + b"not really a jpeg, no valid marker chain here" * 4
|
||||
|
||||
assert _metadata_region(blob) == blob
|
||||
|
||||
@@ -1316,6 +1316,30 @@ class TestAIGCLabel:
|
||||
assert b"ContentProducer" in _png_late_metadata(p, 8)
|
||||
assert b"ContentProducer" in scan_head(p, 8)
|
||||
|
||||
def test_scan_head_collects_webp_metadata_past_window(self, tmp_path: Path):
|
||||
"""WebP stores ``XMP `` AFTER the pixels, so on any WebP above the window a
|
||||
fixed read can stop short of an IPTC "Made with AI" tag."""
|
||||
from remove_ai_watermarks.metadata import _riff_late_metadata, scan_head
|
||||
|
||||
p = tmp_path / "late.webp"
|
||||
xmp = (
|
||||
b"<x:xmpmeta><photoshop:DigitalSourceType>trainedAlgorithmicMedia</photoshop:DigitalSourceType></x:xmpmeta>"
|
||||
)
|
||||
Image.new("RGB", (16, 16)).save(p, "WEBP", xmp=xmp)
|
||||
|
||||
assert b"trainedAlgorithmicMedia" in _riff_late_metadata(p, 12)
|
||||
assert b"trainedAlgorithmicMedia" in scan_head(p, 12)
|
||||
|
||||
def test_riff_late_metadata_ignores_the_coded_image(self, tmp_path: Path):
|
||||
"""The point of stepping chunk by chunk rather than reading through: the
|
||||
pixel payload never enters the scan buffer."""
|
||||
from remove_ai_watermarks.metadata import _riff_late_metadata
|
||||
|
||||
p = tmp_path / "plain.webp"
|
||||
Image.new("RGB", (64, 64), (200, 30, 30)).save(p, "WEBP")
|
||||
|
||||
assert _riff_late_metadata(p, 12) == b""
|
||||
|
||||
|
||||
class TestHuggingFaceJob:
|
||||
"""HuggingFace-hosted job marker (``hf-job-id`` PNG text chunk)."""
|
||||
|
||||
@@ -3,10 +3,12 @@ consolidated metadata strip (formerly legacy metadata helper)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import struct
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks._internal.c2pa import (
|
||||
_parse_c2pa_chunk,
|
||||
@@ -835,3 +837,38 @@ class TestTc260ContainerRouting:
|
||||
|
||||
readers = _tc260_container_readers()
|
||||
assert [r.__module__.rsplit(".", 1)[-1] for r in readers] == ["isobmff", "ebml", "riff", "flv"]
|
||||
|
||||
|
||||
class TestC2paReaderFailureIsVisible:
|
||||
"""A reader failure and a file with no manifest both return None, so the caller
|
||||
cannot tell them apart -- and the consequence is not symmetric. A file with no
|
||||
manifest is a normal verdict; a reader that could not read a file it was handed
|
||||
can silently downgrade one, so the log level must make the failure observable."""
|
||||
|
||||
def _records(self, caplog, path: str) -> list[str]:
|
||||
from remove_ai_watermarks._internal import c2pa
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="remove_ai_watermarks._internal.c2pa"):
|
||||
assert c2pa._manifest_json_uncached(path) is None
|
||||
return [f"{r.levelname} {r.getMessage()}" for r in caplog.records]
|
||||
|
||||
def test_an_unreadable_file_warns(self, caplog):
|
||||
records = self._records(caplog, "/nonexistent/definitely-not-here.png")
|
||||
|
||||
assert any(r.startswith("WARNING") for r in records), records
|
||||
|
||||
def test_an_unsupported_container_stays_quiet(self, caplog, tmp_path: Path):
|
||||
target = tmp_path / "notes.txt"
|
||||
target.write_text("plain text, not a container the reader handles")
|
||||
|
||||
records = self._records(caplog, str(target))
|
||||
|
||||
assert not any(r.startswith("WARNING") for r in records), records
|
||||
|
||||
def test_a_plain_image_without_a_manifest_logs_nothing(self, caplog, tmp_path: Path):
|
||||
target = tmp_path / "plain.png"
|
||||
Image.new("RGB", (8, 8)).save(target)
|
||||
|
||||
records = self._records(caplog, str(target))
|
||||
|
||||
assert records == []
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Tests for the portable metadata record.
|
||||
|
||||
The contract is one property: a verdict built from the record equals the verdict
|
||||
built from the file. Everything here exists to pin that, or to pin a placement the
|
||||
record could plausibly drop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks.identify import (
|
||||
PROVENANCE_REPORT_SCHEMA_VERSION,
|
||||
ProvenanceReport,
|
||||
evidence_from_metadata_record,
|
||||
identify,
|
||||
identify_from_evidence,
|
||||
identify_metadata_record,
|
||||
)
|
||||
from remove_ai_watermarks.metadata_record import (
|
||||
HEAD_WINDOW,
|
||||
METADATA_RECORD_SCHEMA_VERSION,
|
||||
METADATA_RECORD_TYPE,
|
||||
collect_metadata_record,
|
||||
)
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance"
|
||||
COMPARED = ("is_ai_generated", "platform", "confidence", "ai_source_kind", "ai_from_metadata")
|
||||
|
||||
|
||||
def _verdict_via_record(path: Path) -> ProvenanceReport:
|
||||
"""The contractor's path: collect, serialize, judge -- without the file."""
|
||||
record = json.loads(json.dumps(collect_metadata_record(path)))
|
||||
return identify_metadata_record(record, path=path)
|
||||
|
||||
|
||||
def _assert_same_verdict(path: Path) -> None:
|
||||
via_record = _verdict_via_record(path)
|
||||
via_file = identify(path, check_visible=False, check_invisible=False)
|
||||
|
||||
for field in COMPARED:
|
||||
assert getattr(via_record, field) == getattr(via_file, field), field
|
||||
assert sorted(s.name for s in via_record.signals) == sorted(s.name for s in via_file.signals)
|
||||
assert sorted(via_record.watermarks) == sorted(via_file.watermarks)
|
||||
|
||||
|
||||
def _noise_png(path: Path, size: tuple[int, int] = (700, 700)) -> Path:
|
||||
"""A PNG whose IDAT is incompressible, so the file exceeds the head window."""
|
||||
rng = np.random.default_rng(0)
|
||||
Image.fromarray(rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8)).save(path)
|
||||
return path
|
||||
|
||||
|
||||
def _insert_png_chunk(path: Path, chunk_type: bytes, payload: bytes) -> None:
|
||||
"""Splice a chunk in just before IEND, i.e. AFTER the whole pixel stream."""
|
||||
data = path.read_bytes()
|
||||
end = data.rindex(b"IEND") - 4
|
||||
chunk = struct.pack(">I", len(payload)) + chunk_type + payload
|
||||
chunk += struct.pack(">I", zlib.crc32(chunk_type + payload) & 0xFFFFFFFF)
|
||||
path.write_bytes(data[:end] + chunk + data[end:])
|
||||
|
||||
|
||||
class TestRecordReproducesTheFileVerdict:
|
||||
"""The whole point of the record. Every tracked provenance fixture is compared,
|
||||
which covers C2PA, the China TC260 label, the xAI EXIF pair and the IPTC tag."""
|
||||
|
||||
@pytest.mark.skipif(not FIXTURES.is_dir(), reason="provenance fixtures not present")
|
||||
@pytest.mark.parametrize("name", sorted(p.name for p in FIXTURES.iterdir()) if FIXTURES.is_dir() else [])
|
||||
def test_fixture(self, name: str):
|
||||
_assert_same_verdict(FIXTURES / name)
|
||||
|
||||
@pytest.mark.skipif(not FIXTURES.is_dir(), reason="provenance fixtures not present")
|
||||
def test_the_fixtures_actually_exercise_several_signals(self):
|
||||
"""Guards the test above: if the fixture set ever narrows to one signal
|
||||
family, equality across it stops meaning much."""
|
||||
found = {
|
||||
signal.name
|
||||
for path in FIXTURES.iterdir()
|
||||
for signal in identify(path, check_visible=False, check_invisible=False).signals
|
||||
}
|
||||
assert len(found) >= 4, found
|
||||
|
||||
|
||||
class TestPlacementsTheRecordCouldDrop:
|
||||
def test_a_trailer_after_eoi_survives(self, tmp_path: Path):
|
||||
"""Samsung Galaxy AI appends its marker past the JPEG EOI, and the value it is
|
||||
gated on can sit further back still. A record that stopped at the last
|
||||
structural marker would report no signal at all."""
|
||||
path = tmp_path / "edited.jpg"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
with path.open("ab") as handle:
|
||||
handle.write(b'PhotoEditor_Re_Edit_Data{"genAIType":17}')
|
||||
|
||||
assert identify(path, check_visible=False, check_invisible=False).confidence == "medium"
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_a_metadata_chunk_past_the_head_window_survives(self, tmp_path: Path):
|
||||
"""A PNG encoder may put the label chunk after the pixels. The file is larger
|
||||
than the record's head window, so only the seek-past-IDAT reader finds it."""
|
||||
path = _noise_png(tmp_path / "late.png")
|
||||
assert path.stat().st_size > HEAD_WINDOW
|
||||
# An XMP packet in the namespaced TC260 form, which is how the label travels
|
||||
# when it is not a bare ``AIGC`` keyword chunk.
|
||||
label = json.dumps({"Label": "1", "ContentProducer": "001191110102MACQD9K64010000"})
|
||||
xmp = (
|
||||
b'<x:xmpmeta xmlns:x="adobe:ns:meta/"><rdf:RDF><rdf:Description '
|
||||
b'xmlns:TC260="http://www.tc260.org.cn/ns/AIGC/1.0/"><TC260:AIGC>'
|
||||
+ label.encode()
|
||||
+ b"</TC260:AIGC></rdf:Description></rdf:RDF></x:xmpmeta>"
|
||||
)
|
||||
_insert_png_chunk(path, b"iTXt", b"XML:com.adobe.xmp\x00\x00\x00\x00\x00" + xmp)
|
||||
|
||||
assert "aigc" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_an_exif_pair_survives(self, tmp_path: Path):
|
||||
"""xAI is recognized from an (ImageDescription, Artist) PAIR, and both are read
|
||||
by tag NAME. A record carrying only raw bytes loses it."""
|
||||
import piexif
|
||||
|
||||
path = tmp_path / "grok.jpg"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
exif = {
|
||||
"0th": {
|
||||
piexif.ImageIFD.ImageDescription: b"Signature: " + b"A" * 80,
|
||||
piexif.ImageIFD.Artist: b"3f2504e0-4f89-11d3-9a0c-0305e82c3301",
|
||||
}
|
||||
}
|
||||
piexif.insert(piexif.dump(exif), str(path))
|
||||
|
||||
assert "xai_signature" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals}
|
||||
_assert_same_verdict(path)
|
||||
|
||||
|
||||
class TestRecordShape:
|
||||
def test_the_record_survives_json(self, tmp_path: Path):
|
||||
record = collect_metadata_record(_noise_png(tmp_path / "plain.png"))
|
||||
|
||||
assert json.loads(json.dumps(record, allow_nan=False))["container"] == "png"
|
||||
assert record["schema_version"] == METADATA_RECORD_SCHEMA_VERSION == 1
|
||||
assert record["record_type"] == METADATA_RECORD_TYPE == "provenance_metadata"
|
||||
|
||||
@pytest.mark.parametrize("keep_version", [True, False])
|
||||
def test_transport_filename_is_not_evidence(self, tmp_path: Path, keep_version: bool):
|
||||
"""The path labels a record; detector tokens in it do not describe pixels."""
|
||||
path = tmp_path / "jumb-c2pa-OpenAI-trainedAlgorithmicMedia.jpg"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG")
|
||||
record = collect_metadata_record(path)
|
||||
if not keep_version:
|
||||
record.pop("schema_version")
|
||||
record.pop("record_type")
|
||||
|
||||
report = identify_metadata_record(record, path=path)
|
||||
|
||||
assert report.is_ai_generated is None
|
||||
assert report.platform is None
|
||||
assert report.confidence == "none"
|
||||
|
||||
def test_pixels_are_not_carried(self, tmp_path: Path):
|
||||
"""The reason the record walks regions instead of shipping the head: a record
|
||||
that carried the pixel stream would be the size of the image."""
|
||||
path = _noise_png(tmp_path / "big.png", size=(1200, 1200))
|
||||
record = collect_metadata_record(path)
|
||||
|
||||
idat = path.read_bytes()
|
||||
start = idat.index(b"IDAT") + 4
|
||||
assert idat[start : start + 512] not in json.dumps(record).encode()
|
||||
assert len(json.dumps(record)) < path.stat().st_size // 10
|
||||
|
||||
def test_an_unreadable_container_still_yields_a_record(self, tmp_path: Path):
|
||||
path = tmp_path / "junk.bin"
|
||||
path.write_bytes(b"\x00\x01\x02not an image at all" * 100)
|
||||
|
||||
record = collect_metadata_record(path)
|
||||
|
||||
assert record["container"] == "unknown"
|
||||
assert json.dumps(record) # serializable, and no exception on the way here
|
||||
_assert_same_verdict(path)
|
||||
|
||||
def test_a_missing_file_does_not_raise(self, tmp_path: Path):
|
||||
"""Collection runs over whatever a caller hands it, including a path that
|
||||
vanished between listing and reading."""
|
||||
record = collect_metadata_record(tmp_path / "gone.png")
|
||||
|
||||
assert record["container"] == "unknown"
|
||||
assert record["metadata_base64"] == ""
|
||||
assert record["status"] == "error"
|
||||
assert record["issues"] == [{"stage": "source", "code": "unavailable"}]
|
||||
|
||||
def test_unknown_record_schema_is_rejected(self, tmp_path: Path):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
record = collect_metadata_record(path)
|
||||
record["schema_version"] = 2
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported provenance metadata schema"):
|
||||
identify_metadata_record(record, path=path)
|
||||
|
||||
@pytest.mark.parametrize("schema_version", [True, 1.0, "1", None])
|
||||
def test_native_record_schema_requires_the_integer_one(self, tmp_path: Path, schema_version: object):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
record = collect_metadata_record(path)
|
||||
record["schema_version"] = schema_version
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported provenance metadata schema"):
|
||||
identify_metadata_record(record, path=path)
|
||||
|
||||
@pytest.mark.parametrize("status", [None, "partial", "unknown", True])
|
||||
def test_native_record_requires_complete_collection_status(self, tmp_path: Path, status: object):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
record = collect_metadata_record(path)
|
||||
if status is None:
|
||||
record.pop("status")
|
||||
else:
|
||||
record["status"] = status
|
||||
|
||||
with pytest.raises(ValueError, match="collection status"):
|
||||
identify_metadata_record(record, path=path)
|
||||
|
||||
@pytest.mark.parametrize("schema_version", [2, True, 1.0])
|
||||
def test_collection_rejects_unsupported_output_schema_before_reading(self, tmp_path: Path, schema_version: object):
|
||||
with pytest.raises(ValueError, match="Unsupported provenance metadata schema"):
|
||||
collect_metadata_record(
|
||||
tmp_path / "missing.png",
|
||||
schema_version=schema_version, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
def test_broad_forensic_record_is_not_detector_input(self, tmp_path: Path):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported metadata record type"):
|
||||
identify_metadata_record(
|
||||
{"record_type": "forensic_metadata", "schema_version": 1},
|
||||
path=path,
|
||||
)
|
||||
|
||||
def test_failed_collection_cannot_be_judged_as_an_unknown_image(self, tmp_path: Path):
|
||||
path = tmp_path / "missing.png"
|
||||
|
||||
with pytest.raises(ValueError, match="collection failed"):
|
||||
identify_metadata_record(collect_metadata_record(path), path=path)
|
||||
|
||||
|
||||
class TestReportTransport:
|
||||
def test_report_contract_is_versioned_json_and_omits_local_path(self, tmp_path: Path):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
|
||||
payload = identify_metadata_record(collect_metadata_record(path), path=path).to_dict()
|
||||
|
||||
assert payload["schema_version"] == PROVENANCE_REPORT_SCHEMA_VERSION == 1
|
||||
assert "path" not in payload
|
||||
assert json.loads(json.dumps(payload, allow_nan=False)) == payload
|
||||
|
||||
@pytest.mark.parametrize("schema_version", [2, True, 1.0])
|
||||
def test_report_rejects_unsupported_output_schema(self, tmp_path: Path, schema_version: object):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
report = identify_metadata_record(collect_metadata_record(path), path=path)
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported provenance report schema"):
|
||||
report.to_dict(schema_version=schema_version) # type: ignore[arg-type]
|
||||
|
||||
def test_convenience_entry_point_matches_explicit_sequence(self, tmp_path: Path):
|
||||
path = _noise_png(tmp_path / "plain.png")
|
||||
record = collect_metadata_record(path)
|
||||
|
||||
explicit = identify_from_evidence(evidence_from_metadata_record(record, path=path))
|
||||
convenience = identify_metadata_record(record, path=path)
|
||||
|
||||
assert convenience == explicit
|
||||
|
||||
|
||||
def test_a_webp_record_matches(tmp_path: Path):
|
||||
"""RIFF has its own walk; a chunk kept or dropped wrongly shows up here."""
|
||||
path = tmp_path / "image.webp"
|
||||
Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "WEBP", xmp=b"<x:xmpmeta>plain</x:xmpmeta>")
|
||||
|
||||
assert collect_metadata_record(path)["container"] == "webp"
|
||||
_assert_same_verdict(path)
|
||||
|
||||
|
||||
def test_a_webp_record_collects_metadata_after_a_large_frame(tmp_path: Path):
|
||||
"""The RIFF walker seeks past coded pixels instead of stopping at the head window."""
|
||||
path = tmp_path / "late.webp"
|
||||
rng = np.random.default_rng(7)
|
||||
pixels = rng.integers(0, 255, (900, 900, 3), dtype=np.uint8)
|
||||
xmp = b"<x:xmpmeta><photoshop:DigitalSourceType>trainedAlgorithmicMedia</photoshop:DigitalSourceType></x:xmpmeta>"
|
||||
Image.fromarray(pixels).save(path, "WEBP", lossless=True, xmp=xmp)
|
||||
|
||||
assert path.stat().st_size > HEAD_WINDOW
|
||||
_assert_same_verdict(path)
|
||||
|
||||
|
||||
def test_a_webp_record_does_not_carry_animation_frame_pixels(tmp_path: Path):
|
||||
"""ANMF is a coded-frame container, not a metadata chunk."""
|
||||
from remove_ai_watermarks.metadata_record import _riff_regions
|
||||
|
||||
frame = b"jumb c2pa OpenAI trainedAlgorithmicMedia" * 20
|
||||
riff = b"RIFF" + (4 + 8 + len(frame)).to_bytes(4, "little") + b"WEBP"
|
||||
riff += b"ANMF" + len(frame).to_bytes(4, "little") + frame
|
||||
|
||||
assert frame not in _riff_regions(riff)
|
||||
|
||||
|
||||
def test_an_invalid_short_riff_size_does_not_turn_the_container_into_a_trailer(tmp_path: Path):
|
||||
from remove_ai_watermarks.metadata_record import _trailer
|
||||
|
||||
path = tmp_path / "invalid.webp"
|
||||
path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPjumb c2pa trainedAlgorithmicMedia")
|
||||
|
||||
assert _trailer(path, "webp") == b""
|
||||
|
||||
|
||||
def test_the_record_never_reopens_the_source(tmp_path: Path, monkeypatch):
|
||||
"""The reason the record exists: the verdict must run where the file is not."""
|
||||
path = FIXTURES / "chatgpt-1.png" if (FIXTURES / "chatgpt-1.png").exists() else _noise_png(tmp_path / "x.png")
|
||||
record = json.loads(json.dumps(collect_metadata_record(path)))
|
||||
|
||||
def fail_if_called(*args, **kwargs):
|
||||
raise AssertionError("the record path must not open the source file")
|
||||
|
||||
monkeypatch.setattr("builtins.open", fail_if_called)
|
||||
monkeypatch.setattr(Path, "open", fail_if_called)
|
||||
monkeypatch.setattr(Image, "open", fail_if_called)
|
||||
|
||||
report = identify_from_evidence(evidence_from_metadata_record(record, path=path))
|
||||
|
||||
assert isinstance(report, ProvenanceReport)
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Tests for the pixel-forensics collector.
|
||||
|
||||
These pin the service contract and the edge cases a consumer cannot infer safely:
|
||||
a family is empty rather than wrong when the image is too small for it, one failing
|
||||
family does not lose the other five, and artifacts that identify the source image
|
||||
stay behind their opt-in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from remove_ai_watermarks.pixel_evidence import (
|
||||
AC_POSITIONS,
|
||||
PIXEL_EVIDENCE_SCHEMA_VERSION,
|
||||
PixelEvidence,
|
||||
_dct_matrix,
|
||||
dct_features,
|
||||
extract_pixel_evidence,
|
||||
is_available,
|
||||
perceptual_hash,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
FAMILIES = ("dct", "fft", "noise", "ela", "gradient", "color")
|
||||
|
||||
|
||||
def _textured(path: Path, size: tuple[int, int] = (192, 160), *, seed: int = 0) -> Path:
|
||||
"""A textured image. Flat color would make several families degenerate (zero
|
||||
residual, empty gradient histogram) and hide a real break."""
|
||||
rng = np.random.default_rng(seed)
|
||||
base = rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8)
|
||||
ramp = np.linspace(0, 255, size[0], dtype=np.uint8)[None, :, None]
|
||||
Image.fromarray(np.clip(base // 2 + ramp // 2, 0, 255).astype(np.uint8)).save(path)
|
||||
return path
|
||||
|
||||
|
||||
class TestFamilies:
|
||||
def test_every_family_is_measured_on_a_textured_image(self, tmp_path: Path):
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"))
|
||||
|
||||
assert evidence.decoded
|
||||
for family in FAMILIES:
|
||||
assert getattr(evidence, family), family
|
||||
|
||||
def test_the_same_image_measures_the_same_twice(self, tmp_path: Path):
|
||||
"""Determinism is what makes these comparable across runs and machines; the
|
||||
residual is computed in row chunks, which is exactly the kind of optimization
|
||||
that can perturb the last bits."""
|
||||
path = _textured(tmp_path / "textured.png")
|
||||
|
||||
first, second = extract_pixel_evidence(path), extract_pixel_evidence(path)
|
||||
|
||||
assert dataclasses.asdict(first) == dataclasses.asdict(second)
|
||||
|
||||
def test_source_dimensions_survive_the_downscale(self, tmp_path: Path):
|
||||
"""The recorded size is the SOURCE size, read before the 2048px cap. Recording
|
||||
the analysed size instead would still pass every statistic check, because
|
||||
those run on the downscaled array either way."""
|
||||
path = tmp_path / "oversize.png"
|
||||
Image.fromarray(np.zeros((80, 3000, 3), dtype=np.uint8)).save(path)
|
||||
|
||||
assert extract_pixel_evidence(path).decode == {"width": 3000, "height": 80}
|
||||
|
||||
def test_selected_dct_coefficients_match_the_full_transform(self):
|
||||
gray = np.random.default_rng(7).uniform(0, 255, (24, 32)).astype(np.float32)
|
||||
basis = _dct_matrix(np)
|
||||
blocks = gray.reshape(3, 8, 4, 8).swapaxes(1, 2)
|
||||
full = np.einsum("ij,abjk,lk->abil", basis, blocks, basis)
|
||||
bins = np.linspace(-20.5, 20.5, 22)
|
||||
expected = [
|
||||
np.histogram(full[:, :, row, column].ravel(), bins=bins)[0].tolist() for row, column in AC_POSITIONS
|
||||
]
|
||||
|
||||
assert dct_features(gray)["dct_ac_hist"] == expected
|
||||
|
||||
def test_perceptual_hash_matches_the_full_transform(self):
|
||||
gray = np.random.default_rng(8).uniform(0, 255, (32, 32)).astype(np.float32)
|
||||
basis = _dct_matrix(np, 32)
|
||||
coefficients = basis @ gray @ basis.T
|
||||
low = coefficients[:8, :8].ravel()[1:]
|
||||
bits = low > np.median(low)
|
||||
expected = f"{int(''.join('1' if bit else '0' for bit in bits), 2):016x}"
|
||||
|
||||
assert perceptual_hash(gray) == expected
|
||||
|
||||
|
||||
class TestDegenerateInputs:
|
||||
def test_undecodable_file_reports_the_error_and_stays_empty(self, tmp_path: Path):
|
||||
path = tmp_path / "broken.png"
|
||||
path.write_bytes(b"not an image")
|
||||
|
||||
evidence = extract_pixel_evidence(path, artifacts=True)
|
||||
|
||||
assert evidence.decoded is False
|
||||
assert "error" in evidence.decode
|
||||
assert all(getattr(evidence, family) == {} for family in FAMILIES)
|
||||
assert evidence.artifacts == {}
|
||||
|
||||
def test_image_too_small_for_a_family_leaves_it_empty(self, tmp_path: Path):
|
||||
"""8x8 is below the FFT's 32px floor but at the block DCT's. A consumer must
|
||||
not assume a fixed feature width, so the narrow case is pinned."""
|
||||
path = tmp_path / "tiny.png"
|
||||
Image.fromarray(np.arange(8 * 8 * 3, dtype=np.uint8).reshape(8, 8, 3)).save(path)
|
||||
|
||||
evidence = extract_pixel_evidence(path)
|
||||
|
||||
assert evidence.decoded is True
|
||||
assert evidence.fft == {}
|
||||
assert evidence.color != {}
|
||||
|
||||
def test_a_failing_family_does_not_lose_the_others(self, tmp_path: Path, monkeypatch):
|
||||
path = _textured(tmp_path / "textured.png")
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise ValueError(f"family failed for {path}")
|
||||
|
||||
monkeypatch.setattr("remove_ai_watermarks.pixel_evidence.color_features", boom)
|
||||
evidence = extract_pixel_evidence(path)
|
||||
|
||||
assert evidence.color == {"error": "ValueError"}
|
||||
assert evidence.status == "partial"
|
||||
assert evidence.to_dict()["status"] == "partial"
|
||||
assert evidence.dct != {}
|
||||
assert evidence.gradient != {}
|
||||
|
||||
|
||||
class TestArtifactsAreOptIn:
|
||||
"""The artifacts identify the source image -- a thumbnail is a picture, a
|
||||
perceptual hash matches one. Everything else is a scalar or a fixed-length
|
||||
histogram. That difference in kind is the reason for the flag, so the flag is
|
||||
what these guard."""
|
||||
|
||||
def test_off_by_default(self, tmp_path: Path):
|
||||
assert extract_pixel_evidence(_textured(tmp_path / "textured.png")).artifacts == {}
|
||||
|
||||
def test_on_request_it_returns_the_spatial_layer(self, tmp_path: Path):
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), artifacts=True)
|
||||
|
||||
assert set(evidence.artifacts) == {"phash", "thumbnail_jpeg_b64", "ela_map", "noise_residual", "fft_phase"}
|
||||
assert len(evidence.artifacts["phash"]) == 16
|
||||
|
||||
def test_the_thumbnail_is_a_readable_image_of_the_source(self, tmp_path: Path):
|
||||
"""Stated plainly because it is the privacy claim: this field reconstructs
|
||||
the picture, at 128px."""
|
||||
import base64
|
||||
import io
|
||||
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), artifacts=True)
|
||||
|
||||
with Image.open(io.BytesIO(base64.b64decode(evidence.artifacts["thumbnail_jpeg_b64"]))) as thumb:
|
||||
assert max(thumb.size) <= 128
|
||||
|
||||
def test_a_different_image_hashes_differently(self, tmp_path: Path):
|
||||
one = extract_pixel_evidence(_textured(tmp_path / "a.png", seed=1), artifacts=True)
|
||||
two = extract_pixel_evidence(_textured(tmp_path / "b.png", seed=2), artifacts=True)
|
||||
|
||||
assert one.artifacts["phash"] != two.artifacts["phash"]
|
||||
|
||||
def test_the_statistics_carry_no_array_payloads(self, tmp_path: Path):
|
||||
"""Without the flag nothing array-shaped may appear: that is what makes the
|
||||
default set aggregates rather than content."""
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"))
|
||||
|
||||
for family in FAMILIES:
|
||||
for key, value in getattr(evidence, family).items():
|
||||
assert isinstance(value, (int, float, str, list)), (family, key)
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
assert isinstance(item, (int, float, list)), (family, key)
|
||||
|
||||
|
||||
def test_is_available_reports_the_optional_dependency():
|
||||
assert is_available() is True # the test environment installs the pixels extra
|
||||
|
||||
|
||||
def test_evidence_is_frozen(tmp_path: Path):
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"))
|
||||
assert isinstance(evidence, PixelEvidence)
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
evidence.decode = {} # type: ignore[misc]
|
||||
|
||||
|
||||
def test_transport_contract_is_versioned_json_and_omits_path(tmp_path: Path):
|
||||
import json
|
||||
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), timings=True)
|
||||
payload = evidence.to_dict()
|
||||
|
||||
assert payload["schema_version"] == PIXEL_EVIDENCE_SCHEMA_VERSION == 1
|
||||
assert payload["status"] == "complete"
|
||||
assert "path" not in payload
|
||||
assert payload["timing_ms"]["total"] >= 0
|
||||
assert json.loads(json.dumps(payload, allow_nan=False)) == payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize("schema_version", [2, True, 1.0])
|
||||
def test_transport_rejects_unsupported_output_schema(tmp_path: Path, schema_version: object):
|
||||
evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"))
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported pixel evidence schema"):
|
||||
evidence.to_dict(schema_version=schema_version) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_decode_error_transport_does_not_leak_the_local_path(tmp_path: Path):
|
||||
import json
|
||||
|
||||
path = tmp_path / "broken.png"
|
||||
path.write_bytes(b"not an image")
|
||||
|
||||
payload = extract_pixel_evidence(path).to_dict()
|
||||
|
||||
assert payload["status"] == "error"
|
||||
assert payload["decode"]["error"] == "UnidentifiedImageError"
|
||||
assert str(path) not in json.dumps(payload)
|
||||
|
||||
|
||||
def test_timings_are_opt_in(tmp_path: Path):
|
||||
path = _textured(tmp_path / "textured.png")
|
||||
|
||||
assert extract_pixel_evidence(path).timing_ms == {}
|
||||
assert set(extract_pixel_evidence(path, artifacts=True, timings=True).timing_ms) == {
|
||||
"decode",
|
||||
"noise",
|
||||
"fft",
|
||||
"ela",
|
||||
"dct",
|
||||
"gradient",
|
||||
"color",
|
||||
"full_artifacts",
|
||||
"total",
|
||||
}
|
||||
|
||||
|
||||
def test_pixel_decode_registers_optional_heif_opener(tmp_path: Path, monkeypatch):
|
||||
registered = False
|
||||
|
||||
def mark_registered():
|
||||
nonlocal registered
|
||||
registered = True
|
||||
|
||||
monkeypatch.setattr("remove_ai_watermarks.image_io._register_heif", mark_registered)
|
||||
|
||||
extract_pixel_evidence(_textured(tmp_path / "textured.png"))
|
||||
|
||||
assert registered is True
|
||||
@@ -18,10 +18,14 @@ from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import tracemalloc
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from remove_ai_watermarks import metadata
|
||||
from remove_ai_watermarks._internal import c2pa, isobmff
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
||||
_HUGE = 0x7FFFFFFF # ~2 GiB declared length on a tiny file
|
||||
|
||||
@@ -128,3 +132,28 @@ class TestIsobmffStripFailSafe:
|
||||
assert stripped == 0
|
||||
assert cleaned == data
|
||||
assert len(cleaned) == len(data)
|
||||
|
||||
|
||||
class TestRiffLateMetadataIsBounded:
|
||||
"""A metadata scan must not become a near-full-file read because one chunk lied
|
||||
about its length. This runs on the memoized verdict path, over images from
|
||||
arbitrary sources."""
|
||||
|
||||
def _webp_with_declared_length(self, path: Path, declared: int, payload: bytes) -> Path:
|
||||
chunk = b"XMP " + declared.to_bytes(4, "little") + payload
|
||||
body = b"WEBP" + b"VP8 " + (4).to_bytes(4, "little") + b"\x00\x00\x00\x00" + chunk
|
||||
path.write_bytes(b"RIFF" + len(body).to_bytes(4, "little") + body)
|
||||
return path
|
||||
|
||||
def test_a_chunk_claiming_the_whole_file_is_clamped_to_what_remains(self, tmp_path: Path):
|
||||
target = self._webp_with_declared_length(tmp_path / "liar.webp", 1 << 30, b"AI" * 64)
|
||||
|
||||
collected = metadata._riff_late_metadata(target, 12)
|
||||
|
||||
assert collected == b"AI" * 64
|
||||
|
||||
def test_the_total_is_capped(self, tmp_path: Path):
|
||||
payload = b"x" * 4096
|
||||
target = self._webp_with_declared_length(tmp_path / "big.webp", len(payload), payload)
|
||||
|
||||
assert len(metadata._riff_late_metadata(target, 12, max_total=512)) == 512
|
||||
|
||||
Reference in New Issue
Block a user