diff --git a/.claude/rules/development.md b/.claude/rules/development.md index c58230b..e447580 100644 --- a/.claude/rules/development.md +++ b/.claude/rules/development.md @@ -23,6 +23,8 @@ Run `bash maintain.sh` from the repository root. The authoritative type gate is Boundary modules for cv2, Torch, and Diffusers may carry narrow per-file relaxations for unknown third-party types. Keep pure-logic files strict, preserve the local piexif stub, and fix real errors before widening a pragma. +From a worktree, `uv run` imports the package from the MAIN checkout -- that is where the editable install points. A script measuring a worktree's edit must insert that worktree's `src` at `sys.path[0]` and assert `module.__file__` resolves inside it, or it silently compares unmodified code against itself. + ## Model-adjacent tests Do not classify an entire module as untestable because its main path downloads a model. Keep pure behavior covered without downloads, including: @@ -78,8 +80,22 @@ rules follow, and both were broken in practice before they were written down: `TextMarkDetection.match_box` and the registry threads the detection into the mask builder; a mask path that re-runs its own sweep is how the two drift apart. +The C2PA manifest-store JSON is NOT stable across reads: the reader regenerates manifest +URNs and instance ids. Compare the derived `c2pa_info`, never the raw store. + +A third seam reaches the same verdict: `collect_metadata_record` -> +`evidence_from_metadata_record` -> `identify_from_evidence`, the path a caller uses when +collection and verdict run on different machines. Its contract is equality with +`identify(path, check_visible=False, check_invisible=False)` on the same image, and it +can break from EITHER side -- a region the collector stops walking, or a placement the +file path learns to read and the record does not. `tests/test_metadata_record.py` pins +it over the tracked fixtures; a separate local evaluation corpus catches placements the +fixtures do not cover. Change either side and re-run both. + Before changing anything in the detection path, record the detectors' exact verdicts over a local sample first and diff them after. A refactor here is only correct if that -record is byte-identical, and a green test suite does not establish that on its own. +record is byte-identical, and a green test suite does not establish that on its own. A +change that is meant to FIX detection is the exception that proves the rule: the diff +must then be exactly the files you intended to change, named in advance. Environment setup, dependency recovery, CI behavior, and fixture policy: [`../../docs/development.md`](../../docs/development.md). diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf3d1f8..a10d52f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,27 @@ jobs: - name: Run tests run: uv run pytest -q + transport-contract-python-314: + name: versioned transport contracts py3.14 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.14" + - name: Create clean runtime contract environment + run: uv venv --python 3.14 .venv-contract + - name: Install runtime contract dependencies + run: >- + uv pip install --python .venv-contract/bin/python + ".[pixels,heif]" "numpy>=2,<3" "pytest>=8" + - name: Run versioned transport contract tests + run: >- + .venv-contract/bin/python -m pytest -q + tests/test_forensic_metadata.py + tests/test_metadata_record.py + tests/test_pixel_evidence.py + video-e2e: name: video full-clip end-to-end runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 567e892..598c94d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,7 @@ Thumbs.db *.swp *.swo -# SynthID corpus reference fills (synthetic black/white calibration tiles, -# regenerable; the labeled pos/neg/cleaned images ARE tracked, see README) +# Generated calibration inputs data/synthid_corpus/refs/ # Reference materials @@ -35,11 +34,7 @@ yolov8n.pt .claude/settings.local.json .claude/scheduled_tasks.lock -# Visible-watermark alpha calibration. The solid black/gray/white CAPTURES are -# committed (content-free: a solid colour + the watermark; the source for -# scripts/visible_alpha_solve.py so the alpha assets are reproducible). The -# synthetic seeds (regenerable) and any real-content validation download (a real -# generated scene, kept local for privacy) are NOT committed. +# Generated calibration inputs and local evaluation artifacts data/doubao_capture/seeds/ data/jimeng_capture/seeds/ data/jimeng_capture/captures/jimeng_content_*.png @@ -48,10 +43,8 @@ data/gemini_capture/captures/gemini_content_*.png data/samsung_capture/seeds/ data/samsung_capture/captures/samsung_content_* -# Leftover GFPGAN weights dir from the retired face-restore experiments -# (GFPGAN wrote RetinaFace/parsing weights to a CWD ./gfpgan/weights/ working -# dir on first use). Runtime artifact, never committed. +# Runtime model artifacts gfpgan/ -# Local-only working data for analysis (not a committed corpus; never tracked) +# Local evaluation data .local-eval/ diff --git a/docs/module-internals.md b/docs/module-internals.md index ed4a425..c827690 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -340,7 +340,16 @@ metadata scanners and `remove_ai_metadata`. Key contracts: -- `scan_head` is the shared cached input for bounded byte scans. +- `scan_head` is the shared cached input for bounded byte scans. It fills the buffer + in two layers. Structural readers first, one per container, each seeking past the + pixel payload to reach metadata placed beyond the window: `isobmff.scan_c2pa_region`, + `_png_late_metadata`, `_riff_late_metadata`. A decoder-backed fallback last, + `_decoder_visible_text`, for metadata the raw bytes do not spell at all — a + zlib-compressed PNG `zTXt` packet is readable only after inflation. The layers are + ordered that way because the structural readers work on files no decoder can open. +- A C2PA reader failure is logged at warning, not debug. It returns the same `None` as + a file with no manifest, so nothing downstream can distinguish "no credentials" from + "the credentials could not be read", and the second silently downgrades a verdict. - JPEG stripping walks metadata segments and preserves the entropy-coded image scan. - ISOBMFF containers use @@ -379,13 +388,90 @@ metadata extraction from verdict logic: - `extract_provenance_evidence` reads the supported metadata signals into `ProvenanceEvidence`. - `evidence_from_metadata_record` normalizes an externally collected nested - metadata record into the same evidence type without file access. Diagnostic - values under `error` and `kind` are excluded from evidence while nested raw - bytes remain available through encoded binary fields. -- `identify_from_evidence` evaluates that evidence without reopening the source. + metadata record into the same evidence type without file access. Versioned native + records accept only source-derived fields; filenames, hashes, timings, errors, + prior verdicts, and pixel results cannot become evidence. Unknown native schema + versions and other record types are rejected. +- The vendor registries are matched over `_metadata_region(head)`, not the whole scan + buffer: they see the container's metadata and not its coded pixels. The tokens are + raw substrings and the shortest are four and five bytes, so over a megabyte of + compressed data one turns up by chance, and the entry it hits may assert AI. The + trim happens only when the container parses -- a malformed or unknown one is left + whole, because dropping real evidence to avoid a chance match is the wrong trade. +- `identify_from_evidence` evaluates that evidence without reopening the source. Rules + that decide a verdict live here, not in extraction: extraction has two + implementations, and a rule in only one of them is a rule the other lacks. The + SynthID proxy is the worked example — its structured form comes from the manifest, + and the byte-scan fallback for containers no parser reaches runs in the verdict, so + both extractors reach the same answer. It did not, and the record path silently + reported no SynthID for images the file path flagged. - `identify` preserves the path-based API and adds the optional registered visible-mark and open invisible-watermark decoders after extraction. +### Portable metadata record + +[`metadata_record.py`](../src/remove_ai_watermarks/metadata_record.py) produces the +record `evidence_from_metadata_record` consumes, so collection and verdict can run +on different machines. Its contract is equality with the file path, and the three +defects found while establishing that equality are the reason each rule exists: + +- It walks the file's RAW head, never the `scan_head` buffer. That buffer is the + head concatenated with late metadata payloads, so a structural walk runs off the + end of the real head and parses appended bytes as chunks, inflating the record and + creating false signals. +- Samsung Galaxy AI splits its evidence: the `PhotoEditor_Re_Edit_Data` marker sits + in the post-EOI trailer while the `genAIType` value it is gated on can sit inside + the entropy-coded scan. A marked file therefore keeps the whole tail window, not + just the trailer. +- PIL's info keys are emitted in the file path's own candidate order + (`Software`, `Source`, `Title`, `Description`, then EXIF). `generator_from_metadata` + returns the FIRST candidate carrying a known token, so preserving candidate order + is part of verdict equivalence. + +The transport is independently versioned as `provenance_metadata` schema 1. Native +records require the exact integer schema version and a `complete` status. Source +read failures are explicit error records and cannot be judged. WebP walks the full +declared RIFF container by seeking over `VP8`, `VP8L`, `ALPH`, and `ANMF`, so late +XMP/C2PA remains visible without shipping coded frames or parsing appended trailer +bytes as chunks. + +Pixel forensics are deliberately absent: the provenance path does not read them. +Verdict equivalence is checked over tracked fixtures and a separate local evaluation +corpus. + +### Broad forensic metadata + +[`forensic_metadata.py`](../src/remove_ai_watermarks/forensic_metadata.py) owns the +wide metadata-only inspection record: hashes and timestamps, full EXIF/IPTC, C2PA, +container inventories, bounded binary metadata, and embedded-thumbnail forensics. +It is a separate `forensic_metadata` record type and is deliberately rejected by the +provenance normalizer. Integration code publishes the strict +`ProvenanceReport.to_dict()` alongside it rather than letting operational fields or +derived results influence detection. + +### Pixel forensics + +[`pixel_evidence.py`](../src/remove_ai_watermarks/pixel_evidence.py) measures six +families of scale-robust pixel statistics (block-DCT histograms and Benford +deviation, FFT band energies and CFA peaks, high-pass residual, error level, +gradient, color) in a single decode, sharing the intermediate maps between them. + +It remains independent of verdict and removal. `PixelEvidence.to_dict()` is the +versioned service boundary: it omits the local path, exposes complete/partial/error +status, keeps exception details in logs, and can include opt-in per-stage timings. + +The provenance metadata collector, broad forensic collector, provenance report, +and pixel report all accept an explicit output `schema_version`. Package releases +may add an output schema while retaining older serializers, so a rolling consumer +can keep requesting the version it already understands. Within one schema, changes +are additive; existing fields, types, meanings, signal names, and watermark labels +remain stable. Unsupported selections raise before a different shape is returned. + +`artifacts=True` additionally returns the spatial layer: a perceptual hash, a 128px +JPEG thumbnail, and coarse ELA, residual and phase maps. Those identify the source +image rather than describe it, which is why they are opt-in and a separate field: a +caller storing them is handling image content, not statistics about it. + The DWT-DCT detector and the visible-mark stage share a single decode of the source, held by a per-call `_SharedDecode`. It exposes two accessors because the two arms need diff --git a/docs/python-api.md b/docs/python-api.md index a28353e..8839c7d 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -182,8 +182,61 @@ evidence = extract_provenance_evidence(Path("input.png")) report = identify_from_evidence(evidence) ``` -If metadata was collected by another component, normalize its nested record -without reopening the original file: +### Collect once, judge elsewhere + +`collect_metadata_record` splits the two halves apart: it is the only step that +touches the file, and it returns a JSON-serializable record the verdict can be +built from on another machine, in another process, or later. + +```python +import json + +from remove_ai_watermarks.identify import identify_metadata_record +from remove_ai_watermarks.metadata_record import collect_metadata_record + +record = collect_metadata_record(Path("input.png"), schema_version=1) # reads the file +blob = json.dumps(record) # ship it anywhere + +report = identify_metadata_record(json.loads(blob), path=Path("input.png")) # reads nothing +payload = report.to_dict(schema_version=1) # versioned JSON contract +``` + +The collection record has `record_type="provenance_metadata"`, +`schema_version=1`, and a `status`. A vanished or unreadable source produces an +`error` record with structured `issues`; `identify_metadata_record` rejects that +record instead of turning a collection failure into an unknown-image verdict. +Unknown schema versions, non-integer aliases, and native records without a +`complete` collection status are rejected explicitly. + +The verdict is the same one `identify(path, check_visible=False, +check_invisible=False)` returns for that file. That equality is the record's whole +contract and is verified over the tracked provenance fixtures and a separate local +evaluation corpus. `ProvenanceReport.to_dict()` is the stable service boundary: it +adds a `schema_version`, contains only JSON-safe values, and deliberately omits the +local source path. + +Package and transport versions evolve independently. Long-lived consumers should +request the schema they implement, as above, instead of assuming the installed +package's latest schema. Within schema 1, existing fields, types, meanings, +`signals[].name` values, and `watermarks[]` labels remain compatible; releases may +add fields that consumers must ignore. A breaking change requires a new schema while +the schema 1 serializer remains available for rolling upgrades. Asking a release for +an unsupported schema raises `ValueError` rather than silently returning another +shape. + +A record carries metadata regions, not the primary coded-pixel stream: marker +segments before the JPEG scan, every PNG chunk except `IDAT`, RIFF chunks except the +coded image, the ISOBMFF provenance boxes, the container's trailer, the parsed EXIF tags the +verdict reads by name, PIL's info mapping, and the C2PA manifest store. Record size +is bounded by those metadata regions and trailers; images with large embedded +manifests naturally produce larger records. + +The `path` argument is metadata: it labels the report and is never opened by +either function, so a record collected elsewhere can be judged against a path that +does not exist locally. + +If metadata was collected by another component instead, normalize its nested +record the same way: ```python from remove_ai_watermarks.identify import ( @@ -199,13 +252,54 @@ evidence = evidence_from_metadata_record(record, path=Path("input.png")) report = identify_from_evidence(evidence) ``` -The normalizer recursively preserves text and byte values. It also decodes +Unversioned third-party records are normalized recursively for compatibility. The +normalizer preserves text and byte values. It also decodes strings prefixed with `hex:` and fields named `base64` or ending in -`_base64`. Diagnostic values under `error` and `kind` are ignored because they -describe the collector rather than the source file. Pass a C2PA manifest-store -dictionary in `record["c2pa_store"]`, or through the explicit +`_base64`. Diagnostic, transport, timing, hash, provenance-result, and pixel-result +subtrees are ignored because they describe the collector or a derived result rather +than the source file. A versioned portable record is stricter still: only +`metadata_base64`, `tail_base64`, `pil`, `exif`, and `c2pa_store` are accepted as +source evidence. Other `record_type` values are rejected, so do not pass a broad +forensic inspection record to this API. Pass a C2PA manifest-store dictionary in +`record["c2pa_store"]`, or through the explicit `c2pa_manifest_store` argument. +### Broad metadata inspection + +`collect_forensic_metadata` provides the wide metadata-only record used by forensic +inspection and migration adapters. It preserves hashes and timestamps, full EXIF and +IPTC, C2PA, container inventories, bounded raw metadata payloads, and embedded +thumbnail forensics. It does not calculate a provenance verdict or pixel statistics. + +```python +from remove_ai_watermarks.forensic_metadata import collect_forensic_metadata + +record = collect_forensic_metadata(Path("input.png"), schema_version=1) +assert record["record_type"] == "forensic_metadata" +``` + +This record is intentionally not accepted by `identify_metadata_record`. Collect the +small strict provenance record separately and publish the resulting +`ProvenanceReport.to_dict()` as the detector contract. + +### Pixel evidence + +`extract_pixel_evidence` decodes once and calculates the DCT, FFT, residual, ELA, +gradient, and color families. Its versioned `to_dict()` result has a semantic +`status`: `complete`, `partial` when an individual family failed, or `error` when the +source could not be decoded. Transported errors contain only the exception class, so +local paths stay in the caller's logs rather than crossing the service boundary. + +```python +from remove_ai_watermarks.pixel_evidence import extract_pixel_evidence + +pixels = extract_pixel_evidence(Path("input.png"), artifacts=False, timings=True) +payload = pixels.to_dict(schema_version=1) +``` + +Timings and spatial artifacts are opt-in. Artifacts include image-identifying data +such as a thumbnail and perceptual hash; aggregate feature families do not. + `identify_from_evidence` does not reopen the source file by default: it evaluates metadata only, and registered visible marks and pixel-backed invisible watermarks remain in the path-based `identify` call. diff --git a/docs/release-and-distribution.md b/docs/release-and-distribution.md index e034c9d..9e1b17d 100644 --- a/docs/release-and-distribution.md +++ b/docs/release-and-distribution.md @@ -71,10 +71,10 @@ The source distribution uses an explicit allowlist for `/src`, `/LICENSE`, `[tool.hatch.build.targets.sdist]` in `pyproject.toml`. It also defensively excludes `/data`, `/tmp`, and `/.sc`. Keep both controls: calibration captures, test corpora, generated research outputs, and local session state do not belong -in the published package archive. `.gitignore` covers `tmp/` and `.sc/`, so those -never reach a commit, but ignore rules are not the build boundary -- hatchling may -include untracked files, and `data/` is deliberately tracked, so the sdist exclude -is the only control keeping it out of the archive. +in the published package archive. Hatchling always adds the root `.gitignore` to +the sdist, so keep its comments generic and free of local operational context. +Ignore rules are not the build boundary: `data/` is deliberately tracked, while +the sdist configuration keeps it and the other excluded paths out of the archive. ## Build backend @@ -97,6 +97,15 @@ write access. ## Release verification +Forensic transports are versioned independently from the package. Before publishing +a change to provenance metadata, provenance reports, broad forensic metadata, or +pixel evidence, run their schema 1 contract tests. Additive fields are compatible; +renaming a field, changing its type or meaning, changing a signal name or watermark +label, or removing a field requires a new output schema. Add the new serializer +without removing schema 1 so long-lived consumers can update separately. A package +release must never silently substitute its latest schema when a caller explicitly +requests an older supported one. + After publication, verify: - both wheel and source distribution exist on PyPI; diff --git a/docs/watermarking-landscape.md b/docs/watermarking-landscape.md index b624343..586ba57 100644 --- a/docs/watermarking-landscape.md +++ b/docs/watermarking-landscape.md @@ -68,7 +68,7 @@ payloads. Removal remuxes either container through ffmpeg with stream copy. Metadata stripping for supported audio containers is a separate implemented path. -**Box detection window — now handled (v0.6.8):** detection no longer relies on a fixed first-MB read. `metadata.scan_head(path, size)` reads the first `size` bytes and, for ISOBMFF, appends the payloads of late provenance boxes found by `isobmff.scan_c2pa_region` (a file-seeking top-level box walker that skips past `mdat` by size without reading it), so a C2PA/AIGC/IPTC manifest placed AFTER a large `mdat` in a streaming/non-faststart MP4 is now caught. Every C2PA/marker byte scan (`has_ai_metadata`, `aigc_label`, `iptc_ai_system`, `synthid_source`, `exif_generator` XMP, `get_ai_metadata` soft-binding, and `identify`) goes through `scan_head`; for PNG it likewise appends the payloads of `tEXt` / `iTXt` / `zTXt` / `eXIf` / `iCCP` chunks that start beyond the window (`_png_late_metadata`, seeking past `IDAT`), which is how a TC260 AIGC label appended after the pixel stream is caught; for every other input, and for any file that fits inside `size`, it is exactly `f.read(size)`. +**Box detection window — now handled (v0.6.8):** detection no longer relies on a fixed first-MB read. `metadata.scan_head(path, size)` reads the first `size` bytes and, for ISOBMFF, appends the payloads of late provenance boxes found by `isobmff.scan_c2pa_region` (a file-seeking top-level box walker that skips past `mdat` by size without reading it), so a C2PA/AIGC/IPTC manifest placed AFTER a large `mdat` in a streaming/non-faststart MP4 is now caught. Every C2PA/marker byte scan (`has_ai_metadata`, `aigc_label`, `iptc_ai_system`, `synthid_source`, `exif_generator` XMP, `get_ai_metadata` soft-binding, and `identify`) goes through `scan_head`; for PNG it likewise appends the payloads of `tEXt` / `iTXt` / `zTXt` / `eXIf` / `iCCP` chunks that start beyond the window (`_png_late_metadata`, seeking past `IDAT`), which is how a TC260 AIGC label appended after the pixel stream is caught; for WebP it appends the `EXIF` / `XMP ` / `ICCP` / `C2PA` chunks past the window (`_riff_late_metadata`, stepping over the coded image), which is how an IPTC "Made with AI" tag stored after the pixels is caught; and for a file at least `size` bytes long it finally appends the metadata text the decoder reaches but a raw read cannot (`_decoder_visible_text`), which covers a compressed PNG `zTXt` packet no byte scan can spell; for any file that fits inside `size`, it is exactly `f.read(size)`. Native TC260 MP4/MOV tags do not live in those top-level provenance boxes. `tc260_aigc_payloads` separately seeks through `moov.udta.meta.keys/ilst`, so diff --git a/pyproject.toml b/pyproject.toml index cc14b2d..c93695e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "remove-ai-watermarks" -version = "0.25.0" +version = "0.26.0" description = "AI watermark remover for visible, invisible, and provenance marks in images and video" readme = "README.md" requires-python = ">=3.10.1" @@ -37,6 +37,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Multimedia :: Graphics", "Topic :: Multimedia :: Graphics :: Graphics Conversion", "Topic :: Scientific/Engineering :: Artificial Intelligence", diff --git a/scripts/ai_score.py b/scripts/ai_score.py deleted file mode 100644 index d2f06c2..0000000 --- a/scripts/ai_score.py +++ /dev/null @@ -1,436 +0,0 @@ -"""AI-generation scorer for scanned datasets. - -Trains a gradient-boosted classifier on the metadata-derived labels of a -scan_dataset.py output and scores every file, WITHOUT needing the -metadata to be present at scoring time: the features are pixel and -container statistics, so a metadata-stripped file still gets a score. -The model is distribution-specific by design; evaluate it again before -using it on an unrelated target distribution. - -Modes: - uv run --with scikit-learn python scripts/ai_score.py train [v1|v2] - uv run --with scikit-learn python scripts/ai_score.py score - - is a glob of scan_dataset shards, e.g. 'data/scan/part_*.jsonl'. -score output: one JSON line per file with file, sha256, ai_score, and the -label evidence when a metadata label existed (for monitoring drift). -""" - -import glob -import json -import math -import pickle -import sys -from collections import defaultdict -from pathlib import Path -from typing import Any, Literal - -import numpy as np - -# AI-generator names in C2PA claim_generator_info (metadata-level truth). -AI_GENERATORS = ( - "openai", - "adobe firefly", - "adobe_firefly", - "microsoft responsible ai", - "microsoft_designer", - "black forest labs", - "fal-ai", - "bria", - "chatgpt", - "stability", - "dreamina", - "canva", -) -# Human-origin software (negative evidence, NOT proof by itself). -HUMAN_SOFTWARE = ( - "photoshop", - "lightroom", - "picsart", - "snapseed", - "paint.net", - "gimp", - "capture one", - "meitu", - "xingtu", - "snow", -) - -_FEAT_SCALAR = [ - ("noise", "noise_std"), - ("noise", "noise_kurtosis"), - ("fft", "cfa_peak"), - ("ela", "ela_mean"), - ("ela", "ela_p95"), - ("gradient", "laplacian_var"), - ("color", "saturation_mean"), - ("color", "value_mean"), - ("dct", "benford_mad"), -] -FeatureSchema = Literal["v1", "v2"] -_FFT_BANDS = 8 -_GRADIENT_BINS = 10 -_COLOR_BINS = 64 -_DCT_AC_POSITIONS = 8 -_DCT_AC_BINS = 21 -_JPEG_QUANT_TABLES = 2 -_JPEG_QUANT_VALUES = 64 -_SCORE_BATCH = 4096 - -_V1_FEATURE_NAMES = ( - *(key for _, key in _FEAT_SCALAR), - *(f"fft_band_energy_{index}" for index in range(_FFT_BANDS)), - *(f"gradient_hist_{index}" for index in range(_GRADIENT_BINS)), - *(f"color_hist_4x4x4_{index}" for index in range(_COLOR_BINS)), - "jpeg_444", - "jpeg_progressive", - "megapixels", - "aspect_ratio", - "format_jpeg", - "format_png", -) -_V2_EXTRA_FEATURE_NAMES = ( - "cfa_peak_0", - "cfa_peak_1", - *(f"dct_ac_{position}_{bin_index}" for position in range(_DCT_AC_POSITIONS) for bin_index in range(_DCT_AC_BINS)), - *(f"jpeg_quant_{table}_{index}" for table in range(_JPEG_QUANT_TABLES) for index in range(_JPEG_QUANT_VALUES)), - "jpeg_quant_table_count", - "jpeg_huffman_table_count", - "jpeg_huffman_total_bytes", - "jpeg_scan_count", - "jpeg_restart_interval", - "jpeg_precision_bits", - "jpeg_adobe_transform", - "jpeg_jfif_present", - "format_webp", - "format_isobmff", - "format_other", -) -_SCHEMA_FEATURE_NAMES: dict[str, tuple[str, ...]] = { - "v1": _V1_FEATURE_NAMES, - "v2": _V1_FEATURE_NAMES + _V2_EXTRA_FEATURE_NAMES, -} - - -def parse_schema(value: Any) -> FeatureSchema: - """Validate a feature-schema token (CLI argument or model-bundle field).""" - if value == "v1" or value == "v2": - return value - raise ValueError(f"Unknown feature schema: {value!r}") - - -def feature_names(schema: FeatureSchema = "v1") -> tuple[str, ...]: - """Return the stable ordered feature names for a model schema.""" - return _SCHEMA_FEATURE_NAMES[parse_schema(schema)] - - -def _fixed_values(values: Any, count: int, *, normalize: bool = False) -> list[float]: - """Return a fixed-width float vector, padding absent values with NaN.""" - if not isinstance(values, (list, tuple)): - return [float("nan")] * count - result = [float(value) for value in values[:count]] - if normalize and result: - total = sum(value for value in result if math.isfinite(value)) - if total > 0: - result = [value / total for value in result] - return result + [float("nan")] * (count - len(result)) - - -def _optional_float(mapping: dict[str, Any], key: str) -> float: - value = mapping.get(key) - return float(value) if isinstance(value, (int, float)) else float("nan") - - -def label_of(record: dict[str, Any]) -> int | None: - """1 = strong metadata AI, 0 = human-origin, None = unlabeled.""" - store = record.get("c2pa_store") or {} - for m in (store.get("manifests") or {}).values(): - for cgi in m.get("claim_generator_info") or []: - if any(g in str(cgi.get("name", "")).lower() for g in AI_GENERATORS): - return 1 - for a in m.get("assertions") or []: - d = a.get("data") if isinstance(a.get("data"), dict) else {} - if "trainedAlgorithmicMedia" in str(d.get("digitalSourceType", "")): - return 1 - for c in record.get("png_chunks", []): - t = c.get("text", "") - kw = t.split("\x00")[0] - if kw in ("prompt", "workflow", "parameters") or "AIGC" in t[:80]: - return 1 - exif = record.get("exif", {}) - z = exif.get("0th") or {} - e = exif.get("Exif") or {} - if z.get("Make") and z.get("Model") and (e.get("MakerNote") or e.get("LensModel") or e.get("LensMake")): - return 0 - for c in record.get("png_chunks", []): - if c.get("apple_screenshot_marker"): - return 0 - for v in (record.get("iptc") or {}).values(): - if str(v).strip() == "Screenshot": - return 0 - if any(t in str(z.get("Software", "")).lower() for t in HUMAN_SOFTWARE): - return 0 - return None - - -def features_of(record: dict[str, Any], *, schema: FeatureSchema = "v1") -> list[float]: - """The structural feature vector (pixel + container stats, no metadata).""" - v = [float((record.get(s) or {}).get(k, np.nan)) for s, k in _FEAT_SCALAR] - fft = record.get("fft") or {} - gradient = record.get("gradient") or {} - color = record.get("color") or {} - v.extend(_fixed_values(fft.get("fft_band_energy"), _FFT_BANDS)) - v.extend(_fixed_values(gradient.get("gradient_hist"), _GRADIENT_BINS)) - v.extend(_fixed_values(color.get("color_hist_4x4x4"), _COLOR_BINS)) - jf = record.get("jpeg_forensics") or {} - v.append(1.0 if jf.get("subsampling") == "4:4:4" else 0.0) - v.append(1.0 if jf.get("progressive") else 0.0) - pil = record.get("pil") or {} - w, h = pil.get("width") or 0, pil.get("height") or 0 - v += [float(w * h) / 1e6, float(w) / max(h, 1)] - fmt = record.get("content_format") - v += [1.0 if fmt == "jpeg" else 0.0, 1.0 if fmt == "png" else 0.0] - if schema == "v2": - v.extend(_fixed_values(fft.get("cfa_peaks"), 2)) - dct_hist = (record.get("dct") or {}).get("dct_ac_hist") - for position in range(_DCT_AC_POSITIONS): - values = dct_hist[position] if isinstance(dct_hist, list) and position < len(dct_hist) else None - v.extend(_fixed_values(values, _DCT_AC_BINS, normalize=True)) - quant_tables = jf.get("quant_tables") - for table in range(_JPEG_QUANT_TABLES): - values = quant_tables.get(str(table)) if isinstance(quant_tables, dict) else None - v.extend(_fixed_values(values, _JPEG_QUANT_VALUES)) - is_jpeg = fmt == "jpeg" - huffman_tables = jf.get("huffman_tables_hex") - huffman_values = huffman_tables if isinstance(huffman_tables, list) else [] - v.extend( - [ - float(len(quant_tables)) if isinstance(quant_tables, dict) else (0.0 if is_jpeg else float("nan")), - float(len(huffman_values)) if is_jpeg else float("nan"), - ( - float(sum(len(value) // 2 for value in huffman_values if isinstance(value, str))) - if is_jpeg - else float("nan") - ), - _optional_float(jf, "scan_count"), - _optional_float(jf, "restart_interval"), - _optional_float(jf, "precision_bits"), - _optional_float(jf, "adobe_transform"), - 1.0 if jf.get("jfif") else (0.0 if is_jpeg else float("nan")), - 1.0 if fmt == "webp" else 0.0, - 1.0 if isinstance(fmt, str) and fmt.startswith("isobmff:") else 0.0, - ( - 1.0 - if fmt is not None and fmt not in {"jpeg", "png", "webp"} and not str(fmt).startswith("isobmff:") - else 0.0 - ), - ] - ) - elif schema != "v1": - raise ValueError(f"Unknown feature schema: {schema}") - expected = len(feature_names(schema)) - if len(v) != expected: - raise ValueError(f"Feature schema {schema} produced {len(v)} values; expected {expected}") - return v - - -def grouped_stratified_split( - labels: np.ndarray, - hashes: np.ndarray, - *, - test_size: float = 0.3, - random_state: int = 0, -) -> tuple[np.ndarray, np.ndarray]: - """Split records by hash while preserving label balance across hash groups.""" - if len(labels) != len(hashes): - raise ValueError("labels and hashes must have equal length") - if not 0 < test_size < 1: - raise ValueError("test_size must be between 0 and 1") - groups: dict[str, list[int]] = defaultdict(list) - group_labels: dict[str, int] = {} - for index, (label, digest) in enumerate(zip(labels, hashes, strict=True)): - key = str(digest) if digest else f"__row_{index}" - int_label = int(label) - if key in group_labels and group_labels[key] != int_label: - raise ValueError(f"Hash {key!r} has conflicting labels") - group_labels[key] = int_label - groups[key].append(index) - keys_by_label: dict[int, list[str]] = defaultdict(list) - for key, label in group_labels.items(): - keys_by_label[label].append(key) - rng = np.random.default_rng(random_state) - test_set: set[str] = set() - for class_keys in keys_by_label.values(): - if len(class_keys) < 2: - raise ValueError("Each label needs at least two distinct hash groups") - shuffled = np.asarray(class_keys) - rng.shuffle(shuffled) - test_count = min(max(round(len(shuffled) * test_size), 1), len(shuffled) - 1) - test_set.update(shuffled[:test_count].tolist()) - train_set = set(groups) - test_set - train = np.asarray([index for key, indices in groups.items() if key in train_set for index in indices]) - test = np.asarray([index for key, indices in groups.items() if key in test_set for index in indices]) - return np.sort(train), np.sort(test) - - -def temporal_holdout_split( - dates: np.ndarray, - hashes: np.ndarray, - *, - train_fraction: float = 0.7, -) -> tuple[np.ndarray, np.ndarray, str]: - """Split chronologically and remove later copies of training hashes.""" - if len(dates) != len(hashes): - raise ValueError("dates and hashes must have equal length") - if not 0 < train_fraction < 1: - raise ValueError("train_fraction must be between 0 and 1") - unique_dates = sorted({str(date) for date in dates}) - if len(unique_dates) < 2: - raise ValueError("Temporal holdout needs at least two distinct dates") - cutoff_index = min(int(len(unique_dates) * train_fraction), len(unique_dates) - 1) - cutoff = unique_dates[cutoff_index] - train = np.flatnonzero(dates < cutoff) - test = np.flatnonzero((dates >= cutoff) & ~np.isin(hashes, hashes[train])) - return train, test, cutoff - - -def model_schema(bundle: dict[str, Any]) -> FeatureSchema: - """Read a model's feature schema, defaulting legacy bundles to v1.""" - return parse_schema(bundle.get("feature_schema", "v1")) - - -def iter_records(pattern: str) -> Any: - for path in sorted(glob.glob(pattern)): - with open(path) as fh: - for line in fh: - yield json.loads(line) - - -def cmd_train(pattern: str, model_path: str, schema: FeatureSchema = "v2") -> None: - from sklearn.ensemble import HistGradientBoostingClassifier - from sklearn.impute import SimpleImputer - from sklearn.metrics import average_precision_score, roc_auc_score - - # keep the feature vector, not the record: a parsed scan record is an order of - # magnitude larger than the row it collapses to. - labeled: dict[str, tuple[list[float], int, str]] = {} - missing_hash_index = 0 - for r in iter_records(pattern): - if "noise" not in r: - continue - lab = label_of(r) - if lab is None: - continue - digest = r.get("sha256") - if not digest: - digest = f"__missing_{missing_hash_index}" - missing_hash_index += 1 - date = Path(r["file"]).parent.name - previous = labeled.get(str(digest)) - if previous is not None and previous[1] != lab: - raise ValueError(f"Hash {digest!r} has conflicting labels") - if previous is None or date < previous[2]: - labeled[str(digest)] = (features_of(r, schema=schema), lab, date) - - rows, labels, row_dates = zip(*labeled.values(), strict=True) - X = np.asarray(rows) - y = np.asarray(labels) - dates = np.asarray(row_dates) - hashes = np.asarray(list(labeled)) - print( - f"unique labeled: {len(y)} " - f"(pos {int(y.sum())}, neg {int((1 - y).sum())}, features {X.shape[1]}, schema {schema})" - ) - - def fit(train: np.ndarray) -> tuple[Any, Any]: - rows = X[train] - imputer = SimpleImputer(strategy="median") - classifier = HistGradientBoostingClassifier( - random_state=0, - max_iter=300, - l2_regularization=5.0, - early_stopping=False, - ) - classifier.fit(imputer.fit_transform(rows), y[train]) - return imputer, classifier - - train, test = grouped_stratified_split(y, hashes, test_size=0.3, random_state=0) - imp, clf = fit(train) - p = clf.predict_proba(imp.transform(X[test]))[:, 1] - print(f"SHA-grouped holdout: AUC {roc_auc_score(y[test], p):.4f} | AP {average_precision_score(y[test], p):.4f}") - - temporal_train, temporal_test, cutoff = temporal_holdout_split(dates, hashes) - if len(temporal_train) > 100 and len(temporal_test) > 100: - imp_t, clf_t = fit(temporal_train) - pt = clf_t.predict_proba(imp_t.transform(X[temporal_test]))[:, 1] - auc_t = roc_auc_score(y[temporal_test], pt) - ap_t = average_precision_score(y[temporal_test], pt) - print(f"temporal (<{cutoff}): AUC {auc_t:.4f} | AP {ap_t:.4f}") - - all_indices = np.arange(len(y)) - imp, clf = fit(all_indices) - bundle = { - "imputer": imp, - "clf": clf, - "feature_schema": schema, - "feature_names": feature_names(schema), - } - with open(model_path, "wb") as f: - pickle.dump(bundle, f) - print(f"model written: {model_path}") - - -def cmd_score(pattern: str, model_path: str, out_path: str) -> None: - # the model file is produced locally by `train`; do not load third-party pickles - with open(model_path, "rb") as f: - bundle = pickle.load(f) # noqa: S301 - imp, clf = bundle["imputer"], bundle["clf"] - schema = model_schema(bundle) - stored_names = bundle.get("feature_names") - if stored_names is not None and tuple(stored_names) != feature_names(schema): - raise ValueError(f"Model feature names do not match schema {schema}") - n = 0 - batch_rows: list[list[float]] = [] - batch_meta: list[dict[str, Any]] = [] - - def flush(out: Any) -> None: - """Score one batch: per-record predict_proba is dominated by call overhead.""" - nonlocal n - if not batch_rows: - return - scores = clf.predict_proba(imp.transform(np.asarray(batch_rows)))[:, 1] - for meta, score in zip(batch_meta, scores, strict=True): - out.write(json.dumps({**meta, "ai_score": round(float(score), 4)}) + "\n") - n += len(batch_rows) - batch_rows.clear() - batch_meta.clear() - print(f" {n}", flush=True) - - with open(out_path, "w") as out: - for r in iter_records(pattern): - if "noise" not in r: - continue - batch_rows.append(features_of(r, schema=schema)) - batch_meta.append({"file": r["file"], "sha256": r.get("sha256"), "metadata_label": label_of(r)}) - if len(batch_rows) >= _SCORE_BATCH: - flush(out) - flush(out) - print(f"scored {n} -> {out_path}") - - -def main() -> None: - if len(sys.argv) < 2 or sys.argv[1] not in ("train", "score"): - print(__doc__) - sys.exit(2) - if sys.argv[1] == "train" and len(sys.argv) in (4, 5): - schema = parse_schema(sys.argv[4]) if len(sys.argv) == 5 else "v2" - cmd_train(sys.argv[2], sys.argv[3], schema) - elif sys.argv[1] == "score" and len(sys.argv) == 5: - cmd_score(sys.argv[2], sys.argv[3], sys.argv[4]) - else: - print(__doc__) - sys.exit(2) - - -if __name__ == "__main__": - main() diff --git a/scripts/detection_timing.py b/scripts/detection_timing.py new file mode 100644 index 0000000..2d1838a --- /dev/null +++ b/scripts/detection_timing.py @@ -0,0 +1,241 @@ +"""Per-method wall time for metadata extraction and the verdict built on it. + +WHY THIS EXISTS + The detection path is documented by CAPABILITY -- which signals it reads, in what + order, with what confidence -- but not by COST. Batch throughput and the question + "which probe dominates, and does it depend on container or on file size" have never + been measured. + +WHAT IT MEASURES + Only the file-backed half: metadata extraction, then the verdict evaluated on the + extracted evidence. The pixel-backed detectors are deliberately out of scope. + + COLD pass: one measurement on a file the process has never touched -- + ``extract_provenance_evidence``. Only the first read of a file is genuinely cold, + so it buys exactly one number, and that is the one a single-shot run pays. + + WARM pass, with the filesystem cache now hot: + 1. ``extract_provenance_evidence`` as a whole. + 2. Its nine components, timed IN THE ORDER the dataclass constructs them and with + the per-file caches cleared once beforehand -- so ``extract_c2pa_info`` carries + the Rust manifest reader and the later ``get_ai_metadata`` sees the same warm + cache it sees in production. Timing them in any other order moves that cost to + a different row and flatters whichever ran second. + 3. ``identify_from_evidence``: pure verdict logic, the source is never reopened. + 4. ``identify(check_visible=False, check_invisible=False)`` -- extraction plus + verdict as one call, the cross-check that 1 + 3 is the whole metadata path. + + Every ``@lru_cache`` in the metadata and C2PA modules is cleared before each timed + unit. Without that the second measurement of a file answers from the memo and + reports a cost of zero -- the caches are keyed on (path, mtime, size) and this + script reads each file several times. + +READING IT + Component times do NOT sum to the ``extract_provenance_evidence`` total for free: + they come from a separate cache-cleared run, so the sum is a cross-check. A gap + means a component is missing from the list. Both numbers are written. + +DATA SAFETY + Read-only over a local dataset. Writes only to the given output prefix, which + belongs outside the repository. + + uv run python scripts/detection_timing.py --limit 200 +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +# The package's OWN tree, not the repository root: from a worktree, an editable +# install resolves `remove_ai_watermarks` to the MAIN checkout, so a script measuring +# this tree would silently import a different one. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from remove_ai_watermarks import identify as identify_mod +from remove_ai_watermarks import metadata as metadata_mod +from remove_ai_watermarks._internal import c2pa as c2pa_mod + +log = logging.getLogger(__name__) + +SUPPORTED = frozenset({".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif", ".avif"}) + +# Timed in the order ``ProvenanceEvidence`` is constructed in ``identify.py``. The +# report script imports this to label its columns, so the order is the contract. +COMPONENTS: tuple[tuple[str, Callable[[Path], Any]], ...] = ( + ("c2pa_info", c2pa_mod.extract_c2pa_info), + ("ai_metadata", metadata_mod.get_ai_metadata), + ("scan_head", lambda p: metadata_mod.scan_head(p, identify_mod._SCAN_BYTES)), + ("iptc_ai_system", metadata_mod.iptc_ai_system), + ("aigc_label", metadata_mod.aigc_label), + ("exif_generator", metadata_mod.exif_generator), + ("xai_signature", metadata_mod.xai_signature), + ("huggingface_job", metadata_mod.huggingface_job), + ("samsung_genai", metadata_mod.samsung_genai), +) + + +def _cache_clearers() -> tuple[Callable[[], None], ...]: + """Every per-file memo in the metadata path, found by attribute, not by name list. + + A hand-written list silently goes stale the next time a probe gains a cache, and a + stale entry shows up as a suspiciously fast row rather than as an error. + """ + found: list[Callable[[], None]] = [] + for module in (metadata_mod, c2pa_mod): + for name in dir(module): + clear = getattr(getattr(module, name, None), "cache_clear", None) + if callable(clear): + found.append(clear) + return tuple(found) + + +_CLEARERS = _cache_clearers() + + +def _clear() -> None: + for clear in _CLEARERS: + clear() + + +def _ms(fn: Callable[[], Any]) -> tuple[float, Any]: + """Wall time in milliseconds plus the call's result.""" + start = time.perf_counter_ns() + value = fn() + return (time.perf_counter_ns() - start) / 1e6, value + + +def _pixel_geometry(path: Path) -> tuple[str | None, int | None, int | None]: + """Container format and pixel dimensions from the header alone.""" + try: + from PIL import Image + + with Image.open(path) as img: + return img.format, img.width, img.height + except Exception: # unreadable or an unsupported container + return None, None, None + + +def _warm_breakdown(path: Path, row: dict[str, Any]) -> None: + """Fill ``row`` with the warm-cache per-method breakdown.""" + _clear() + row["extract_evidence_ms"], evidence = _ms(lambda: identify_mod.extract_provenance_evidence(path)) + + _clear() + component_sum = 0.0 + for name, fn in COMPONENTS: + elapsed, _ = _ms(lambda fn=fn: fn(path)) + row[f"meta_{name}_ms"] = elapsed + component_sum += elapsed + row["meta_components_sum_ms"] = component_sum + + row["verdict_from_evidence_ms"], report = _ms(lambda: identify_mod.identify_from_evidence(evidence)) + + _clear() + row["identify_metadata_only_ms"], full = _ms( + lambda: identify_mod.identify(path, check_visible=False, check_invisible=False) + ) + + row["scan_bytes"] = len(evidence.scan) + row["has_c2pa"] = bool(evidence.c2pa_info) + row["has_ai_metadata"] = bool(evidence.ai_metadata) + row["is_ai_generated"] = full.is_ai_generated + row["confidence"] = full.confidence + row["platform"] = full.platform + row["signals"] = [signal.name for signal in full.signals] + # The two verdict paths must agree; a mismatch means the breakdown timed a + # different code path than the end-to-end call and the rows are not comparable. + row["verdict_agrees"] = (report.confidence, report.platform) == (full.confidence, full.platform) + + +def _measure(path: Path) -> dict[str, Any]: + row: dict[str, Any] = {"path": str(path), "ext": path.suffix.lower()} + try: + row["bytes"] = path.stat().st_size + except OSError as exc: + return {**row, "error": f"stat: {exc}"} + + # COLD first: this is the only moment the file is untouched by this process. + _clear() + try: + row["cold_extract_evidence_ms"], _ = _ms(lambda: identify_mod.extract_provenance_evidence(path)) + except Exception as exc: + return {**row, "error": f"cold extract: {type(exc).__name__}: {exc}"} + + row["format"], row["width"], row["height"] = _pixel_geometry(path) + width, height = row["width"], row["height"] + row["megapixels"] = round(width * height / 1e6, 3) if width and height else None + + try: + _warm_breakdown(path, row) + except Exception as exc: + row["error"] = f"warm pass: {type(exc).__name__}: {exc}" + return row + + +def _iter_images(root: Path) -> Iterator[Path]: + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix.lower() in SUPPORTED: + yield path + + +def _done_paths(out_path: Path) -> set[str]: + """Paths already recorded, so a long run resumes instead of restarting.""" + if not out_path.exists(): + return set() + done: set[str] = set() + with out_path.open(encoding="utf-8") as handle: + for line in handle: + try: + done.add(json.loads(line)["path"]) + except (ValueError, KeyError): + continue + return done + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("dataset", type=Path, help="directory of images, scanned recursively") + parser.add_argument("out_prefix", type=Path, help="output prefix; writes .jsonl") + parser.add_argument("--limit", type=int, default=0, help="stop after N files (0 = all)") + parser.add_argument("--progress-every", type=int, default=200) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + out_path = args.out_prefix.with_suffix(".jsonl") + out_path.parent.mkdir(parents=True, exist_ok=True) + done = _done_paths(out_path) + if done: + log.info("resuming: %d files already recorded", len(done)) + + processed = 0 + started = time.monotonic() + with out_path.open("a", encoding="utf-8") as handle: + for path in _iter_images(args.dataset): + if str(path) in done: + continue + row = _measure(path) + handle.write(json.dumps(row, ensure_ascii=False, default=str) + "\n") + handle.flush() + processed += 1 + if processed % args.progress_every == 0: + log.info("%d files, %.2f files/s", processed, processed / (time.monotonic() - started)) + if args.limit and processed >= args.limit: + break + + elapsed = time.monotonic() - started + rate = processed / max(elapsed, 1e-9) + log.info("done: %d files in %.1f s (%.2f files/s) -> %s", processed, elapsed, rate, out_path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/detection_timing_report.py b/scripts/detection_timing_report.py new file mode 100644 index 0000000..bb7ca86 --- /dev/null +++ b/scripts/detection_timing_report.py @@ -0,0 +1,232 @@ +"""Aggregate ``detection_timing.py`` records into per-method, per-segment tables. + +WHAT IT PRODUCES + Reading it back: the per-file JSONL is one row per image with a millisecond field + per method. This collapses it into percentiles per method, then repeats that per + segment -- container, megapixels, file size, C2PA presence, verdict confidence -- + because a single median hides a path whose cost is carried entirely by one + container or by the files that actually have a manifest. + + Writes ``_summary.csv`` (long form: segment_kind, segment, method, n, p50, + p90, p99, mean) and prints a markdown report. + + Percentiles are computed by nearest-rank on the sorted sample, not interpolated: + every reported number is a time some real file actually took. + +DATA SAFETY + Reads and writes only the given prefix, which belongs outside the repository. + + uv run python scripts/detection_timing_report.py .local-eval/timing/full +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +import sys +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from detection_timing import COMPONENTS as _TIMED + +# Taken from the script that WROTE the rows, in its order, so a probe added, removed +# or reordered there cannot silently leave a column missing or mislabelled here. +COMPONENTS = tuple(name for name, _ in _TIMED) +METHODS = ( + ("cold_extract_evidence_ms", "extract_provenance_evidence (cold)"), + ("extract_evidence_ms", "extract_provenance_evidence (warm)"), + *((f"meta_{name}_ms", f" {name}") for name in COMPONENTS), + ("meta_components_sum_ms", " (sum of components)"), + ("verdict_from_evidence_ms", "identify_from_evidence (verdict)"), + ("identify_metadata_only_ms", "identify(metadata only), end to end"), +) +# Median-only columns in the per-segment tables; the full percentile set goes to the CSV. +HEADLINE_LABELS = ( + ("extract_evidence_ms", "extract p50"), + ("verdict_from_evidence_ms", "verdict p50"), + ("identify_metadata_only_ms", "end-to-end p50"), +) +HEADLINE = tuple(key for key, _ in HEADLINE_LABELS) + + +def _bucket(value: float | None, edges: tuple[float, ...], labels: tuple[str, ...]) -> str: + if value is None: + return "unknown" + for edge, label in zip(edges, labels, strict=False): + if value < edge: + return label + return labels[-1] + + +SEGMENTS: tuple[tuple[str, Callable[[dict[str, Any]], str]], ...] = ( + ("container", lambda r: str(r.get("format") or "unknown")), + ( + "megapixels", + lambda r: _bucket(r.get("megapixels"), (1, 4, 12), ("<1 MP", "1-4 MP", "4-12 MP", ">12 MP")), + ), + ( + "file size", + lambda r: _bucket( + (r["bytes"] / 1e6) if r.get("bytes") is not None else None, + (1, 5, 20), + ("<1 MB", "1-5 MB", "5-20 MB", ">20 MB"), + ), + ), + ("c2pa", lambda r: "with C2PA" if r.get("has_c2pa") else "no C2PA"), + ("verdict", lambda r: f"confidence={r.get('confidence') or 'n/a'}"), +) + + +def _numeric(records: list[dict[str, Any]], key: str) -> list[float]: + """The numeric values of ``key``, skipping rows where it is absent or non-numeric. + + One definition of "counts as a measurement", so the printed table and the CSV can + never disagree about which rows a method was measured on. + """ + return [float(r[key]) for r in records if isinstance(r.get(key), (int, float))] + + +def _percentile(sorted_values: list[float], q: float) -> float: + """Nearest-rank percentile: the reported value is one a real file produced.""" + index = max(0, math.ceil(q * len(sorted_values)) - 1) + return sorted_values[index] + + +def _stats(values: list[float]) -> dict[str, float | int]: + ordered = sorted(values) + return { + "n": len(ordered), + "p50": _percentile(ordered, 0.50), + "p90": _percentile(ordered, 0.90), + "p99": _percentile(ordered, 0.99), + "mean": statistics.fmean(ordered), + } + + +def _rows(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + for line in handle: + try: + yield json.loads(line) + except ValueError: # a truncated tail while the run is still writing + continue + + +def _table(rows: list[dict[str, Any]], methods: tuple[tuple[str, str], ...]) -> list[str]: + out = ["| method | n | p50 | p90 | p99 | mean |", "|---|---:|---:|---:|---:|---:|"] + for key, label in methods: + values = _numeric(rows, key) + if not values: + continue + s = _stats(values) + out.append(f"| {label} | {s['n']} | {s['p50']:.3f} | {s['p90']:.3f} | {s['p99']:.3f} | {s['mean']:.3f} |") + return out + + +def _correlation(rows: list[dict[str, Any]], x_key: str, y_key: str) -> float | None: + both = [r for r in rows if isinstance(r.get(x_key), (int, float)) and isinstance(r.get(y_key), (int, float))] + if len(both) < 3: + return None + xs = _numeric(both, x_key) + ys = _numeric(both, y_key) + try: + return statistics.correlation(xs, ys) + except statistics.StatisticsError: # a constant column has no correlation + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("prefix", type=Path, help="prefix given to detection_timing.py") + args = parser.parse_args() + + jsonl = args.prefix.with_suffix(".jsonl") + all_rows = list(_rows(jsonl)) + failed = [r for r in all_rows if "error" in r] + rows = [r for r in all_rows if "error" not in r] + if not rows: + print(f"no usable records in {jsonl}") + return 1 + + disagreed = [r for r in rows if r.get("verdict_agrees") is False] + + lines = [ + f"# Detection timing over {len(rows)} images", + "", + f"Source: `{jsonl}`. Failed rows: {len(failed)}. Verdict-path mismatches: {len(disagreed)}.", + "", + "All times in milliseconds. Percentiles are nearest-rank.", + "", + "## Whole corpus", + "", + *_table(rows, METHODS), + "", + ] + + csv_rows: list[dict[str, Any]] = [] + for key, label in METHODS: + values = _numeric(rows, key) + if values: + csv_rows.append({"segment_kind": "all", "segment": "all", "method": label.strip(), **_stats(values)}) + + for kind, classify in SEGMENTS: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + groups[classify(row)].append(row) + lines += [f"## By {kind}", ""] + header = ["| segment | n | " + " | ".join(label for _, label in HEADLINE_LABELS) + " |"] + header.append("|---|---:|" + "---:|" * len(HEADLINE)) + lines += header + for segment, group in sorted(groups.items(), key=lambda item: -len(item[1])): + cells = [] + for key in HEADLINE: + values = _numeric(group, key) + cells.append(f"{_percentile(sorted(values), 0.5):.2f}" if values else "-") + lines.append(f"| {segment} | {len(group)} | " + " | ".join(cells) + " |") + for key, label in METHODS: + values = _numeric(group, key) + if values: + csv_rows.append( + {"segment_kind": kind, "segment": segment, "method": label.strip(), **_stats(values)} + ) + lines.append("") + + corr = _correlation(rows, "scan_bytes", "verdict_from_evidence_ms") + corr_size = _correlation(rows, "bytes", "extract_evidence_ms") + lines += [ + "## Scaling", + "", + f"- verdict time vs scan-buffer size: r = {corr:.3f}" if corr is not None else "- verdict correlation: n/a", + ( + f"- extraction time vs file size: r = {corr_size:.3f}" + if corr_size is not None + else "- extraction correlation: n/a" + ), + "", + ] + + summary_csv = args.prefix.with_name(args.prefix.name + "_summary.csv") + columns = ["segment_kind", "segment", "method", "n", "p50", "p90", "p99", "mean"] + with summary_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=columns) + writer.writeheader() + writer.writerows(csv_rows) + + report = "\n".join(lines) + args.prefix.with_name(args.prefix.name + "_report.md").write_text(report + "\n", encoding="utf-8") + print(report) + print(f"\nwrote {summary_csv}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/record_parity_audit.py b/scripts/record_parity_audit.py new file mode 100644 index 0000000..acb0226 --- /dev/null +++ b/scripts/record_parity_audit.py @@ -0,0 +1,224 @@ +"""Audit the record path against the file path over a whole dataset. + +WHY THIS EXISTS + +Two seams reach the same provenance verdict: + + identify(path, check_visible=False, check_invisible=False) + + identify_metadata_record(collect_metadata_record(path), path=path) + +Their equality is the record's entire contract, and it can break from either side -- +a region the collector stops walking, or a placement the file path learns to read and +the record does not. ``tests/test_metadata_record.py`` pins it over the tracked +fixtures; those cover the signal families we already know about. This covers the ones +we do not: every real placement in a real corpus, which is where all three defects +found so far actually came from. + +The record is round-tripped through ``json.dumps``/``loads`` before it is judged, so +a value that only survives in memory fails here rather than at a customer. + +WHAT IT REPORTS + +One JSONL row per image: both verdicts, whether they agree, the record size, and any +exception from either side. The summary counts disagreements by field and by signal, +so "the record lost samsung_genai on 13 files" reads directly off the output instead +of being reconstructed. + +Pass ``--baseline`` with an earlier run to also diff against it. That answers the +other question a detection change raises: which files changed verdict, and are they +exactly the ones that were meant to. + +DATA SAFETY + +Read-only over a local dataset. Writes only the given output path, which belongs +outside the repository. Resumable: rerunning skips files already recorded. + + uv run python scripts/record_parity_audit.py data/spaces/originals .local-eval/parity.jsonl + uv run python scripts/record_parity_audit.py --baseline .local-eval/previous.jsonl +""" + +from __future__ import annotations + +import argparse +import collections +import json +import logging +import sys +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + +# The package's OWN tree, not the repository root: from a worktree, an editable +# install resolves `remove_ai_watermarks` to the MAIN checkout, so a script measuring +# this tree would silently import a different one. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from remove_ai_watermarks.identify import identify, identify_metadata_record +from remove_ai_watermarks.metadata_record import collect_metadata_record + +log = logging.getLogger(__name__) + +SUPPORTED = frozenset({".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif", ".avif"}) +# Every field of the verdict a caller can act on. `path` is excluded: it is extraction +# context, and the two paths are handed the same one by construction. +COMPARED = ("is_ai_generated", "platform", "confidence", "ai_source_kind", "ai_from_metadata") + + +def _verdict(report: Any) -> dict[str, Any]: + return { + **{field: getattr(report, field) for field in COMPARED}, + "signals": sorted(signal.name for signal in report.signals), + "watermarks": sorted(report.watermarks), + } + + +def _audit(path: Path) -> dict[str, Any]: + row: dict[str, Any] = {"path": str(path)} + try: + row["bytes"] = path.stat().st_size + except OSError as exc: + return {**row, "error": f"stat: {exc}"} + + try: + started = time.perf_counter() + record = json.loads(json.dumps(collect_metadata_record(path))) + row["collect_ms"] = (time.perf_counter() - started) * 1000 + row["record_bytes"] = len(json.dumps(record)) + row["container"] = record.get("container") + via_record = _verdict(identify_metadata_record(record, path=path)) + except Exception as exc: + return {**row, "error": f"record path: {type(exc).__name__}: {exc}"} + + try: + via_file = _verdict(identify(path, check_visible=False, check_invisible=False)) + except Exception as exc: + return {**row, "error": f"file path: {type(exc).__name__}: {exc}"} + + row["record"] = via_record + row["file"] = via_file + row["agree"] = via_record == via_file + return row + + +def _iter_images(root: Path) -> Iterator[Path]: + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix.lower() in SUPPORTED: + yield path + + +def _done(out_path: Path) -> set[str]: + if not out_path.exists(): + return set() + done: set[str] = set() + with out_path.open(encoding="utf-8") as handle: + for line in handle: + try: + done.add(json.loads(line)["path"]) + except (ValueError, KeyError): + continue + return done + + +def _summarize(rows: list[dict[str, Any]], baseline: Path | None) -> None: + failed = [r for r in rows if "error" in r] + usable = [r for r in rows if "error" not in r] + disagreed = [r for r in usable if not r["agree"]] + + print(f"\nimages: {len(rows)} errors: {len(failed)} compared: {len(usable)}") + print(f"record path disagrees with file path: {len(disagreed)}") + for row in failed[:10]: + print(f" ERROR {Path(row['path']).name}: {row['error']}") + + fields: collections.Counter[str] = collections.Counter() + for row in disagreed: + fields.update(field for field in COMPARED if row["record"][field] != row["file"][field]) + for name in set(row["file"]["signals"]) - set(row["record"]["signals"]): + fields[f"signal missing from record: {name}"] += 1 + for name in set(row["record"]["signals"]) - set(row["file"]["signals"]): + fields[f"signal only in record: {name}"] += 1 + for label, count in fields.most_common(): + print(f" {label}: {count}") + for row in disagreed[:10]: + print(f" {Path(row['path']).name}\n record: {row['record']}\n file: {row['file']}") + + if baseline is None: + return + previous = {} + with baseline.open(encoding="utf-8") as handle: + for line in handle: + try: + item = json.loads(line) + except ValueError: + continue + if "error" not in item: + previous[Path(item["path"]).name] = item + + changed = [] + for row in usable: + was = previous.get(Path(row["path"]).name) + if was is None: + continue + # The WHOLE verdict, not a chosen subset. A first version compared confidence + # and signals only and reported "0 changed" for a run whose single intended + # correction was a watermark line -- the change it existed to show. + before = was.get("file") or {} + if before and before != row["file"]: + changed.append((row["path"], before, row["file"])) + print(f"\nverdicts changed against the baseline: {len(changed)}") + moved: collections.Counter[str] = collections.Counter() + for _, before, after in changed: + for name in set(after["signals"]) - set(before.get("signals") or []): + moved[f"gained signal {name}"] += 1 + for name in set(before.get("signals") or []) - set(after["signals"]): + moved[f"LOST signal {name}"] += 1 + for field in (*COMPARED, "watermarks"): + if before.get(field) != after.get(field): + moved[f"{field} changed"] += 1 + for label, count in moved.most_common(): + print(f" {label}: {count}") + for path, before, after in changed[:10]: + print(f" {Path(path).name}\n before: {before}\n after: {after}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("dataset", type=Path) + parser.add_argument("out", type=Path) + parser.add_argument("--baseline", type=Path, default=None, help="an earlier run to diff verdicts against") + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--progress-every", type=int, default=2000) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + args.out.parent.mkdir(parents=True, exist_ok=True) + done = _done(args.out) + if done: + log.info("resuming: %d images already audited", len(done)) + + processed = 0 + started = time.monotonic() + with args.out.open("a", encoding="utf-8") as handle: + for path in _iter_images(args.dataset): + if str(path) in done: + continue + handle.write(json.dumps(_audit(path), ensure_ascii=False, default=str) + "\n") + handle.flush() + processed += 1 + if processed % args.progress_every == 0: + log.info("%d images, %.1f/s", processed, processed / (time.monotonic() - started)) + if args.limit and processed >= args.limit: + break + + log.info("audited %d images in %.1f s", processed, time.monotonic() - started) + with args.out.open(encoding="utf-8") as handle: + rows = [json.loads(line) for line in handle] + _summarize(rows, args.baseline) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index 2679702..7197d67 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -32,7 +32,7 @@ _os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error") _warnings.filterwarnings("ignore", message=r".*ImageProcessorFast.*") -__version__ = "0.25.0" +__version__ = "0.26.0" __all__ = [ "BatchSummary", diff --git a/src/remove_ai_watermarks/_internal/c2pa.py b/src/remove_ai_watermarks/_internal/c2pa.py index 0eb2b94..d89255a 100644 --- a/src/remove_ai_watermarks/_internal/c2pa.py +++ b/src/remove_ai_watermarks/_internal/c2pa.py @@ -29,7 +29,9 @@ if TYPE_CHECKING: from typing import BinaryIO _C2paReader: Any = None +_C2paError: Any = None with contextlib.suppress(Exception): + from c2pa import C2paError as _C2paError # pyright: ignore[reportMissingTypeStubs] from c2pa import Reader as _C2paReader # pyright: ignore[reportMissingTypeStubs] _C2PA_READER_AVAILABLE = _C2paReader is not None @@ -48,10 +50,22 @@ def reader_available() -> bool: def _manifest_json_uncached(path: str) -> str | None: + """The manifest store as JSON, or None when this file has no readable manifest. + + Two outcomes are routine and stay at debug: a file with no manifest (``try_create`` + returns None) and a container the reader does not support. ANY other failure is + logged at warning, because the caller cannot tell the difference from the return + value and the consequence is severe: the verdict silently falls back to the raw + byte scan and can lose a high-confidence signal. The log line preserves the + diagnostic context needed to investigate an intermittent reader failure. + """ try: reader = _C2paReader.try_create(path) + except _C2paError.NotSupported as error: + logger.debug("C2PA reader does not support %s: %s", path, error) + return None except Exception as error: - logger.debug("C2PA reader rejected %s: %s", path, error) + logger.warning("C2PA reader failed to open %s: %s: %s", path, type(error).__name__, error) return None if reader is None: return None @@ -59,7 +73,9 @@ def _manifest_json_uncached(path: str) -> str | None: with reader: return cast("str", reader.json()) except Exception as error: - logger.debug("C2PA reader could not serialize %s: %s", path, error) + # The reader opened the file, so a manifest is there; failing to serialize it + # is never routine. + logger.warning("C2PA reader could not serialize %s: %s: %s", path, type(error).__name__, error) return None diff --git a/src/remove_ai_watermarks/_internal/constants.py b/src/remove_ai_watermarks/_internal/constants.py index 2ff85ce..46b370b 100644 --- a/src/remove_ai_watermarks/_internal/constants.py +++ b/src/remove_ai_watermarks/_internal/constants.py @@ -20,6 +20,9 @@ AI_KEYWORDS = _tokens( PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" C2PA_CHUNK_TYPE = b"caBX" +PNG_METADATA_CHUNKS = frozenset({b"tEXt", b"iTXt", b"zTXt", b"eXIf", b"iCCP"}) +RIFF_METADATA_CHUNKS = frozenset({b"EXIF", b"XMP ", b"ICCP", b"C2PA"}) +RIFF_CODED_IMAGE_CHUNKS = frozenset({b"VP8 ", b"VP8L", b"ALPH", b"ANMF"}) C2PA_SIGNATURES = tuple( token.encode() for token in _tokens("c2pa|C2PA|jumb|jumd|JUMBF|jumbf|cbor|contentcreds|digid|assertions|manifest") ) diff --git a/src/remove_ai_watermarks/_internal/isobmff.py b/src/remove_ai_watermarks/_internal/isobmff.py index 03f5865..e192c55 100644 --- a/src/remove_ai_watermarks/_internal/isobmff.py +++ b/src/remove_ai_watermarks/_internal/isobmff.py @@ -64,7 +64,7 @@ _AI_LABEL_MARKERS: tuple[bytes, ...] = AIGC_MARKERS + IPTC_AI_MARKERS + IPTC_AI_ # blanked in place (see ``blank_ai_xmp_packets``). _XMP_PACKET_RE = re.compile(rb"<\?xpacket begin=.*?<\?xpacket end=[^>]*?\?>", re.DOTALL) _STREAM_COPY_BYTES = 1024 * 1024 -_STREAM_SCAN_BYTES = 4 * 1024 * 1024 +STREAM_SCAN_BYTES = 4 * 1024 * 1024 # TC260-PG-20257A stores an MP4/MOV label as an ``AIGC`` key in @@ -133,7 +133,7 @@ def _read_box_header( return end, box_type, payload_off -def _iter_file_boxes( +def iter_file_boxes( stream: BinaryIO, start: int, end: int, @@ -193,17 +193,17 @@ def _tc260_aigc_regions( Each tuple is ``(key_start, key_end, value_start, value_end, value)``. """ regions: list[tuple[int, int, int, int, bytes]] = [] - for _moov_start, moov_end, moov_type, moov_payload in _iter_file_boxes(stream, 0, file_size): + for _moov_start, moov_end, moov_type, moov_payload in iter_file_boxes(stream, 0, file_size): if moov_type != b"moov": continue - for _udta_start, udta_end, udta_type, udta_payload in _iter_file_boxes( + for _udta_start, udta_end, udta_type, udta_payload in iter_file_boxes( stream, moov_payload, moov_end, ): if udta_type != b"udta": continue - for _meta_start, meta_end, meta_type, meta_payload in _iter_file_boxes( + for _meta_start, meta_end, meta_type, meta_payload in iter_file_boxes( stream, udta_payload, udta_end, @@ -212,7 +212,7 @@ def _tc260_aigc_regions( continue keys: dict[int, tuple[int, int]] = {} ilst_boxes: list[tuple[int, int]] = [] - for _child_start, child_end, child_type, child_payload in _iter_file_boxes( + for _child_start, child_end, child_type, child_payload in iter_file_boxes( stream, meta_payload + 4, meta_end, @@ -224,7 +224,7 @@ def _tc260_aigc_regions( if not keys: continue for ilst_payload, ilst_end in ilst_boxes: - for _item_start, item_end, item_type, item_payload in _iter_file_boxes( + for _item_start, item_end, item_type, item_payload in iter_file_boxes( stream, ilst_payload, ilst_end, @@ -233,7 +233,7 @@ def _tc260_aigc_regions( key_span = keys.get(index) if key_span is None: continue - for _data_start, data_end, data_type, data_payload in _iter_file_boxes( + for _data_start, data_end, data_type, data_payload in iter_file_boxes( stream, item_payload, item_end, @@ -425,7 +425,7 @@ def strip_isobmff_media_file( source: str | Path, output: str | Path, *, - max_box_scan: int = _STREAM_SCAN_BYTES, + max_box_scan: int = STREAM_SCAN_BYTES, ) -> tuple[int, int]: """Stream-copy an MP4/MOV/M4A while removing supported AI metadata. diff --git a/src/remove_ai_watermarks/_internal/schema.py b/src/remove_ai_watermarks/_internal/schema.py new file mode 100644 index 0000000..8253e9d --- /dev/null +++ b/src/remove_ai_watermarks/_internal/schema.py @@ -0,0 +1,16 @@ +"""Shared validation for versioned JSON transport contracts.""" + +from collections.abc import Collection + + +def require_schema_version( + value: object, + *, + contract: str, + supported: Collection[int], +) -> int: + """Return an explicitly supported integer schema version or raise.""" + if type(value) is not int or value not in supported: + versions = ", ".join(str(version) for version in sorted(supported)) + raise ValueError(f"Unsupported {contract} schema: {value!r}; supported versions: {versions}") + return value diff --git a/scripts/scan_dataset.py b/src/remove_ai_watermarks/forensic_metadata.py similarity index 53% rename from scripts/scan_dataset.py rename to src/remove_ai_watermarks/forensic_metadata.py index 45bb043..c49a930 100644 --- a/scripts/scan_dataset.py +++ b/src/remove_ai_watermarks/forensic_metadata.py @@ -1,78 +1,51 @@ -"""Collect all raw metadata over a dataset, for later offline analysis. -Read-only; analysis is NOT this script's job. For every image it writes -one JSONL record containing: +"""Collect JSON-safe metadata and container forensics for one media file. -- file basics: name, path, size in bytes, container format (content-sniffed), - pixel dimensions, dpi, color mode, sha256, mtime/birthtime -- raw metadata, so nothing has to be re-scanned later: - full EXIF (all IFDs, decoded tag names, MakerNote in full hex), all XMP - packets, all PNG chunks (text chunks decoded and inflated, binary chunks - as base64), all JPEG APP segments (marker + full base64), WebP RIFF - chunks, ISOBMFF box/item inventory for HEIC/AVIF/MOV, IPTC-IIM dataset, - ICC profile (full base64), C2PA manifest store JSON, post-EOI/IEND - trailers, EXIF thumbnail (bytes + its own JPEG structure), JPEG encoder - structure (quant tables, progressive scan script, Huffman tables, - subsampling, JFIF/Adobe markers), file hashes and timestamps, macOS - download provenance xattrs, Live Photo content identifier. Embedded - binary blobs are base64'd in full with a 1 MB ceiling per blob. - -By default everything collected is raw bytes or a mechanical container -decode; no verdicts, no estimates, no edit-trail interpretation. The -optional pixel modes add a clearly-separated derived layer (aggregate -statistics only, still no verdicts). - -Usage: - python scan_dataset.py /path/to/dataset out_prefix [--pixels|--pixels-full] - -Writes out_prefix.jsonl (one record per file) and out_prefix.csv (flat -summary for quick sorting). If the prefix ends with .gz, the JSONL is -gzip-compressed on the fly (3-5x smaller, still streamable line by line). - ---pixels adds aggregate pixel forensics to each record (DCT histograms, -noise/FFT/ELA/gradient/color statistics) from which the image CANNOT be -reconstructed, plus per-section timing_ms for pipeline latency planning. ---pixels-full additionally stores the privacy-lifting artifacts (phash, -128px thumbnail, coarse ELA/noise/FFT-phase maps). Pixel modes require -numpy; the metadata-only mode needs no numpy. - -For a large dataset, parallelize by sharding the input into folders and -running one process per shard (a multiprocessing pool hangs on macOS once -cv2/PIL are loaded; independent processes do not): - - for d in /dataset/*/; do - python scan_dataset.py "$d" "out_$(basename "$d")" & - done - wait - cat out_*.jsonl > dataset.jsonl # or just read the shards lazily - -Rerunning with the same prefix resumes: files already present in the -output are skipped, new files are appended. - -Reading big results: never load the JSONL whole. Stream it: -polars.scan_ndjson (lazy), pandas.read_json(lines=True, chunksize=...), -or a plain line loop. One line = one self-contained JSON record. - -Dependencies: pip install pillow piexif c2pa-python pillow-heif -(pillow-heif is only needed for HEIC/AVIF inputs). +The collector is deliberately evidence-only: it preserves raw EXIF, IPTC, C2PA, +container metadata, encoder structure, hashes, timestamps, and bounded binary +payloads without deciding whether the content is AI-generated. Provenance verdicts +and pixel statistics are separate library stages. """ -from __future__ import annotations - import base64 import contextlib -import csv +import hashlib import io import json +import os +import plistlib +import re import struct -import sys -import time +import zlib from pathlib import Path -from typing import Any +from typing import Any, cast +import piexif from PIL import Image from PIL.IptcImagePlugin import getiptcinfo -SUPPORTED = { +from remove_ai_watermarks import image_io +from remove_ai_watermarks._internal.constants import ( + PNG_METADATA_CHUNKS, + RIFF_CODED_IMAGE_CHUNKS, + RIFF_METADATA_CHUNKS, +) +from remove_ai_watermarks._internal.isobmff import ( + C2PA_BOX_TYPES, + STREAM_SCAN_BYTES, + iter_file_boxes, +) +from remove_ai_watermarks._internal.schema import require_schema_version +from remove_ai_watermarks.metadata import QUICK_SCAN_BYTES +from remove_ai_watermarks.metadata_record import HEAD_WINDOW + +__all__ = [ + "FORENSIC_METADATA_RECORD_TYPE", + "FORENSIC_METADATA_SCHEMA_VERSION", + "SUPPORTED_EXTENSIONS", + "collect_forensic_metadata", +] + +SUPPORTED_EXTENSIONS = { ".png", ".jpg", ".jpeg", @@ -92,7 +65,16 @@ SUPPORTED = { ".jxl", } +FORENSIC_METADATA_SCHEMA_VERSION = 1 +FORENSIC_METADATA_RECORD_TYPE = "forensic_metadata" + _B64_CAP = 1 << 20 # 1 MB safety ceiling per embedded blob +_TEXT_CAP = 1 << 20 # decoded PNG text ceiling per chunk +# Preserve enough top-level ISOBMFF uuid/jumb payload data for downstream +# provenance algorithms without requiring them to reopen the source file. +_PROVENANCE_B64_CAP = STREAM_SCAN_BYTES +_RAW_SCAN_HEAD = HEAD_WINDOW +_RAW_SCAN_TAIL = QUICK_SCAN_BYTES def _safe_str(v: Any) -> str: @@ -102,11 +84,10 @@ def _safe_str(v: Any) -> str: return repr(v) -def _b64(b: bytes) -> str: - """Full base64 of a binary blob, with a 1 MB safety ceiling.""" - if len(b) > _B64_CAP: - return base64.b64encode(b[:_B64_CAP]).decode("ascii") + f"...TRUNCATED({len(b)} bytes total)" - return base64.b64encode(b).decode("ascii") +def _b64(b: bytes, *, cap: int = _B64_CAP) -> str: + """Legacy base64 value, with an explicit marker when the payload is capped.""" + encoded = base64.b64encode(b[:cap]).decode("ascii") + return encoded + f"...TRUNCATED({len(b)} bytes total)" if len(b) > cap else encoded def _decode_exif_value(v: Any) -> Any: @@ -118,8 +99,9 @@ def _decode_exif_value(v: Any) -> Any: except (UnicodeDecodeError, ValueError): return f"hex:{v.hex()}" return f"hex:{v.hex()}" - if isinstance(v, (tuple, list)): - return [_decode_exif_value(x) for x in v] + if isinstance(v, tuple | list): + sequence = cast("list[Any] | tuple[Any, ...]", v) + return [_decode_exif_value(item) for item in sequence] return v @@ -132,10 +114,8 @@ def read_full_exif( ``exif_blob`` is the PIL-exposed EXIF blob (PNG/WebP/HEIC path) so the caller's single Image.open is not repeated here. ``data`` is the already-read file bytes so piexif does not re-read the file.""" - import piexif - try: - exif = piexif.load(data) if data is not None else piexif.load(str(path)) + exif: dict[str, Any] = piexif.load(data) if data is not None else piexif.load(str(path)) except Exception: if not exif_blob: return {}, None @@ -152,18 +132,19 @@ def read_full_exif( continue if not isinstance(tags, dict): continue - tag_names = piexif.TAGS.get(ifd, {}) - decoded = {} - for t, v in tags.items(): - name = tag_names.get(t, {}).get("name", f"tag_{t}") - if name == "MakerNote" and isinstance(v, bytes): - # Full hex, no cap: some Apple manifests are several kilobytes. + all_tag_names = cast("dict[str, dict[int, dict[str, Any]]]", getattr(piexif, "TAGS", {})) + tag_names = all_tag_names.get(ifd, {}) + decoded: dict[str, Any] = {} + for tag, value in cast("dict[int, Any]", tags).items(): + name = str(tag_names.get(tag, {}).get("name", f"tag_{tag}")) + if name == "MakerNote" and isinstance(value, bytes): + # full hex, no cap: measured on real uploads, Apple is ~2 KB # but Canon reaches 28 KB and Sony 38 KB (AF data, serials, # embedded previews) -- a cap would silently drop exactly the # camera-original evidence this scan exists to preserve - decoded[name] = f"hex:{v.hex()}" + decoded[name] = f"hex:{value.hex()}" else: - decoded[name] = _decode_exif_value(v) + decoded[name] = _decode_exif_value(value) out[ifd] = decoded return out, thumbnail @@ -174,47 +155,52 @@ def _png_text_decode(ctype: str, body: bytes) -> str: The compressed forms are where ComfyUI / Automatic1111 hide the generation workflow and prompt, so skipping the inflate would drop the strongest AI-provenance text a PNG can carry.""" - import zlib - if ctype == "tEXt": - return body.decode("utf-8", "replace") + suffix = b"...TRUNCATED" if len(body) > _TEXT_CAP else b"" + return (body[:_TEXT_CAP] + suffix).decode("utf-8", "replace") if ctype == "zTXt": nul = body.find(b"\x00") if nul == -1: - return body.decode("utf-8", "replace") + return body[:_TEXT_CAP].decode("utf-8", "replace") keyword = body[:nul].decode("latin-1", "replace") # body[nul+1] = compression method (0 = zlib) try: - text = zlib.decompress(body[nul + 2 :]).decode("utf-8", "replace") + inflater = zlib.decompressobj() + decoded = inflater.decompress(body[nul + 2 :], _TEXT_CAP + 1) + suffix = "...TRUNCATED" if len(decoded) > _TEXT_CAP else "" + text = decoded[:_TEXT_CAP].decode("utf-8", "replace") + suffix except zlib.error: - text = body.decode("utf-8", "replace") + text = body[:_TEXT_CAP].decode("utf-8", "replace") return f"{keyword}\x00{text}" # iTXt: keyword\0 compflag(1) compmethod(1) lang\0 translated\0 text parts = body.split(b"\x00", 1) if len(parts) < 2: - return body.decode("utf-8", "replace") + return body[:_TEXT_CAP].decode("utf-8", "replace") keyword = parts[0].decode("latin-1", "replace") rest = parts[1] if len(rest) < 2: - return body.decode("utf-8", "replace") + return body[:_TEXT_CAP].decode("utf-8", "replace") compflag = rest[0] tail = rest[2:] for _ in range(2): # skip language tag and translated keyword nul = tail.find(b"\x00") if nul == -1: - return body.decode("utf-8", "replace") + return body[:_TEXT_CAP].decode("utf-8", "replace") tail = tail[nul + 1 :] if compflag: with contextlib.suppress(zlib.error): - tail = zlib.decompress(tail) + inflater = zlib.decompressobj() + tail = inflater.decompress(tail, _TEXT_CAP + 1) + if len(tail) > _TEXT_CAP: + tail = tail[:_TEXT_CAP] + b"...TRUNCATED" return f"{keyword}\x00{tail.decode('utf-8', 'replace')}" -def read_png_chunks(data: bytes) -> tuple[list[dict[str, Any]], int]: +def read_png_chunks(data: bytes) -> tuple[list[dict[str, Any]], bytes]: """Every PNG chunk in order (type, length; text chunks decoded and - inflated, binary chunks as base64) plus the post-IEND trailer size.""" + inflated, binary chunks as base64) plus the post-IEND trailer bytes.""" chunks: list[dict[str, Any]] = [] - post_iend = 0 + post_iend = b"" try: pos = 8 while pos + 12 <= len(data): @@ -248,13 +234,21 @@ def read_png_chunks(data: bytes) -> tuple[list[dict[str, Any]], int]: chunks.append(entry) pos += 12 + length if ctype == "IEND": - post_iend = len(data) - pos + post_iend = data[pos:] break except Exception as exc: chunks.append({"error": _safe_str(exc)}) return chunks, post_iend +def _set_jpeg_trailer(result: dict[str, Any], data: bytes, eoi: int) -> None: + """Preserve bytes after JPEG EOI for Samsung Galaxy AI detection.""" + trailer = data[eoi + 2 :] + result["post_eoi_bytes"] = len(trailer) + if trailer: + result["post_eoi_base64"] = _b64(trailer) + + def read_jpeg_segments(data: bytes) -> dict[str, Any]: """Every JPEG APP segment in order, plus post-EOI trailer size. @@ -269,12 +263,12 @@ def read_jpeg_segments(data: bytes) -> dict[str, Any]: break marker = data[pos + 1] if marker == 0xD9: # EOI - result["post_eoi_bytes"] = len(data) - (pos + 2) + _set_jpeg_trailer(result, data, pos) break if marker == 0xDA: # SOS: entropy-coded data follows eoi = data.rfind(b"\xff\xd9") if eoi != -1: - result["post_eoi_bytes"] = len(data) - (eoi + 2) + _set_jpeg_trailer(result, data, eoi) break if not (0xE0 <= marker <= 0xEF): length = struct.unpack(">H", data[pos + 2 : pos + 4])[0] @@ -284,7 +278,8 @@ def read_jpeg_segments(data: bytes) -> dict[str, Any]: body = data[pos + 4 : pos + 2 + length] name = f"APP{marker - 0xE0}" entry: dict[str, Any] = {"marker": name, "length": length} - if body.startswith(b"http://ns.adobe.com/xap/1.0/\x00"): + # Adobe JPEG XMP APP1 magic (namespace URI in the packet, not a request). + if body.startswith(b"http://ns.adobe.com/xap/1.0/\x00"): # NOSONAR entry["kind"] = "xmp" entry["text"] = body[29:].decode("utf-8", "replace") elif name == "APP2" and body.startswith(b"MPF\x00"): @@ -366,13 +361,20 @@ def read_pil_info(path: Path) -> tuple[dict[str, Any], dict[str, Any], bytes | N def read_c2pa_store(path: Path) -> dict[str, Any]: - """Full C2PA manifest store JSON via the official c2pa-python Reader.""" - try: - from c2pa import Reader + """Full C2PA manifest store through the package's cached reader.""" + from remove_ai_watermarks._internal.c2pa import read_manifest_store_json - with Reader(str(path)) as reader: - return json.loads(reader.json()) - except Exception as exc: + raw = read_manifest_store_json(path) + if raw is None: + return {} + try: + value: Any = json.loads(raw) + return ( + cast("dict[str, Any]", value) + if isinstance(value, dict) + else {"error": "C2PA manifest store is not an object"} + ) + except (TypeError, ValueError) as exc: return {"error": _safe_str(exc)} @@ -394,7 +396,7 @@ def sniff_format(head: bytes) -> str: return f"unknown:{head[:16].hex()}" -# --- forensic helpers: signals that a file is not an untouched original --- +# --- JPEG encoder structure (metadata layer) --- def _jpeg_forensics_bytes(data: bytes) -> dict[str, Any]: @@ -451,7 +453,7 @@ def _jpeg_forensics_bytes(data: bytes) -> dict[str, Any]: out["precision_bits"] = body[0] out["sof_height"] = struct.unpack(">H", body[1:3])[0] out["sof_width"] = struct.unpack(">H", body[3:5])[0] - comps = [] + comps: list[dict[str, int]] = [] for i in range(body[5]): c = body[6 + i * 3 : 9 + i * 3] if len(c) == 3: @@ -499,15 +501,18 @@ def read_webp_chunks(data: bytes) -> list[dict[str, Any]]: chunks: list[dict[str, Any]] = [] try: pos = 12 - while pos + 8 <= len(data): - ctype = data[pos : pos + 4].decode("latin-1") + declared_end = 8 + struct.unpack("= 12 else len(data) + container_end = min(len(data), declared_end) + while pos + 8 <= container_end: + chunk_type = data[pos : pos + 4] + ctype = chunk_type.decode("latin-1") length = struct.unpack(" list[dict[str, Any]]: return chunks -def sha256_of(data: bytes) -> str: - import hashlib +def read_webp_late_metadata_path(path: Path, window: int = _RAW_SCAN_HEAD) -> list[dict[str, Any]]: + """Stream metadata chunks after ``window`` while seeking over coded frames.""" + chunks: list[dict[str, Any]] = [] + try: + file_size = path.stat().st_size + with open(path, "rb") as handle: + header = handle.read(12) + if len(header) < 12 or not header.startswith(b"RIFF") or header[8:12] != b"WEBP": + return chunks + container_end = min(file_size, 8 + struct.unpack("= window: + handle.seek(start) + body = handle.read(min(safe_length, _B64_CAP)) + entry: dict[str, Any] = { + "type": chunk_type.decode("latin-1"), + "length": length, + "base64": _b64(body), + } + if len(body) < safe_length: + entry["truncated"] = True + chunks.append(entry) + position = start + safe_length + (safe_length & 1) + except (OSError, struct.error) as exc: + chunks.append({"error": _safe_str(exc)}) + return chunks + +def sha256_of(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def xattr_where_from(path: Path) -> list[str]: """macOS download-source URLs (kMDItemWhereFroms), empty elsewhere.""" - import os - import plistlib - try: - raw = os.getxattr(path, "com.apple.metadata:kMDItemWhereFroms") + getter = cast("Any", os.getxattr) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + raw = cast("bytes", getter(path, "com.apple.metadata:kMDItemWhereFroms")) value = plistlib.loads(raw) - return [str(v) for v in value] if isinstance(value, list) else [str(value)] + values = cast("list[Any]", value) if isinstance(value, list) else [value] + return [str(item) for item in values] except (AttributeError, OSError, ValueError): return [] @@ -540,10 +579,10 @@ def xattr_where_from(path: Path) -> list[str]: def xattr_quarantine(path: Path) -> str | None: """macOS quarantine string: flags; timestamp; downloading agent (Safari, Telegram, Chrome...). Presence alone means 'came from the internet'.""" - import os - try: - return os.getxattr(path, "com.apple.quarantine").decode("utf-8", "replace")[:500] + getter = cast("Any", os.getxattr) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + raw = cast("bytes", getter(path, "com.apple.quarantine")) + return raw.decode("utf-8", "replace")[:500] except (AttributeError, OSError): return None @@ -554,28 +593,22 @@ def read_isobmff_inventory(data: bytes) -> dict[str, Any]: irot derived images). Strong phone-provenance signal.""" out: dict[str, Any] = {} try: + stream = io.BytesIO(data) def boxes(start: int, end: int) -> list[tuple[str, int, int]]: - result = [] - pos = start - while pos + 8 <= end: - size, btype = struct.unpack(">I4s", data[pos : pos + 8]) - t = btype.decode("latin-1") - header = 8 - if size == 1: - size = struct.unpack(">Q", data[pos + 8 : pos + 16])[0] - header = 16 - elif size == 0: - size = end - pos - if size < header or pos + size > end: - break - result.append((t, pos + header, pos + size)) - pos += size - return result + return [ + (box_type.decode("latin-1"), payload_offset, box_end) + for _, box_end, box_type, payload_offset in iter_file_boxes(stream, start, end) + ] top = boxes(0, len(data)) out["boxes"] = [t for t, _, _ in top] + provenance_boxes: list[dict[str, Any]] = [] for t, s, e in top: + if t.encode("latin-1") in C2PA_BOX_TYPES: + provenance_boxes.append( + {"type": t, "length": e - s, "base64": _b64(data[s:e], cap=_PROVENANCE_B64_CAP)} + ) if t == "moov": for ct, cs, ce in boxes(s, e): if ct == "mvhd" and ce - cs >= 24: @@ -592,7 +625,7 @@ def read_isobmff_inventory(data: bytes) -> dict[str, Any]: # full box + entry count, then infe entries count = struct.unpack(">H", data[cs + 4 : cs + 6])[0] out["meta_item_count"] = count - item_types = [] + item_types: list[str] = [] for it, is_, ie in boxes(cs + 6, ce): if it == "infe" and ie - is_ >= 8: # infe full box: version(1)+flags(3), then @@ -611,17 +644,17 @@ def read_isobmff_inventory(data: bytes) -> dict[str, Any]: props = [t for t, _, _ in boxes(ps, pe)] out["ipco_properties"] = props # auxC holds the auxiliary image type URN - for qt, qs, qe in boxes(ps, pe): - if qt == "auxC": + for box_type, qs, qe in boxes(ps, pe): + if box_type == "auxC": out["auxc_types"] = ( data[qs + 4 : qe].split(b"\x00")[0].decode("latin-1", "replace") ) elif ct == "iref": out["has_iref"] = True + if provenance_boxes: + out["provenance_boxes"] = provenance_boxes # QuickTime metadata keys (©mak/©mod/©swr) for the MOV side of # Live Photos: tolerant printable-string grab after each atom - import re - qt: dict[str, str] = {} for atom, key in ((b"\xa9mak", "make"), (b"\xa9mod", "model"), (b"\xa9swr", "software")): idx = data.find(atom) @@ -636,22 +669,88 @@ def read_isobmff_inventory(data: bytes) -> dict[str, Any]: return out +def read_isobmff_provenance_path(path: Path) -> dict[str, Any]: + """Stream top-level ISOBMFF boxes and preserve provenance payloads. + + This is the large-file counterpart to :func:`read_isobmff_inventory`. + It seeks over media payloads instead of loading them into memory. + """ + out: dict[str, Any] = {"boxes": []} + provenance_boxes: list[dict[str, Any]] = [] + collected = 0 + try: + file_size = path.stat().st_size + with open(path, "rb") as f: + for _, box_end, box_type_raw, payload_offset in iter_file_boxes(f, 0, file_size): + box_type = box_type_raw.decode("latin-1") + out["boxes"].append(box_type) + payload_length = box_end - payload_offset + if box_type_raw in C2PA_BOX_TYPES and collected < _PROVENANCE_B64_CAP: + to_read = min(payload_length, _PROVENANCE_B64_CAP - collected) + f.seek(payload_offset) + payload = f.read(to_read) + entry: dict[str, Any] = { + "type": box_type, + "length": payload_length, + "base64": _b64(payload, cap=_PROVENANCE_B64_CAP), + } + if to_read < payload_length: + entry["truncated"] = True + provenance_boxes.append(entry) + collected += len(payload) + except (OSError, struct.error) as exc: + out["error"] = _safe_str(exc) + if provenance_boxes: + out["provenance_boxes"] = provenance_boxes + return out + + +def read_png_late_metadata_path(path: Path, window: int = _RAW_SCAN_HEAD) -> list[dict[str, Any]]: + """Stream PNG metadata chunks whose payload starts after ``window``.""" + chunks: list[dict[str, Any]] = [] + try: + file_size = path.stat().st_size + with open(path, "rb") as f: + if f.read(8) != b"\x89PNG\r\n\x1a\n": + return chunks + pos = 8 + while pos + 12 <= file_size: + f.seek(pos) + header = f.read(8) + if len(header) < 8: + break + length, chunk_type = struct.unpack(">I4s", header) + data_start = pos + 8 + safe_length = max(0, min(length, file_size - data_start)) + if chunk_type in PNG_METADATA_CHUNKS and data_start >= window: + body = f.read(min(safe_length, _B64_CAP)) + entry: dict[str, Any] = { + "type": chunk_type.decode("latin-1"), + "length": length, + "base64": _b64(body), + } + if len(body) < safe_length: + entry["truncated"] = True + chunks.append(entry) + pos = data_start + safe_length + 4 + if chunk_type == b"IEND": + break + except (OSError, struct.error) as exc: + chunks.append({"error": _safe_str(exc)}) + return chunks + + def apple_live_photo_id(head: bytes) -> str | None: """Apple Live Photo content identifier (links the still to its MOV). The UUID sits in the Apple MakerNote (tag 17) of the still and in the MOV metadata; a raw head scan finds it in either container.""" - import re - # the UUID string sits next to "content.identifier" in the MOV, but in # the STILL it is a bare UUID inside the Apple MakerNote (whose header # is "Apple iOS"), so gate on either marker if b"content.identifier" not in head and b"com.apple.quicktime" not in head and b"Apple iOS" not in head: return None - m = re.search( - rb"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}", - head, - ) + m = re.search(rb"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}", head) return m.group(0).decode("ascii") if m else None @@ -660,8 +759,6 @@ _HEAD_READ = 4 << 20 def _sha256_stream(path: Path) -> str: - import hashlib - h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): @@ -669,7 +766,25 @@ def _sha256_stream(path: Path) -> str: return h.hexdigest() -def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: +def collect_forensic_metadata( + path: Path, + *, + schema_version: int = FORENSIC_METADATA_SCHEMA_VERSION, +) -> dict[str, Any]: + """Collect the versioned, metadata-only forensic record for ``path``. + + This broad inspection record is not provenance-detector input. Use + :func:`remove_ai_watermarks.metadata_record.collect_metadata_record` for the + strict record accepted by ``identify_metadata_record``. Long-lived consumers + should request the schema they implement; unsupported versions raise before the + source is read. + """ + schema_version = require_schema_version( + schema_version, + contract="forensic metadata", + supported=(1,), + ) + image_io._register_heif() # pyright: ignore[reportPrivateUsage] stat = path.stat() oversized = stat.st_size > _MAX_FULL_READ if oversized: @@ -680,6 +795,8 @@ def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: data = path.read_bytes() head = data record: dict[str, Any] = { + "schema_version": schema_version, + "record_type": FORENSIC_METADATA_RECORD_TYPE, "file": str(path), "name": path.name, "extension": path.suffix.lower(), @@ -690,9 +807,14 @@ def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: "content_format": sniff_format(head), } if oversized: - # too big to hold in memory: path-based readers (PIL, piexif, - # c2pa) still run; byte-level walkers are skipped + # Preserve the same bounded byte windows used by downstream provenance + # algorithms while path-based readers (PIL, piexif, C2PA) run normally. record["oversized"] = {"head_scanned_bytes": len(head)} + record["raw_metadata_windows"] = {"head_base64": _b64(head[:_RAW_SCAN_HEAD])} + if stat.st_size > _RAW_SCAN_TAIL: + with open(path, "rb") as f: + f.seek(-_RAW_SCAN_TAIL, 2) + record["raw_metadata_windows"]["tail_base64"] = _b64(f.read()) where_from = xattr_where_from(path) if where_from: record["download_source_urls"] = where_from @@ -710,7 +832,8 @@ def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: if fmt == "png": record["png_chunks"], post_iend = read_png_chunks(data) if post_iend: - record["png_post_iend_bytes"] = post_iend + record["png_post_iend_bytes"] = len(post_iend) + record["png_post_iend_base64"] = _b64(post_iend) elif fmt == "jpeg": record["jpeg"] = read_jpeg_segments(data) record["jpeg_forensics"] = _jpeg_forensics_bytes(data) @@ -718,6 +841,16 @@ def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: record["webp_chunks"] = read_webp_chunks(data) elif fmt.startswith("isobmff"): record["isobmff"] = read_isobmff_inventory(data) + elif record["content_format"] == "png": + late_chunks = read_png_late_metadata_path(path) + if late_chunks: + record["png_late_metadata_chunks"] = late_chunks + elif record["content_format"] == "webp": + late_chunks = read_webp_late_metadata_path(path) + if late_chunks: + record["webp_late_metadata_chunks"] = late_chunks + elif record["content_format"].startswith("isobmff"): + record["isobmff"] = read_isobmff_provenance_path(path) if thumbnail: record["has_exif_thumbnail"] = True # the embedded thumbnail is its own JPEG; after an edit its encoder @@ -725,375 +858,4 @@ def scan_file(path: Path, *, pixel_mode: str | None = None) -> dict[str, Any]: thumb_forensics = _jpeg_forensics_bytes(thumbnail) thumb_forensics["base64"] = _b64(thumbnail) record["exif_thumbnail_forensics"] = thumb_forensics - if pixel_mode is not None: - if oversized: - # symmetric with the metadata oversized marker: distinguish - # "skipped because oversized" from "decode failed" downstream - record["pixel"] = {"skipped": "oversized"} - else: - record.update(scan_pixels_of(path, full=pixel_mode == "full")) return record - - -def scan_pixels_of(path: Path, *, full: bool) -> dict[str, Any]: - """The optional pixel layer for one file (see the pixel-layer section - below). Returns {"pixel_error": ...} when numpy is unavailable.""" - if np is None: - return {"pixel_error": "numpy not installed"} - t0 = time.perf_counter() - timing: dict[str, float] = {} - record: dict[str, Any] = {} - - t = time.perf_counter() - gray, rgb, info = read_gray(path) - record["pixel"] = info - timing["decode"] = time.perf_counter() - t - if gray is not None: - # maps shared between the scalar features and the --pixels-full - # artifacts, computed once (the JPEG re-save and the sliding-window - # conv are the two most expensive features in the scan) - t = time.perf_counter() - residual = noise_residual_map(gray) - timing["noise"] = time.perf_counter() - t - if residual is not None: - record["noise"] = noise_features(residual) - - t = time.perf_counter() - spectrum = fft_decompose(gray) - timing["fft"] = time.perf_counter() - t - mag = phase = None - if spectrum is not None: - mag, phase = spectrum - record["fft"] = fft_features(mag) - - t = time.perf_counter() - ela = ela_map(rgb) - timing["ela"] = time.perf_counter() - t - if ela is not None: - record["ela"] = ela_features(ela) - - for name, fn, arg in ( - ("dct", dct_features, gray), - ("gradient", gradient_features, gray), - ("color", color_features, rgb), - ): - t = time.perf_counter() - try: - result = fn(arg) - except Exception as exc: - result = {"error": _safe_str(exc)} - timing[name] = time.perf_counter() - t - if result: - record[name] = result - if full: - t = time.perf_counter() - try: - record["full"] = full_artifacts(gray, rgb, ela=ela, residual=residual, phase=phase) - except Exception as exc: - record["full"] = {"error": _safe_str(exc)} - timing["full_artifacts"] = time.perf_counter() - t - timing["total"] = time.perf_counter() - t0 - record["timing_ms"] = {k: round(v * 1000, 1) for k, v in timing.items()} - return record - - -def summary_row(record: dict[str, Any]) -> dict[str, Any]: - pil = record.get("pil", {}) - return { - "file": record.get("file"), - "name": record.get("name"), - "size_bytes": record.get("size_bytes"), - "sha256": record.get("sha256"), - "content_format": record.get("content_format"), - "width": pil.get("width"), - "height": pil.get("height"), - "dpi": json.dumps(pil.get("dpi")), - } - - -def main() -> None: - flags, positional = set(), [] - for arg in sys.argv[1:]: - (flags.add if arg.startswith("--") else positional.append)(arg) - if len(positional) != 2 or flags - {"--pixels", "--pixels-full"}: - print(__doc__) - sys.exit(2) - pixel_mode = "full" if "--pixels-full" in flags else ("basic" if "--pixels" in flags else None) - if pixel_mode is not None and np is None: - print("pixel modes require numpy: pip install numpy") - sys.exit(2) - root, prefix = Path(positional[0]), positional[1] - files = sorted(str(p) for p in root.rglob("*") if p.is_file() and p.suffix.lower() in SUPPORTED) - # resume: skip files already present in the output of a previous - # (interrupted) run with the same prefix - gz = prefix.endswith(".gz") - jsonl_name = prefix if gz else f"{prefix}.jsonl" - csv_name = f"{prefix[:-3] if gz else prefix}.csv" - import gzip - - jsonl_path = Path(jsonl_name) - done: set[str] = set() - if jsonl_path.exists(): - opener = gzip.open if gz else open - with opener(jsonl_path, "rt") as existing: # type: ignore[arg-type] - for line in existing: - with contextlib.suppress(Exception): - done.add(json.loads(line).get("file", "")) - files = [f for f in files if f not in done] - print(f"scanning {len(files)} files under {root} ({len(done)} already done)") - n_done = 0 - text_opener = gzip.open if gz else open - csv_exists = Path(csv_name).exists() - with ( - text_opener(jsonl_path, "at") as jsonl, # type: ignore[arg-type] - open(csv_name, "a", newline="") as csvf, - ): - writer = csv.DictWriter(csvf, fieldnames=list(summary_row({}))) - if not csv_exists: - writer.writeheader() - for path_str in files: - try: - record = scan_file(Path(path_str), pixel_mode=pixel_mode) - except Exception as exc: # one corrupt file must not kill the scan - record = {"file": path_str, "error": _safe_str(exc)} - jsonl.write(json.dumps(record, default=str) + "\n") - writer.writerow(summary_row(record)) - n_done += 1 - if n_done % 500 == 0: - print(f" {n_done}/{len(files)}", flush=True) - print(f"done: {jsonl_name}, {csv_name}") - - -# --- optional pixel layer (--pixels / --pixels-full), requires numpy --- - -try: - import numpy as np - from numpy.lib.stride_tricks import sliding_window_view -except ImportError: - np = None # type: ignore[assignment] - -_MAX_SIDE = 2048 # downscale before analysis; stats are scale-robust -_DCT_BINS = np.linspace(-20.5, 20.5, 22) if np is not None else None -_AC_POSITIONS = [(0, 1), (1, 0), (1, 1), (0, 2), (2, 0), (2, 1), (1, 2), (0, 3)] -_FFT_BANDS = 8 -_BAYER_OFFSETS = [(1, 1), (1, -1)] # CFA diagonal periodicity candidates - - -def _arr_b64(arr: np.ndarray) -> dict[str, Any]: - """Compact array payload for the --full spatial maps.""" - return { - "shape": list(arr.shape), - "dtype": str(arr.dtype), - "base64": _b64(arr.tobytes()), - } - - -def _coarse(arr: np.ndarray, side: int = 64) -> np.ndarray: - """Downscale a 2D map to at most `side` on the long edge.""" - h, w = arr.shape - if max(h, w) <= side: - return arr - img = Image.fromarray(arr.astype(np.float32), mode="F") - img.thumbnail((side, side), Image.BILINEAR) - return np.asarray(img) - - -def phash(gray: np.ndarray) -> str: - """64-bit DCT perceptual hash (invertible to a rough layout; --full only).""" - img = Image.fromarray(gray.astype(np.float32), mode="F").resize((32, 32), Image.LANCZOS) - small = np.asarray(img) - m = _dct_matrix(32) - coeff = m @ small @ m.T - low = coeff[:8, :8].ravel()[1:] # drop DC - bits = low > np.median(low) - return f"{int(''.join('1' if b else '0' for b in bits), 2):016x}" - - -def full_artifacts( - gray: np.ndarray, - rgb: np.ndarray, - *, - ela: np.ndarray | None, - residual: np.ndarray | None, - phase: np.ndarray | None, -) -> dict[str, Any]: - """The privacy-lifting set: phash, thumbnail, ELA map, noise residual, - FFT phase. Maps are computed once by the caller and shared with the - scalar feature paths.""" - out: dict[str, Any] = {} - out["phash"] = phash(gray) - img = Image.fromarray(rgb.astype(np.uint8)) - img.thumbnail((128, 128), Image.LANCZOS) - buf = io.BytesIO() - img.save(buf, "JPEG", quality=70) - out["thumbnail_jpeg_b64"] = _b64(buf.getvalue()) - - if ela is not None: - out["ela_map"] = _arr_b64(_coarse(ela)) - if residual is not None: - clipped = np.clip(residual / 4.0, -1, 1) - out["noise_residual"] = _arr_b64(_coarse((clipped * 127).astype(np.int8))) - if phase is not None: - out["fft_phase"] = _arr_b64(_coarse(phase.astype(np.float32), 32)) - return out - - -def read_gray(path: Path) -> tuple[np.ndarray | None, np.ndarray | None, dict[str, Any]]: - """Decode to float32 grayscale (and RGB for color stats), downscaled.""" - try: - with Image.open(path) as img: - info: dict[str, Any] = {"width": img.width, "height": img.height} - if max(img.size) > _MAX_SIDE: - img.thumbnail((_MAX_SIDE, _MAX_SIDE), Image.LANCZOS) - rgb = np.asarray(img.convert("RGB"), dtype=np.float32) - gray = np.asarray(img.convert("L"), dtype=np.float32) - return gray, rgb, info - except Exception as exc: - return None, None, {"error": _safe_str(exc)} - - -def _dct_matrix(n: int = 8) -> np.ndarray: - """Orthonormal n x n DCT-II basis: M[i, j] = cos(pi (2j + 1) i / 2n).""" - i = np.arange(n)[:, None] - j = np.arange(n)[None, :] - m = np.cos(np.pi * (2 * j + 1) * i / (2 * n)) - m[0, :] *= 1 / np.sqrt(2) - return m * np.sqrt(2 / n) - - -_DCT_M = _dct_matrix() if np is not None else None - - -def dct_features(gray: np.ndarray) -> dict[str, Any]: - """AC coefficient histograms over 8x8 block DCT + Benford deviation.""" - h, w = gray.shape - h8, w8 = h // 8 * 8, w // 8 * 8 - if h8 < 8 or w8 < 8: - return {} - blocks = gray[:h8, :w8].reshape(h8 // 8, 8, w8 // 8, 8).swapaxes(1, 2) - coeff = np.einsum("ij,abjk,lk->abil", _DCT_M, blocks, _DCT_M) - hists = [] - lead_vals: list[np.ndarray] = [] - for dy, dx in _AC_POSITIONS: - vals = coeff[:, :, dy, dx].ravel() - hists.append(np.histogram(vals, bins=_DCT_BINS)[0].tolist()) - lead_vals.append(np.abs(vals)) - out: dict[str, Any] = {"dct_ac_hist": hists} - flat = np.abs(np.concatenate(lead_vals)) - flat = flat[flat >= 1] - if flat.size > 100: - leading = (flat / 10 ** np.floor(np.log10(flat))).astype(int) - leading = leading[(leading >= 1) & (leading <= 9)] - if leading.size > 100: - obs = np.bincount(leading, minlength=10)[1:10] / leading.size - ben = np.log10(1 + 1 / np.arange(1, 10)) - out["benford_mad"] = float(np.abs(obs - ben).mean()) - return out - - -def noise_residual_map(gray: np.ndarray) -> np.ndarray | None: - """High-pass residual. The map itself is spatial (shows edges), so it - is only stored under --full; the default mode keeps scalar stats.""" - if gray.shape[0] < 3 or gray.shape[1] < 3: - return None - k = np.array([[-1.0, -1.0, -1.0], [-1.0, 8.0, -1.0], [-1.0, -1.0, -1.0]]) - h, w = gray.shape - # k is float64, so the residual is float64 like the unchunked form - out = np.empty((h - 2, w - 2), dtype=np.float64) - # row-chunked: the (win * k) temp is ~150 MB at 2048px if materialized - # whole; per-element 9-tap sums are computed in the same order, so the - # result is bit-identical to the unchunked form - for y0 in range(0, h - 2, 256): - y1 = min(y0 + 256, h - 2) - win = sliding_window_view(gray[y0 : y1 + 2], (3, 3)) - out[y0:y1] = (win * k).sum(axis=(-1, -2)) - return out - - -def noise_features(residual: np.ndarray) -> dict[str, Any]: - """High-pass residual std/kurtosis; the residual map is NOT stored.""" - flat = residual.ravel() - std = float(flat.std()) - if std < 1e-9: - return {"noise_std": 0.0, "noise_kurtosis": 0.0} - z = (flat - flat.mean()) / std - return {"noise_std": std, "noise_kurtosis": float((z**4).mean() - 3.0)} - - -def fft_decompose(gray: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None: - """Log-magnitude (fftshifted) and phase of the image spectrum.""" - if min(gray.shape) < 32: - return None - spectrum = np.fft.fftshift(np.fft.fft2(gray - gray.mean())) - return np.log1p(np.abs(spectrum)), np.angle(spectrum) - - -def fft_features(mag: np.ndarray) -> dict[str, Any]: - """Radial magnitude band energies (no phase) + CFA periodicity peaks.""" - h, w = mag.shape - cy, cx = h // 2, w // 2 - # 1D broadcast instead of an mgrid: saves ~160 MB of int64 temporaries - # at 2048px; the squares are exact in float64 (values < 2^53), so band - # means are identical to the mgrid form - r2y = (np.arange(h, dtype=np.float64) - cy) ** 2 - r2x = (np.arange(w, dtype=np.float64) - cx) ** 2 - r = np.sqrt(r2y[:, None] + r2x[None, :]) - r_max = r.max() - bands = [] - for i in range(_FFT_BANDS): - mask = (r >= r_max * i / _FFT_BANDS) & (r < r_max * (i + 1) / _FFT_BANDS) - bands.append(float(mag[mask].mean()) if mask.any() else 0.0) - # Bayer CFA shows as symmetric peaks at half the Nyquist on diagonals - peaks = [] - for dy, dx in _BAYER_OFFSETS: - y, x = cy + dy * (h // 4), cx + dx * (w // 4) - neighborhood = mag[y - 2 : y + 3, x - 2 : x + 3] - peaks.append(float(neighborhood.max() - mag.mean())) - return {"fft_band_energy": bands, "cfa_peaks": peaks, "cfa_peak": max(peaks)} - - -def ela_map(rgb: np.ndarray) -> np.ndarray | None: - """Absolute per-pixel error after a q90 JPEG re-save.""" - try: - img = Image.fromarray(rgb.astype(np.uint8)) - buf = io.BytesIO() - img.save(buf, "JPEG", quality=90) - buf.seek(0) - resaved = np.asarray(Image.open(buf).convert("RGB"), dtype=np.float32) - except Exception: - return None - if resaved.shape != rgb.shape: - return None - return np.abs(rgb - resaved).mean(axis=-1) - - -def ela_features(err: np.ndarray) -> dict[str, Any]: - """Error-level stats after a q90 JPEG re-save; global only, no map.""" - return {"ela_mean": float(err.mean()), "ela_p95": float(np.percentile(err, 95))} - - -def gradient_features(gray: np.ndarray) -> dict[str, Any]: - gy, gx = np.gradient(gray) - mag = np.sqrt(gx**2 + gy**2) - hist = np.histogram(mag, bins=10, range=(0, 255))[0].tolist() - lap = np.gradient(gy, axis=0) + np.gradient(gx, axis=1) - return {"gradient_hist": hist, "laplacian_var": float(lap.var())} - - -def color_features(rgb: np.ndarray) -> dict[str, Any]: - small = rgb[::4, ::4] # decimate; histogram is position-blind anyway - bins = (small / 256 * 4).astype(int).clip(0, 3) - idx = bins[..., 0] * 16 + bins[..., 1] * 4 + bins[..., 2] - hist = np.bincount(idx.ravel(), minlength=64).tolist() - mx = small.max(axis=-1) - mn = small.min(axis=-1) - sat = np.where(mx > 0, (mx - mn) / np.maximum(mx, 1e-6), 0) - return { - "color_hist_4x4x4": hist, - "saturation_mean": float(sat.mean()), - "value_mean": float(mx.mean() / 255), - } - - -if __name__ == "__main__": - main() diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index 53efc1b..2dfc324 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -22,6 +22,7 @@ from __future__ import annotations import base64 import itertools import logging +import struct from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast @@ -30,6 +31,8 @@ from remove_ai_watermarks._internal.c2pa import ( cbor_text_after, extract_c2pa_info, soft_binding_vendors_in, + synthid_vendors_in, + synthid_verdict, ) from remove_ai_watermarks._internal.constants import ( C2PA_AI_TOOLS, @@ -37,6 +40,7 @@ from remove_ai_watermarks._internal.constants import ( C2PA_IDENTITY_AI_ORGS, C2PA_ISSUERS, ) +from remove_ai_watermarks._internal.schema import require_schema_version from remove_ai_watermarks.metadata import ( AI_METADATA_KEYS, AIGC_MARKERS, @@ -70,6 +74,10 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Stable JSON contract for callers that pass a verdict between services. Bump this +# only for a breaking shape or semantic change; adding optional fields is compatible. +PROVENANCE_REPORT_SCHEMA_VERSION = 1 + # How much of a non-PNG container to binary-scan for the C2PA issuer. _SCAN_BYTES = 1024 * 1024 @@ -169,7 +177,32 @@ def _external_metadata(value: Any) -> tuple[list[tuple[str, Any]], bytes]: """Index nested metadata and recover common encoded binary values in one pass.""" pairs: list[tuple[str, Any]] = [] parts: list[bytes] = [] - diagnostic_keys = {"error", "kind"} + diagnostic_keys = { + "artifacts", + "birthtime", + "color", + "content_format", + "dct", + "ela", + "error", + "extension", + "fft", + "file", + "filename", + "full", + "gradient", + "kind", + "mtime", + "name", + "noise", + "path", + "pixel", + "provenance", + "sha256", + "signals", + "size_bytes", + "timing_ms", + } def visit(item: Any) -> None: if isinstance(item, dict): @@ -177,7 +210,6 @@ def _external_metadata(value: Any) -> tuple[list[tuple[str, Any]], bytes]: for key, nested in mapping.items(): key_text = str(key) pairs.append((key_text, nested)) - parts.append(key_text.encode("utf-8", "replace")) if key_text.lower() in diagnostic_keys: continue if isinstance(nested, str) and (key_text == "base64" or key_text.endswith("_base64")): @@ -241,17 +273,69 @@ def _external_exif_generator(pairs: list[tuple[str, Any]], scan: bytes) -> str | return generator_from_metadata(candidates, scan) +def _metadata_source_kind(info: dict[str, Any], scan: bytes) -> str | None: + """Normalize the source type wherever it is carried: C2PA or IPTC/XMP. + + A composite marker contains ``TrainedAlgorithmicMedia`` as a substring, so it + is removed before looking for a standalone full-generation marker. When a file + genuinely carries both kinds, full generation wins. + """ + structured = info.get("ai_source_kind") + without_composites = scan.replace(b"compositeWithTrainedAlgorithmicMedia", b"").replace(b"compositeSynthetic", b"") + generated = structured == "generated" or any( + marker in without_composites for marker in (b"trainedAlgorithmicMedia", b"TrainedAlgorithmicMedia") + ) + if generated: + return "generated" + if structured == "enhanced" or any( + marker in scan for marker in (b"compositeWithTrainedAlgorithmicMedia", b"compositeSynthetic") + ): + return "enhanced" + return None + + def evidence_from_metadata_record( record: dict[str, Any], *, path: Path, c2pa_manifest_store: str | dict[str, Any] | None = None ) -> ProvenanceEvidence: """Normalize an externally collected metadata record into provenance evidence. - The record may contain arbitrary nested dictionaries and lists. Text, bytes, - hexadecimal values prefixed with ``hex:``, and fields named ``base64`` or - ending in ``_base64`` are included in the shared byte scan. No source file is - opened. + Unversioned external records may contain arbitrary nested dictionaries and + lists. Versioned native records accept only the source-derived fields emitted by + ``collect_metadata_record``; other native record types and unknown schema + versions are rejected. No source file is opened. """ - pairs, scan = _external_metadata(record) + from remove_ai_watermarks.metadata_record import METADATA_RECORD_SCHEMA_VERSION, METADATA_RECORD_TYPE + + # Records produced by ``collect_metadata_record`` are a versioned transport + # contract. Only their source-derived fields are evidence: the filename, + # container label and schema bookkeeping describe the collector and must never + # become detector input. Shape-detect the pre-versioned form as well so records + # emitted by 0.26 remain safe and readable. + record_type = record.get("record_type") + if record_type not in (None, METADATA_RECORD_TYPE): + raise ValueError(f"Unsupported metadata record type: {record_type!r}") + if record_type == METADATA_RECORD_TYPE: + require_schema_version( + record.get("schema_version"), + contract="provenance metadata", + supported=(METADATA_RECORD_SCHEMA_VERSION,), + ) + status = record.get("status") + if status == "error": + raise ValueError("Provenance metadata collection failed") + if status != "complete": + raise ValueError(f"Unsupported provenance metadata collection status: {status!r}") + is_portable_record = record_type == METADATA_RECORD_TYPE or { + "container", + "metadata_base64", + "tail_base64", + }.issubset(record) + evidence_record = ( + {key: record[key] for key in ("metadata_base64", "tail_base64", "pil", "exif") if key in record} + if is_portable_record + else record + ) + pairs, scan = _external_metadata(evidence_record) store = c2pa_manifest_store if store is None: candidate = record.get("c2pa_store") @@ -358,13 +442,13 @@ class ProvenanceReport: is_ai_generated: bool | None # True / False is never asserted; None = unknown platform: str | None confidence: str # "high" | "medium" | "none" - # Coarse AI-origin kind from the C2PA digital-source-type, so a caller can - # branch on full generation vs an AI-touched real photo: + # Coarse AI-origin kind from a C2PA or standalone IPTC/XMP digital-source-type, + # so a caller can branch on full generation vs an AI-touched real photo: # "generated" -- digitalSourceType trainedAlgorithmicMedia (fully AI). # "enhanced" -- compositeWithTrainedAlgorithmicMedia (real content with an # AI-composited region; scrub the AI region, keep the photo). - # None -- no C2PA AI source-type (verdict, if AI, came from another - # signal: IPTC, AIGC, local gen params, xAI, ...). + # None -- no AI digital-source-type (verdict, if AI, came from another + # signal: AIGC, local gen params, xAI, ...). ai_source_kind: str | None = None # True when the AI verdict rests on a metadata or embedded-invisible signal # (C2PA AI issuer / SynthID proxy, IPTC, AIGC, local gen params, EXIF/xAI, or @@ -383,6 +467,42 @@ class ProvenanceReport: # inconsistent -- a strong tell of spoofed, transplanted, or laundered metadata. integrity_clashes: list[str] = field(default_factory=list[str]) + def to_dict( + self, + *, + schema_version: int = PROVENANCE_REPORT_SCHEMA_VERSION, + ) -> dict[str, Any]: + """Return the versioned, JSON-safe verdict contract. + + ``path`` is deliberately omitted. It is extraction context, not part of the + verdict, and local filesystem paths should not cross a service boundary. + Request an explicit schema for a long-lived transport consumer. + """ + schema_version = require_schema_version( + schema_version, + contract="provenance report", + supported=(1,), + ) + return { + "schema_version": schema_version, + "is_ai_generated": self.is_ai_generated, + "platform": self.platform, + "confidence": self.confidence, + "ai_source_kind": self.ai_source_kind, + "ai_from_metadata": self.ai_from_metadata, + "watermarks": list(self.watermarks), + "signals": [ + { + "name": signal.name, + "detail": signal.detail, + "confidence": signal.confidence, + } + for signal in self.signals + ], + "caveats": list(self.caveats), + "integrity_clashes": list(self.integrity_clashes), + } + def extract_provenance_evidence(image_path: Path) -> ProvenanceEvidence: """Read all file-backed metadata needed by provenance verdict logic once.""" @@ -447,6 +567,72 @@ _DEVICE_C2PA_PLATFORM: tuple[tuple[bytes, str], ...] = ( ) +def _metadata_region(head: bytes) -> bytes: + """The part of the scan buffer that can hold metadata, with the coded pixels cut out. + + The vendor registries are matched as raw substrings, and the shortest tokens are + four and five bytes (``Bria``, ``Adobe``, ``Canva``). Over a megabyte of compressed + pixel data a four-byte sequence appears by chance about once in three thousand + images -- measured: ``Bria`` matched inside the entropy-coded scan of 4 of 14,707 + corpus JPEGs, in none of which the manifest names Bria. That is not a cosmetic + mislabel, because the Bria entry carries ``asserts_ai``: a chance match can declare + an image AI-generated. + + ``c2pa_marker_in`` already refuses a bare ``c2pa`` substring for the same reason. + This is the same defence for the registries: they see the container's metadata and + not its pixels. + + JPEG keeps the marker segments before the entropy-coded scan, PNG every chunk but + ``IDAT``, and both keep the trailer past the end marker. Anything ``scan_head`` + APPENDED past the window is metadata by construction (late chunks, boxes, decoder + text), so it is always kept and never walked -- walking it is what produced 11 MB + records and a phantom AIGC signal in the record collector. + + Trimming happens only when the container actually parses: a JPEG whose marker walk + reaches the coded scan, a PNG whose chunk walk reaches ``IDAT``. Anything else -- + a malformed container, a synthetic blob, a format with no walker here -- is + returned whole. Cutting a buffer this function did not understand would drop real + evidence to avoid a chance match, which is the wrong way round. + """ + raw, appended = head[:_SCAN_BYTES], head[_SCAN_BYTES:] + if raw[:2] == b"\xff\xd8": + index, size = 2, len(raw) + while index + 1 < size: + if raw[index] != 0xFF: + return head # not a marker boundary: the walk is lost, keep everything + marker = raw[index + 1] + if marker in (0xDA, 0xD9): # SOS / EOI: the coded scan follows + end = raw.rfind(b"\xff\xd9") + return raw[:index] + (raw[end + 2 :] if end >= index else b"") + appended + if 0xD0 <= marker <= 0xD7 or marker == 0x01: + index += 2 + continue + if index + 4 > size: + break + length = int.from_bytes(raw[index + 2 : index + 4], "big") + if length < 2 or index + 2 + length > size: + break + index += 2 + length + return head # ran out of buffer before the scan: nothing was skipped anyway + if raw[:8] == b"\x89PNG\r\n\x1a\n": + out = bytearray() + position, size, saw_idat = 8, len(raw), False + while position + 8 <= size: + (length,) = struct.unpack(">I", raw[position : position + 4]) + chunk_type = raw[position + 4 : position + 8] + start = position + 8 + if chunk_type == b"IDAT": + saw_idat = True + else: + out += chunk_type + raw[start : start + min(length, size - start)] + position = start + length + 4 + if chunk_type == b"IEND": + out += raw[position:] + break + return bytes(out) + appended if saw_idat else head + return head + + def _first_token_match(head: bytes, table: tuple[tuple[bytes, str], ...]) -> str | None: """First platform in ``table`` whose token appears in ``head``, else None. @@ -883,23 +1069,22 @@ def _identify_from_evidence( # score, the latter can be a by-product of our own SDXL removal pass, so # neither is a trustworthy "the generator stamped its identity" claim. ai_vendor_claims: dict[str, str] = {} - camera_label = _device_platform(head) - signer_label = _signer_platform(head) + # The vendor registries match short raw substrings, so they read the container's + # metadata rather than its pixels -- see `_metadata_region`. Every other check + # below keeps the full buffer: their markers are long and distinctive. + region = _metadata_region(head) + camera_label = _device_platform(region) + signer_label = _signer_platform(region) # ── C2PA Content Credentials ──────────────────────────────────── has_c2pa = bool(info) or c2pa_marker_in(head) - issuers = [info["issuer"]] if info.get("issuer") else _issuers_in(head) + issuers = [info["issuer"]] if info.get("issuer") else _issuers_in(region) # Full AI generation (trainedAlgorithmicMedia) vs an AI-enhanced real photo # (compositeWithTrainedAlgorithmicMedia). The structured kind is parsed once in # _internal.c2pa._populate_registry_fields (covers PNG + any container the c2pa-python # reader handles); fall back to a raw head scan for the non-PNG raw-blob path # where extract_c2pa_info returns {}. Full generation wins when both appear. - c2pa_source_kind = info.get("ai_source_kind") - if c2pa_source_kind is None: - if b"trainedAlgorithmicMedia" in head: - c2pa_source_kind = "generated" - elif b"compositeWithTrainedAlgorithmicMedia" in head: - c2pa_source_kind = "enhanced" + source_kind = _metadata_source_kind(info, head) # An identity-AI issuer (a pure-generator brand like Dreamina) asserts AI even # without a digitalSourceType -- some ByteDance/Dreamina manifests ship no # trainedAlgorithmicMedia, so the registered generator name is the only signal. @@ -907,7 +1092,7 @@ def _identify_from_evidence( # does not reopen the incidental-mention problem the common-word issuers have. issuer_blob = " ".join(issuers) c2pa_identity_ai = has_c2pa and any(org in issuer_blob for org in C2PA_IDENTITY_AI_ORGS) - c2pa_is_ai = c2pa_source_kind is not None or c2pa_identity_ai + c2pa_is_ai = source_kind is not None or c2pa_identity_ai # Generator string (for the signal detail): structured for PNG, CBOR-scanned # for other containers. Best-effort -- some manifests key it as # `claim_generator_info` (Pixel), so this can be None even when a device is @@ -915,7 +1100,7 @@ def _identify_from_evidence( generator = ( info.get("claim_generator") or cbor_text_after(head, b"claim_generator") - or (", ".join(tools) if (tools := _ai_tools_in(head)) else None) + or (", ".join(tools) if (tools := _ai_tools_in(region)) else None) ) # Platform: a distinctive device/camera token in the manifest wins (it is the # signer/producer), then an editing-app/AI-device signer (Samsung Galaxy, @@ -950,9 +1135,24 @@ def _identify_from_evidence( platform = f"C2PA signer: {cloud_vendor} (cloud manifest)" # ── SynthID metadata proxy ────────────────────────────────────── - # get_ai_metadata already sets synthid_watermark for both PNG (caBX parser) - # and non-PNG (its own synthid_source fallback), so no extra scan is needed. + # Structured first (the PNG caBX parser and the manifest store both fill + # `synthid_watermark`), then the byte scan for the containers that keep the + # manifest where no parser reaches it. + # + # The scan lives HERE, in the verdict, and not in extraction, for the same reason + # `soft_binding` below does: extraction has two implementations -- one reading a + # file, one reading a portable record -- and a rule that lives in only one of them + # is a rule the other silently lacks. It did: 74 corpus images reported SynthID + # through `identify` and not through the record, because `get_ai_metadata`'s own + # fallback has no counterpart on the record side. `get_ai_metadata` keeps its copy + # for its own callers; the verdict no longer depends on which extractor ran. synthid = meta.get("synthid_watermark") + # The literal byte checks mirror `metadata.synthid_source` exactly rather than + # reusing the derived `has_c2pa` / `source_kind` above, which are broader: + # the file path's answer must not move. + trained_source = b"trainedAlgorithmicMedia" in head or b"TrainedAlgorithmicMedia" in head + if not synthid and trained_source and c2pa_marker_in(head) and (vendors := synthid_vendors_in(region)): + synthid = synthid_verdict(", ".join(vendors)) if synthid: watermarks.append(f"SynthID watermark, inferred from C2PA metadata ({synthid})") caveats.append(_SYNTHID_CAVEAT) @@ -964,7 +1164,7 @@ def _identify_from_evidence( # ── C2PA soft-binding: a named forensic/third-party watermark vendor ─ # (Adobe TrustMark, Digimarc, Imatag, ...). Present in the manifest even when # the watermark itself can't be decoded; names whose watermark stamped the pixels. - soft_binding = meta.get("soft_binding") or (", ".join(v) if (v := soft_binding_vendors_in(head)) else None) + soft_binding = meta.get("soft_binding") or (", ".join(v) if (v := soft_binding_vendors_in(region)) else None) if soft_binding: signals.append(Signal("soft_binding", f"C2PA soft binding: {soft_binding}", "high")) watermarks.append(f"Forensic watermark soft binding ({soft_binding})") @@ -1136,9 +1336,9 @@ def _identify_from_evidence( is_ai_generated=is_ai, platform=platform, confidence=confidence, - # Only meaningful when the AI verdict actually came from the C2PA source - # type; a non-C2PA AI signal (IPTC/AIGC/local gen) leaves it None. - ai_source_kind=c2pa_source_kind if (is_ai and has_c2pa) else None, + # Meaningful for the same digitalSourceType whether carried by C2PA or a + # standalone IPTC/XMP label. Other AI signals leave it None. + ai_source_kind=source_kind if (is_ai and (has_c2pa or iptc)) else None, ai_from_metadata=ai_from_metadata, watermarks=watermarks, signals=signals, @@ -1170,6 +1370,16 @@ def identify_from_evidence( ) +def identify_metadata_record(record: dict[str, Any], *, path: Path) -> ProvenanceReport: + """Build a metadata-only verdict from a portable metadata record. + + This is the service-integration entry point: the source file is never opened, + and callers receive the same verdict as the explicit + ``evidence_from_metadata_record`` / ``identify_from_evidence`` sequence. + """ + return identify_from_evidence(evidence_from_metadata_record(record, path=path)) + + def identify( image_path: Path, *, diff --git a/src/remove_ai_watermarks/metadata.py b/src/remove_ai_watermarks/metadata.py index 0c7939e..eeae2e7 100644 --- a/src/remove_ai_watermarks/metadata.py +++ b/src/remove_ai_watermarks/metadata.py @@ -18,6 +18,11 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable from pathlib import Path +from remove_ai_watermarks._internal.constants import ( + PNG_METADATA_CHUNKS, + RIFF_METADATA_CHUNKS, +) + logger = logging.getLogger(__name__) # Smaller scan_head window for the cheap marker checks (has_ai_metadata, @@ -236,11 +241,6 @@ def _is_ai_value(value: str) -> bool: return any(token in value_lower for token in AI_GENERATOR_TOKENS) -# PNG ancillary chunks that can carry provenance metadata (XMP, EXIF, text). -# Never IDAT -- that is the compressed pixel stream. -_PNG_META_CHUNKS: frozenset[bytes] = frozenset({b"tEXt", b"iTXt", b"zTXt", b"eXIf", b"iCCP"}) - - def _png_late_metadata(image_path: Path, window: int) -> bytes: """Payloads of PNG metadata chunks that start *beyond* the first ``window`` bytes, found by seeking past the (large) ``IDAT`` pixel stream. @@ -272,7 +272,7 @@ def _png_late_metadata(image_path: Path, window: int) -> bytes: # Clamp the attacker-controlled 32-bit length to the bytes that # actually remain, so a malformed huge length can't allocate GBs. safe_length = max(0, min(length, file_size - data_start)) - if chunk_type in _PNG_META_CHUNKS and data_start >= window: + if chunk_type in PNG_METADATA_CHUNKS and data_start >= window: f.seek(data_start) out += f.read(safe_length) # Advance by the CLAMPED length: a malformed/inflated `length` that @@ -285,6 +285,55 @@ def _png_late_metadata(image_path: Path, window: int) -> bytes: return bytes(out) +def _riff_late_metadata(image_path: Path, window: int, *, max_total: int = 4 * 1024 * 1024) -> bytes: + """Payloads of RIFF metadata chunks that start *beyond* the first ``window`` + bytes, found by stepping over the (large) coded-image chunk. + + The WebP layout puts ``XMP ``/``EXIF`` AFTER the pixels, so a fixed read can stop + before an IPTC or C2PA AI label. This is the RIFF analogue of + :func:`_png_late_metadata`; it returns only chunks past ``window`` so bytes + already in the head are not duplicated, and empty when there are none. + + ``max_total`` caps what a metadata scan can pull into memory, the same ceiling + ``isobmff.scan_c2pa_region`` applies. Clamping each chunk to the bytes that remain + is not enough on its own: a corrupt or crafted file can declare one ``XMP `` chunk + spanning most of itself, and this runs on the memoized verdict path for images from + arbitrary sources. A label that needs more than 4 MB of XMP does not exist. + """ + out = bytearray() + try: + with open(image_path, "rb") as f: + if f.read(4) != b"RIFF": + return b"" + f.seek(0, 2) + file_size = f.tell() + f.seek(4) + declared_size = f.read(4) + if len(declared_size) < 4: + return b"" + container_end = min(file_size, 8 + struct.unpack("= window: + f.seek(start) + out += f.read(min(safe_length, max_total - len(out))) + position = start + safe_length + (safe_length & 1) # chunks are word-aligned + except OSError as exc: + logger.debug("RIFF late-metadata scan failed on %s: %s", image_path, exc) + return b"" + return bytes(out) + + def _stat_key(image_path: Path) -> tuple[str, int, int] | None: """Cache key identifying this file's exact CONTENT, or None when it cannot stat. @@ -306,11 +355,16 @@ def scan_head(image_path: Path, size: int = 1024 * 1024) -> bytes: past large boxes like ``mdat``) and PNG ``tEXt`` / ``iTXt`` / ``eXIf`` chunks (seeking past ``IDAT``). + A file at least ``size`` bytes long additionally gets the metadata text its + decoder can reach but a raw read cannot (:func:`_decoder_visible_text`): a + compressed PNG ``zTXt`` packet, or a chunk past the window in a container with no + late-chunk reader here. A file that fits inside ``size`` is exactly + ``f.read(size)``, since the raw read already holds every byte. + This is the shared input for every C2PA / AIGC / IPTC byte scan. The extensions catch a manifest or XMP packet placed AFTER the media data -- a non-faststart MP4 manifest, or a PNG XMP packet appended after the pixels -- - which a fixed first-MB read would miss. For other inputs, and for files that - fit within ``size``, it is exactly ``f.read(size)`` -- behavior-neutral. + which a fixed first-MB read would miss. The result is memoized per (path, size, mtime): one ``identify``/``get_ai_metadata`` call fans out to ~8 byte-scan detectors that each call this on the same file, so @@ -347,9 +401,63 @@ def _scan_head_impl(image_path: Path, size: int) -> bytes: # len(head) == size means the file is at least `size` bytes, so metadata # chunks may lie beyond the window; otherwise the whole PNG is in `head`. head += _png_late_metadata(image_path, size) + elif head[:4] == b"RIFF" and head[8:12] == b"WEBP" and len(head) == size: + head += _riff_late_metadata(image_path, size) + if len(head) >= size: + head += _decoder_visible_text(image_path, head) return head +# Text values the image decoder can reach that a raw byte read cannot. Bounded: a +# packet larger than this is not a provenance label. +_DECODED_TEXT_LIMIT = 512 * 1024 +# Decoder values that are binary payloads with their own readers, not metadata text. +# An ICC profile is colour data and can run to hundreds of kilobytes; appending it +# would bloat the buffer every later detector re-scans, for no signal. +_DECODER_BINARY_KEYS = frozenset({"icc_profile"}) + + +def _decoder_visible_text(image_path: Path, head: bytes) -> bytes: + """Metadata text PIL can decode but the raw window does not contain. + + This is the last of two layers, not the first. Metadata placed BEYOND the window + is the structural readers' job (``_png_late_metadata``, ``_riff_late_metadata``, + the ISOBMFF box walk), and they work on a file no decoder can open. What is left + for this one is metadata the bytes do not spell at all: + + * COMPRESSED -- a PNG ``zTXt`` chunk is zlib-deflated, so an XMP packet carrying + a TC260 AIGC label is unreadable as bytes while PIL inflates it on open. + + It stays container-agnostic on purpose: it is the net under a placement no + structural reader here knows about yet. + + Only text ALREADY MISSING from ``head`` is appended, so the common case adds + nothing and no detector sees a value twice. Skipped entirely when the file fits + inside the window, since then the raw read already holds every byte. + """ + try: + from PIL import Image + + with Image.open(image_path) as img: + values = [value for key, value in img.info.items() if key not in _DECODER_BINARY_KEYS] + except Exception as exc: # a container PIL cannot open: the raw scan stands alone + logger.debug("decoder-visible text unavailable for %s: %s", image_path, exc) + return b"" + + out = bytearray() + for value in values: + if isinstance(value, str): + encoded = value.encode("utf-8", "replace") + elif isinstance(value, bytes): + encoded = value + else: + continue + if len(encoded) > _DECODED_TEXT_LIMIT or not encoded or encoded in head: + continue + out += b"\x00" + encoded + return bytes(out) + + def has_ai_metadata(image_path: Path) -> bool: """Check if an image contains AI-generation metadata. @@ -1485,3 +1593,16 @@ def xai_signature(image_path: Path) -> bool: if key is None: return _xai_signature_impl(image_path) return _xai_signature_cached(*key) + + +# ── Shared with the portable metadata record ──────────────────────── +# `metadata_record` must read exactly the windows and markers the file path reads: a +# record built from a different window is a record whose verdict can disagree with +# `identify` on the same image. Aliased rather than renamed because the private names +# are load-bearing in this module's own tests and in a corpus script. +QUICK_SCAN_BYTES = _QUICK_SCAN_BYTES +SAMSUNG_EDITOR_MARKER = _SAMSUNG_EDITOR_MARKER +read_file_tail = _read_file_tail +png_late_metadata = _png_late_metadata +riff_late_metadata = _riff_late_metadata +exif_text = _exif_text diff --git a/src/remove_ai_watermarks/metadata_record.py b/src/remove_ai_watermarks/metadata_record.py new file mode 100644 index 0000000..ade323b --- /dev/null +++ b/src/remove_ai_watermarks/metadata_record.py @@ -0,0 +1,404 @@ +"""Collect one image's provenance metadata into a portable, JSON-safe record. + +WHY THIS EXISTS + +``extract_provenance_evidence`` reads a file and hands back evidence in memory, so +collection and verdict must happen in the same process, on the machine holding the +image. This module splits them: collect here, judge anywhere, from a record that +survives JSON. + + record = collect_metadata_record(path) # touches the file + evidence = evidence_from_metadata_record(record, path=path) + report = identify_from_evidence(evidence) # touches nothing + +WHAT GOES IN, AND WHY NOT SIMPLY THE FILE HEAD + +The verdict reads a scan buffer that ``scan_head`` fills with the first mebibyte of +the file. Shipping that verbatim would make a record larger than a phone photo's +worth of metadata by two orders of magnitude, because for a PNG almost all of that +mebibyte is compressed pixel data in ``IDAT`` -- bytes no provenance token can ever +live in. A record carries the metadata REGIONS instead, walked per container: the +JPEG marker segments before the coded scan, every PNG chunk but ``IDAT``, the RIFF +chunks that are not coded image, the ISOBMFF provenance boxes, and in every case the +container's trailer. + +COMPLETENESS IS A MEASURED PROPERTY, NOT A CLAIM + +A region walker is only correct if nothing the verdict reads falls outside the +regions it keeps, and no test over fixtures can establish that: the failure mode is +a container placement nobody thought of. The contract is therefore ALSO verified +against the file path over a real corpus -- same image, both paths, identical +``ProvenanceReport``. + +The placements that defeated an earlier draft of this collector, and the reason each +rule below exists, are recorded in ``docs/module-internals.md`` under "Portable +metadata record". +""" + +from __future__ import annotations + +import base64 +import logging +import struct +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + +from remove_ai_watermarks._internal.constants import PNG_SIGNATURE, RIFF_CODED_IMAGE_CHUNKS +from remove_ai_watermarks._internal.schema import require_schema_version +from remove_ai_watermarks.metadata import ( + QUICK_SCAN_BYTES, + SAMSUNG_EDITOR_MARKER, + exif_text, + read_file_tail, +) + +logger = logging.getLogger(__name__) + +# The structural walk covers the same window the file path reads raw, so the two +# cannot disagree about a chunk type inside it. A smaller window would be cheaper but +# opens a blind spot: past the window only ``png_late_metadata``'s ALLOWLIST is +# collected, while the file path still sees every chunk type up to its own window -- +# and a C2PA ``caBX`` chunk is in neither that allowlist nor ``IDAT``. Walking here +# costs little because the payload of the pixel stream is skipped, not copied. +HEAD_WINDOW = 1024 * 1024 +# The window searched for the container's end marker. Matches the quick-scan window +# the file path uses when it goes looking for a Samsung trailer, so a trailer visible +# to one path is visible to the other. +TAIL_WINDOW = QUICK_SCAN_BYTES +# Kept from the tail when no end marker is found, so an unrecognized container still +# contributes its last bytes without carrying half a photo. +UNKNOWN_TRAILER_WINDOW = 64 * 1024 + +# PNG text keys the file path reads for a generator tag, in ITS order. NovelAI stamps +# Software/Source/Title rather than EXIF, and the first match wins, so order matters. +_GENERATOR_TEXT_KEYS = ("Software", "Source", "Title", "Description") +# Stable transport contract for records produced by this module. The version is +# deliberately separate from the verdict version: collection and interpretation can +# evolve independently as long as old records remain readable. +METADATA_RECORD_SCHEMA_VERSION = 1 +METADATA_RECORD_TYPE = "provenance_metadata" + + +def _jpeg_regions(data: bytes) -> bytes: + """Every marker segment up to the entropy-coded scan, plus the trailer after EOI. + + The scan itself is skipped by walking to SOS and then jumping to the trailing + EOI, so a 20 MB photo contributes only its markers. + + TWIN: ``metadata._strip_jpeg_metadata_lossless`` walks the same marker chain. The + two were left separate on purpose -- that one couples the walk to "return False and + fall back to a PIL re-encode", a decision the lossless strip path owns and this one + must not inherit -- so a fix to marker handling belongs in BOTH. + """ + out = bytearray() + index, size = 2, len(data) + while index + 1 < size: + if data[index] != 0xFF: + break # malformed boundary: keep what was collected, the tail still follows + marker = data[index + 1] + if marker in (0xDA, 0xD9): # SOS / EOI: the coded scan follows + break + if 0xD0 <= marker <= 0xD7 or marker == 0x01: # standalone, no length + index += 2 + continue + if index + 4 > size: + break + segment_length = int.from_bytes(data[index + 2 : index + 4], "big") + end = index + 2 + segment_length + if segment_length < 2 or end > size: + break + out += data[index:end] + index = end + return bytes(out) + + +def _png_regions(data: bytes) -> bytes: + """Every chunk except the ``IDAT`` payloads, plus whatever follows IEND. + + TWIN: ``metadata._png_late_metadata`` walks the same chunk chain by SEEKING over + the file rather than over a buffer, and keeps an allowlist rather than skipping + ``IDAT``. Both filters are deliberate: inside the window the file path sees every + chunk type raw, past it only the allowlist survives. + """ + out = bytearray() + size = len(data) + position = len(PNG_SIGNATURE) + while position + 8 <= size: + (length,) = struct.unpack(">I", data[position : position + 4]) + chunk_type = data[position + 4 : position + 8] + start = position + 8 + # Clamp the length to the bytes that remain: a malformed 32-bit length must + # not push the walk past EOF and abandon a genuine label chunk after it. + safe_length = max(0, min(length, size - start)) + if chunk_type != b"IDAT": + out += chunk_type + data[start : start + safe_length] + position = start + safe_length + 4 # payload + CRC + if chunk_type == b"IEND": + out += data[position:] # a trailer past IEND is metadata too + break + return bytes(out) + + +def _riff_regions(data: bytes) -> bytes: + """Every RIFF chunk except the coded image payloads. + + TWIN: ``metadata._riff_late_metadata`` (seek-based, past the scan window) and + ``_internal.riff`` (AVI ``LIST/INFO``). Same chunk-stepping arithmetic, three + input models. + """ + out = bytearray(data[:12]) # 'RIFF' + size + 'WEBP' + declared_end = 8 + struct.unpack("= 12 else len(data) + size = min(len(data), declared_end) + position = 12 + while position + 8 <= size: + chunk_type = data[position : position + 4] + (length,) = struct.unpack(" bytes: + """Header window plus the provenance regions the bounded box walkers find. + + ISOBMFF hides a manifest in a ``uuid``/``jumb`` box that can sit after a + multi-megabyte ``mdat``, and a TC260 label in ``moov.udta``. Both walkers seek + rather than read the media, so neither pulls the payload in. + """ + from remove_ai_watermarks._internal.isobmff import scan_c2pa_region, tc260_aigc_payloads + + out = bytearray(head[:HEAD_WINDOW]) + try: + out += scan_c2pa_region(image_path) + except Exception as exc: + logger.debug("ISOBMFF C2PA region scan failed on %s: %s", image_path, exc) + try: + for payload in tc260_aigc_payloads(image_path): + out += payload + except Exception as exc: + logger.debug("ISOBMFF TC260 scan failed on %s: %s", image_path, exc) + return bytes(out) + + +def _container_regions(image_path: Path, head: bytes) -> tuple[str, bytes]: + """(container label, metadata bytes) for the container ``head`` starts with. + + ``head`` must be the file's raw first bytes. Handing this the ``scan_head`` + buffer instead is a trap: that buffer is the head CONCATENATED with late metadata + payloads, so a structural walk runs off the end of the real head and parses the + appended bytes as chunks, inflating the record and creating false signals. + """ + from remove_ai_watermarks._internal.isobmff import is_isobmff + from remove_ai_watermarks.metadata import png_late_metadata, riff_late_metadata + + if head.startswith(b"\xff\xd8"): + return "jpeg", _jpeg_regions(head) + if head.startswith(PNG_SIGNATURE): + # Chunks placed after the pixel stream (an XMP packet at 2.7 MB, say) are + # past the window; the same seek-past-IDAT reader the file path uses gets them. + return "png", _png_regions(head) + png_late_metadata(image_path, HEAD_WINDOW) + if head.startswith(b"RIFF") and head[8:12] == b"WEBP": + return "webp", _riff_regions(head) + riff_late_metadata(image_path, HEAD_WINDOW) + if is_isobmff(head): + return "isobmff", _isobmff_regions(image_path, head) + return "unknown", head + + +def _raw_head(image_path: Path) -> bytes: + """The file's first bytes, unmodified -- the input every structural walk needs.""" + try: + with open(image_path, "rb") as handle: + return handle.read(HEAD_WINDOW) + except OSError as exc: + logger.debug("head read failed for %s: %s", image_path, exc) + return b"" + + +def _trailer(image_path: Path, container: str) -> bytes: + """The bytes that follow the container's end marker, and nothing else. + + A fixed-size tail read would be almost entirely pixels: the trailer of a 20 MB + photo is a few kilobytes at most. So the end marker is located in the tail window + and only what follows it is kept. When no marker is found (an unknown container, + or one whose end lies before the window) the window is kept as-is, bounded -- + that is what a byte scan of the same file would have seen anyway. + """ + if container == "webp": + # RIFF declares its structural end in bytes 4..8. A fixed tail window is + # normally the last animation/frame payload, not a trailer, so preserve + # only bytes appended after the declared RIFF container. + try: + with open(image_path, "rb") as handle: + header = handle.read(12) + if len(header) < 12 or not header.startswith(b"RIFF"): + return b"" + declared_end = 8 + struct.unpack("= file_size: + return b"" + handle.seek(declared_end) + return handle.read(min(file_size - declared_end, UNKNOWN_TRAILER_WINDOW)) + except OSError as exc: + logger.debug("RIFF trailer read failed for %s: %s", image_path, exc) + return b"" + if container == "isobmff": + # ISOBMFF has no out-of-container trailer convention. Its bounded box + # walkers already collect late provenance while skipping ``mdat``; keeping + # a blind tail here would carry coded media bytes. + return b"" + + tail = read_file_tail(image_path, TAIL_WINDOW) + if SAMSUNG_EDITOR_MARKER in tail: + # Galaxy AI splits its evidence: the marker sits in the post-EOI trailer, but + # the `genAIType` value it is gated on can sit INSIDE the entropy-coded scan. + # Keeping only the trailer therefore carries the marker without the value and the + # verdict silently drops the Samsung signal, so a marked file keeps the whole + # window. Only Samsung-marked files pay for it. + return tail + marker = {"jpeg": b"\xff\xd9", "png": b"IEND\xae\x42\x60\x82"}.get(container) + if marker is None: + return tail[-UNKNOWN_TRAILER_WINDOW:] + index = tail.rfind(marker) + return tail[index + len(marker) :] if index >= 0 else tail[-UNKNOWN_TRAILER_WINDOW:] + + +def _decoder_info(image_path: Path) -> dict[str, Any]: + """PIL's ``info`` mapping, read once. + + One open for both consumers below. They want different parts of the same mapping + (the text keys, and the raw EXIF blob), and opening twice repeats the container + header parse and, for a PNG carrying ``zTXt``, the zlib inflate with it. + """ + try: + from PIL import Image + + with Image.open(image_path) as img: + # PIL types this mapping with a non-string key union (a DPI tuple key + # exists), so the keys are normalized here rather than assumed. + return {str(key): value for key, value in img.info.items()} + except Exception as exc: # a container PIL cannot open + logger.debug("PIL info unavailable for %s: %s", image_path, exc) + return {} + + +def _exif_pairs(info: dict[str, Any]) -> dict[str, str]: + """The 0th-IFD tags the verdict reads, under their tag NAMES. + + Not a convenience: two probes key on names rather than on the raw bytes already + in the regions. ``xai_signature_pair`` wants an (ImageDescription, Artist) pair, + and ``_external_exif_generator`` looks for Software / Make / Artist / + ImageDescription. Ship the bytes alone and both silently return nothing, which + is how a collector can silently lose Grok and NovelAI verdicts. + """ + exif_bytes = info.get("exif") + if not exif_bytes: + return {} + try: + import piexif + + tags = piexif.load(exif_bytes).get("0th", {}) + except Exception as exc: # malformed EXIF + logger.debug("EXIF parse failed: %s", exc) + return {} + + return { + name: text + for name, tag in ( + ("Software", piexif.ImageIFD.Software), + ("Make", piexif.ImageIFD.Make), + ("Artist", piexif.ImageIFD.Artist), + ("ImageDescription", piexif.ImageIFD.ImageDescription), + ) + if (text := exif_text(tags, tag)) + } + + +def _pil_info(info: dict[str, Any]) -> dict[str, str]: + """PIL's ``info`` mapping as strings, the source of PNG text keys and ``hf-job-id``.""" + + def text_of(value: Any) -> str: + return value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value) + + # Emitted in the file path's own candidate order. ``generator_from_metadata`` + # returns the FIRST candidate carrying a known token. A record using PIL's natural + # dict order can therefore choose a different platform string than the file path, + # even though the two paths are supposed to be indistinguishable. + out: dict[str, str] = {} + for key in _GENERATOR_TEXT_KEYS: + value = info.get(key) + if value is not None and not isinstance(value, (dict, list, tuple)): + out[f"info:{key}"] = text_of(value) + for key, value in info.items(): + if key in _GENERATOR_TEXT_KEYS or isinstance(value, (dict, list, tuple)): + continue + out[f"info:{key}"] = text_of(value) + return out + + +def collect_metadata_record( + image_path: Path, + *, + schema_version: int = METADATA_RECORD_SCHEMA_VERSION, +) -> dict[str, Any]: + """Collect everything the provenance verdict reads, as a JSON-safe record. + + The record is the transport format for + :func:`identify.evidence_from_metadata_record`: it carries the metadata regions + (base64), the C2PA manifest store, and PIL's info mapping without carrying the + primary coded-pixel stream. Schema and collection status are explicit so a + consumer cannot mistake a failed read for an unknown provenance verdict. + + Args: + image_path: Path to the image. + schema_version: Output schema implemented by the consumer. + + Returns: + A versioned JSON-serializable dict. ``metadata_base64`` holds the + concatenated container regions, ``tail_base64`` the file trailer. + """ + schema_version = require_schema_version( + schema_version, + contract="provenance metadata", + supported=(1,), + ) + + from remove_ai_watermarks._internal.c2pa import read_manifest_store_json + + try: + image_path.stat() + status = "complete" + issues: list[dict[str, str]] = [] + except OSError as exc: + logger.debug("metadata source unavailable for %s: %s", image_path, exc) + status = "error" + issues = [{"stage": "source", "code": "unavailable"}] + + container, regions = _container_regions(image_path, _raw_head(image_path)) + + info = _decoder_info(image_path) + record: dict[str, Any] = { + "schema_version": schema_version, + "record_type": METADATA_RECORD_TYPE, + "status": status, + "issues": issues, + "container": container, + "name": image_path.name, + "metadata_base64": base64.b64encode(regions).decode("ascii"), + # Always collected: Samsung's Galaxy AI marker is a post-EOI trailer, and a + # record without it loses that verdict outright. + "tail_base64": base64.b64encode(_trailer(image_path, container)).decode("ascii"), + # PIL info BEFORE exif: the file path prefers a PNG text tag over an EXIF + # one, and the normalizer walks the record in insertion order. + "pil": _pil_info(info), + "exif": _exif_pairs(info), + } + store = read_manifest_store_json(image_path) + if store is not None: + record["c2pa_store"] = store + return record diff --git a/src/remove_ai_watermarks/pixel_evidence.py b/src/remove_ai_watermarks/pixel_evidence.py new file mode 100644 index 0000000..a78cccc --- /dev/null +++ b/src/remove_ai_watermarks/pixel_evidence.py @@ -0,0 +1,484 @@ +# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportMissingTypeStubs=false +"""The complete pixel-forensics layer for one image. + +STATUS + +Independent from provenance verdicts, removal, and the CLI. Consumers use the +versioned :meth:`PixelEvidence.to_dict` boundary; feature extraction failures are +reported per family without discarding successful measurements. + +WHAT IS MEASURED + +One decode, then six families of scale-robust statistics over it: + +* ``dct`` -- AC coefficient histograms over the 8x8 block DCT, plus the deviation of + leading digits from Benford's law. +* ``fft`` -- radial band energies of the log-magnitude spectrum, plus the + color-filter-array periodicity peaks a demosaiced camera capture leaves. +* ``noise`` -- standard deviation and kurtosis of a high-pass residual. +* ``ela`` -- error level after a quality-90 JPEG re-save. +* ``gradient`` -- gradient-magnitude histogram and Laplacian variance. +* ``color`` -- 4x4x4 RGB histogram, mean saturation, mean value. + +and, in ``artifacts``, the spatial layer those statistics are computed from: a +64-bit perceptual hash, a 128px JPEG thumbnail, and coarse ELA, noise-residual and +FFT-phase maps. + +THE ARTIFACTS ARE NOT AGGREGATES + +Everything above ``artifacts`` is a scalar or a fixed-length histogram, and an image +cannot be reconstructed from those. ``artifacts`` is different in kind: a thumbnail +is a picture, a perceptual hash identifies one, and the coarse maps carry layout. +Collecting them makes a record that identifies the source image, so a caller storing +or forwarding them is handling image content, not statistics about it. That is why +they are a separate field and not merged into the families. + +REQUIREMENTS + +Needs the ``pixels`` extra (numpy). Guard a call with :func:`is_available` when the +caller must not hard-depend on it. +""" + +from __future__ import annotations + +import base64 +import io +import logging +import time +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from remove_ai_watermarks._internal.schema import require_schema_version + +if TYPE_CHECKING: + from pathlib import Path + +logger = logging.getLogger(__name__) + +# Analysis resolution. Every statistic here is scale-robust, and a 2048px cap keeps +# the FFT and the sliding-window residual bounded on a 100 MP input. +MAX_SIDE = 2048 +# The eight lowest-frequency AC positions of the 8x8 block DCT, zig-zag order. +AC_POSITIONS = ((0, 1), (1, 0), (1, 1), (0, 2), (2, 0), (2, 1), (1, 2), (0, 3)) +FFT_BANDS = 8 +# A Bayer CFA shows as symmetric peaks at half the Nyquist on the diagonals. +BAYER_OFFSETS = ((1, 1), (1, -1)) +INSTALL_HINT = "install the pixel extra: uv add 'remove-ai-watermarks[pixels]'" +PIXEL_EVIDENCE_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class PixelEvidence: + """Pixel statistics for one image, and the spatial artifacts behind them. + + ``decode`` carries the source dimensions, or ``{"error": ...}`` when the image + could not be decoded -- in which case every other field is empty. A family is also + empty when the image is too small for it (the block DCT needs 8x8, the FFT 32x32, + the residual 3x3), so a caller must treat every field as optional rather than + assume a fixed feature width. + """ + + path: Path + decode: dict[str, Any] + dct: dict[str, Any] = field(default_factory=dict[str, Any]) + fft: dict[str, Any] = field(default_factory=dict[str, Any]) + noise: dict[str, Any] = field(default_factory=dict[str, Any]) + ela: dict[str, Any] = field(default_factory=dict[str, Any]) + gradient: dict[str, Any] = field(default_factory=dict[str, Any]) + color: dict[str, Any] = field(default_factory=dict[str, Any]) + # Identifies the source image; see the module note. Empty unless asked for. + artifacts: dict[str, Any] = field(default_factory=dict[str, Any]) + # Opt-in timings for callers measuring pipeline latency. Empty by default so + # repeated evidence collection remains value-deterministic. + timing_ms: dict[str, float] = field(default_factory=dict[str, float]) + + @property + def decoded(self) -> bool: + """False when the source could not be decoded at all.""" + return "error" not in self.decode + + @property + def status(self) -> str: + """``complete``, ``partial`` for a failed family, or ``error`` on decode.""" + if not self.decoded: + return "error" + sections = (self.dct, self.fft, self.noise, self.ela, self.gradient, self.color, self.artifacts) + return "partial" if any("error" in section for section in sections) else "complete" + + def to_dict( + self, + *, + schema_version: int = PIXEL_EVIDENCE_SCHEMA_VERSION, + ) -> dict[str, Any]: + """Return the selected JSON-safe transport schema without a local path.""" + schema_version = require_schema_version( + schema_version, + contract="pixel evidence", + supported=(1,), + ) + return { + "schema_version": schema_version, + "status": self.status, + "decode": dict(self.decode), + "dct": dict(self.dct), + "fft": dict(self.fft), + "noise": dict(self.noise), + "ela": dict(self.ela), + "gradient": dict(self.gradient), + "color": dict(self.color), + "artifacts": dict(self.artifacts), + "timing_ms": dict(self.timing_ms), + } + + +def is_available() -> bool: + """True when the optional pixel dependencies are installed.""" + from remove_ai_watermarks.optional_deps import module_available + + return module_available("numpy") + + +def _numpy() -> Any: + from remove_ai_watermarks.optional_deps import module_available + + if not module_available("numpy"): + raise RuntimeError(f"Pixel evidence needs numpy -- {INSTALL_HINT}") + import numpy as np + + return np + + +def _dct_matrix(np: Any, n: int = 8) -> Any: + """Orthonormal n x n DCT-II basis: M[i, j] = cos(pi (2j + 1) i / 2n).""" + i = np.arange(n)[:, None] + j = np.arange(n)[None, :] + m = np.cos(np.pi * (2 * j + 1) * i / (2 * n)) + m[0, :] *= 1 / np.sqrt(2) + return m * np.sqrt(2 / n) + + +def read_gray(image_path: Path) -> tuple[Any, Any, dict[str, Any]]: + """Decode to float32 grayscale (and RGB for color stats), downscaled. + + Pillow, not cv2, and the source dimensions are recorded BEFORE the downscale. + """ + np = _numpy() + from PIL import Image + + from remove_ai_watermarks import image_io + + try: + image_io._register_heif() # pyright: ignore[reportPrivateUsage] + with Image.open(image_path) as img: + info: dict[str, Any] = {"width": img.width, "height": img.height} + if max(img.size) > MAX_SIDE: + img.thumbnail((MAX_SIDE, MAX_SIDE), Image.Resampling.LANCZOS) + rgb = np.asarray(img.convert("RGB"), dtype=np.float32) + gray = np.asarray(img.convert("L"), dtype=np.float32) + except Exception as exc: + logger.debug("pixel decode failed for %s: %s", image_path, exc) + # Exception text from Pillow commonly embeds the absolute source path. + # Keep that detail in the log, not in the pathless transport contract. + return None, None, {"error": type(exc).__name__} + return gray, rgb, info + + +def dct_features(gray: Any) -> dict[str, Any]: + """AC coefficient histograms over the 8x8 block DCT + Benford deviation.""" + np = _numpy() + height, width = gray.shape + h8, w8 = height // 8 * 8, width // 8 * 8 + if h8 < 8 or w8 < 8: + return {} + basis = _dct_matrix(np) + bins = np.linspace(-20.5, 20.5, 22) + blocks = gray[:h8, :w8].reshape(h8 // 8, 8, w8 // 8, 8).swapaxes(1, 2) + rows = basis[[row for row, _ in AC_POSITIONS]] + columns = basis[[column for _, column in AC_POSITIONS]] + coeff = np.einsum("ki,abij,kj->abk", rows, blocks, columns) + hists = [] + lead_vals: list[Any] = [] + for index in range(len(AC_POSITIONS)): + values = coeff[:, :, index].ravel() + hists.append(np.histogram(values, bins=bins)[0].tolist()) + lead_vals.append(np.abs(values)) + out: dict[str, Any] = {"dct_ac_hist": hists} + flat = np.abs(np.concatenate(lead_vals)) + flat = flat[flat >= 1] + if flat.size > 100: + leading = (flat / 10 ** np.floor(np.log10(flat))).astype(int) + leading = leading[(leading >= 1) & (leading <= 9)] + if leading.size > 100: + observed = np.bincount(leading, minlength=10)[1:10] / leading.size + benford = np.log10(1 + 1 / np.arange(1, 10)) + out["benford_mad"] = float(np.abs(observed - benford).mean()) + return out + + +def noise_residual_map(gray: Any) -> Any: + """High-pass residual, the map the noise statistics are computed from.""" + np = _numpy() + from numpy.lib.stride_tricks import sliding_window_view + + if gray.shape[0] < 3 or gray.shape[1] < 3: + return None + kernel = np.array([[-1.0, -1.0, -1.0], [-1.0, 8.0, -1.0], [-1.0, -1.0, -1.0]]) + height, width = gray.shape + # kernel is float64, so the residual is float64 like the unchunked form + out = np.empty((height - 2, width - 2), dtype=np.float64) + # Row-chunked: the (window * kernel) temporary is ~150 MB at 2048px if + # materialized whole. Per-element 9-tap sums are computed in the same order, + # so the result is bit-identical to the unchunked form. + for y0 in range(0, height - 2, 256): + y1 = min(y0 + 256, height - 2) + window = sliding_window_view(gray[y0 : y1 + 2], (3, 3)) + out[y0:y1] = (window * kernel).sum(axis=(-1, -2)) + return out + + +def noise_features(residual: Any) -> dict[str, Any]: + """High-pass residual std and kurtosis.""" + flat = residual.ravel() + std = float(flat.std()) + if std < 1e-9: + return {"noise_std": 0.0, "noise_kurtosis": 0.0} + z = (flat - flat.mean()) / std + return {"noise_std": std, "noise_kurtosis": float((z**4).mean() - 3.0)} + + +def fft_decompose(gray: Any) -> tuple[Any, Any] | None: + """Log-magnitude (fftshifted) and phase of the image spectrum.""" + np = _numpy() + if min(gray.shape) < 32: + return None + spectrum = np.fft.fftshift(np.fft.fft2(gray - gray.mean())) + return np.log1p(np.abs(spectrum)), np.angle(spectrum) + + +def fft_features(mag: Any) -> dict[str, Any]: + """Radial magnitude band energies (no phase) + CFA periodicity peaks.""" + np = _numpy() + height, width = mag.shape + cy, cx = height // 2, width // 2 + # 1D broadcast instead of an mgrid: saves ~160 MB of int64 temporaries at + # 2048px. The squares are exact in float64 (values < 2^53), so band means + # are identical to the mgrid form. + r2y = (np.arange(height, dtype=np.float64) - cy) ** 2 + r2x = (np.arange(width, dtype=np.float64) - cx) ** 2 + radius = np.sqrt(r2y[:, None] + r2x[None, :]) + r_max = radius.max() + bands = [] + for index in range(FFT_BANDS): + mask = (radius >= r_max * index / FFT_BANDS) & (radius < r_max * (index + 1) / FFT_BANDS) + bands.append(float(mag[mask].mean()) if mask.any() else 0.0) + peaks = [] + for dy, dx in BAYER_OFFSETS: + y, x = cy + dy * (height // 4), cx + dx * (width // 4) + neighborhood = mag[y - 2 : y + 3, x - 2 : x + 3] + peaks.append(float(neighborhood.max() - mag.mean())) + return {"fft_band_energy": bands, "cfa_peaks": peaks, "cfa_peak": max(peaks)} + + +def ela_map(rgb: Any) -> Any: + """Absolute per-pixel error after a quality-90 JPEG re-save.""" + np = _numpy() + from PIL import Image + + try: + buffer = io.BytesIO() + Image.fromarray(rgb.astype(np.uint8)).save(buffer, "JPEG", quality=90) + buffer.seek(0) + resaved = np.asarray(Image.open(buffer).convert("RGB"), dtype=np.float32) + except Exception as exc: + logger.debug("ELA re-save failed: %s", exc) + return None + if resaved.shape != rgb.shape: + return None + return np.abs(rgb - resaved).mean(axis=-1) + + +def ela_features(err: Any) -> dict[str, Any]: + """Error-level stats after a quality-90 JPEG re-save.""" + np = _numpy() + return {"ela_mean": float(err.mean()), "ela_p95": float(np.percentile(err, 95))} + + +def gradient_features(gray: Any) -> dict[str, Any]: + np = _numpy() + gy, gx = np.gradient(gray) + mag = np.sqrt(gx**2 + gy**2) + hist = np.histogram(mag, bins=10, range=(0, 255))[0].tolist() + laplacian = np.gradient(gy, axis=0) + np.gradient(gx, axis=1) + return {"gradient_hist": hist, "laplacian_var": float(laplacian.var())} + + +def color_features(rgb: Any) -> dict[str, Any]: + np = _numpy() + small = rgb[::4, ::4] # decimate; the histogram is position-blind anyway + bins = (small / 256 * 4).astype(int).clip(0, 3) + index = bins[..., 0] * 16 + bins[..., 1] * 4 + bins[..., 2] + hist = np.bincount(index.ravel(), minlength=64).tolist() + mx = small.max(axis=-1) + mn = small.min(axis=-1) + saturation = np.where(mx > 0, (mx - mn) / np.maximum(mx, 1e-6), 0) + return { + "color_hist_4x4x4": hist, + "saturation_mean": float(saturation.mean()), + "value_mean": float(mx.mean() / 255), + } + + +def perceptual_hash(gray: Any) -> str: + """64-bit DCT perceptual hash. Identifies an image; see the module note.""" + np = _numpy() + from PIL import Image + + small = np.asarray(Image.fromarray(gray.astype(np.float32), mode="F").resize((32, 32), Image.Resampling.LANCZOS)) + basis = _dct_matrix(np, 32) + low_basis = basis[:8] + low = (low_basis @ small @ low_basis.T).ravel()[1:] # drop DC + bits = low > np.median(low) + return f"{int(''.join('1' if bit else '0' for bit in bits), 2):016x}" + + +def _coarse(np: Any, arr: Any, side: int = 64) -> Any: + """Downscale a 2D map to at most ``side`` on the long edge.""" + from PIL import Image + + height, width = arr.shape + if max(height, width) <= side: + return arr + img = Image.fromarray(arr.astype(np.float32), mode="F") + img.thumbnail((side, side), Image.Resampling.BILINEAR) + return np.asarray(img) + + +def _array_payload(arr: Any) -> dict[str, Any]: + return { + "shape": list(arr.shape), + "dtype": str(arr.dtype), + "base64": base64.b64encode(arr.tobytes()).decode("ascii"), + } + + +def spatial_artifacts(gray: Any, rgb: Any, *, ela: Any, residual: Any, phase: Any) -> dict[str, Any]: + """Perceptual hash, thumbnail, and coarse ELA / residual / phase maps. + + These identify the source image rather than describe it -- see the module note. + The maps are the ones the statistics were computed from, passed in rather than + recomputed. + """ + np = _numpy() + from PIL import Image + + out: dict[str, Any] = {"phash": perceptual_hash(gray)} + thumbnail = Image.fromarray(rgb.astype(np.uint8)) + thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS) + buffer = io.BytesIO() + thumbnail.save(buffer, "JPEG", quality=70) + out["thumbnail_jpeg_b64"] = base64.b64encode(buffer.getvalue()).decode("ascii") + + if ela is not None: + out["ela_map"] = _array_payload(_coarse(np, ela)) + if residual is not None: + clipped = np.clip(residual / 4.0, -1, 1) + out["noise_residual"] = _array_payload(_coarse(np, (clipped * 127).astype(np.int8))) + if phase is not None: + out["fft_phase"] = _array_payload(_coarse(np, phase.astype(np.float32), 32)) + return out + + +def extract_pixel_evidence(image_path: Path, *, artifacts: bool = False, timings: bool = False) -> PixelEvidence: + """Measure every pixel-statistic family for one image in a single decode. + + The image is decoded ONCE and the intermediate maps (high-pass residual, ELA + error, FFT magnitude and phase) are computed once and shared, because the + residual's sliding window and the ELA re-save are the two expensive steps and + each family would otherwise redo them. + + A family that fails or does not apply is left empty rather than raising: an + undecodable file, or one too small for the block DCT, still returns a + :class:`PixelEvidence` whose ``decoded`` / empty fields say so. Missing numpy is + the one hard error, since then nothing can be measured at all. + + Args: + image_path: Path to the image. Any container Pillow can open. + artifacts: Also return the spatial layer -- perceptual hash, thumbnail and + coarse maps. Off by default: those identify the source image, so asking + for them is a decision the caller makes explicitly. + timings: Measure each stage and include rounded milliseconds in + :attr:`PixelEvidence.timing_ms`. + + Returns: + A :class:`PixelEvidence`. + """ + started = time.perf_counter() + stage_started = started + measured: dict[str, float] = {} + + gray, rgb, info = read_gray(image_path) + measured["decode"] = time.perf_counter() - stage_started + if gray is None or rgb is None: + measured["total"] = time.perf_counter() - started + timing_ms = {name: round(seconds * 1000, 1) for name, seconds in measured.items()} if timings else {} + return PixelEvidence(path=image_path, decode=info, timing_ms=timing_ms) + + families: dict[str, dict[str, Any]] = {} + + residual = None + stage_started = time.perf_counter() + try: + residual = noise_residual_map(gray) + families["noise"] = noise_features(residual) if residual is not None else {} + except Exception as exc: + logger.debug("pixel family noise failed for %s: %s", image_path, exc) + families["noise"] = {"error": type(exc).__name__} + measured["noise"] = time.perf_counter() - stage_started + + spectrum = None + stage_started = time.perf_counter() + try: + spectrum = fft_decompose(gray) + families["fft"] = fft_features(spectrum[0]) if spectrum is not None else {} + except Exception as exc: + logger.debug("pixel family fft failed for %s: %s", image_path, exc) + families["fft"] = {"error": type(exc).__name__} + measured["fft"] = time.perf_counter() - stage_started + + error = None + stage_started = time.perf_counter() + try: + error = ela_map(rgb) + families["ela"] = ela_features(error) if error is not None else {} + except Exception as exc: + logger.debug("pixel family ela failed for %s: %s", image_path, exc) + families["ela"] = {"error": type(exc).__name__} + measured["ela"] = time.perf_counter() - stage_started + + for name, compute in ( + ("dct", lambda: dct_features(gray)), + ("gradient", lambda: gradient_features(gray)), + ("color", lambda: color_features(rgb)), + ): + stage_started = time.perf_counter() + try: + families[name] = compute() + except Exception as exc: # one bad family must not lose the other five + logger.debug("pixel family %s failed for %s: %s", name, image_path, exc) + families[name] = {"error": type(exc).__name__} + measured[name] = time.perf_counter() - stage_started + + if artifacts: + stage_started = time.perf_counter() + try: + families["artifacts"] = spatial_artifacts( + gray, rgb, ela=error, residual=residual, phase=spectrum[1] if spectrum is not None else None + ) + except Exception as exc: + logger.debug("pixel artifacts failed for %s: %s", image_path, exc) + families["artifacts"] = {"error": type(exc).__name__} + measured["full_artifacts"] = time.perf_counter() - stage_started + + measured["total"] = time.perf_counter() - started + timing_ms = {name: round(seconds * 1000, 1) for name, seconds in measured.items()} if timings else {} + return PixelEvidence(path=image_path, decode=info, timing_ms=timing_ms, **families) diff --git a/tests/test_ai_score.py b/tests/test_ai_score.py deleted file mode 100644 index 96336e5..0000000 --- a/tests/test_ai_score.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Tests for the standalone structural AI-generation scorer.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import Any - -import numpy as np - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) - -import ai_score - - -def _complete_record() -> dict[str, Any]: - return { - "noise": {"noise_std": 1.0, "noise_kurtosis": 2.0}, - "fft": { - "cfa_peak": 3.0, - "cfa_peaks": [2.5, 3.0], - "fft_band_energy": list(range(8)), - }, - "ela": {"ela_mean": 4.0, "ela_p95": 5.0}, - "gradient": {"laplacian_var": 6.0, "gradient_hist": list(range(10))}, - "color": { - "saturation_mean": 0.5, - "value_mean": 0.75, - "color_hist_4x4x4": list(range(64)), - }, - "dct": { - "benford_mad": 0.1, - "dct_ac_hist": [[1] * 21 for _ in range(8)], - }, - "jpeg_forensics": { - "subsampling": "4:4:4", - "progressive": True, - "quant_tables": { - "0": list(range(1, 65)), - "1": list(range(65, 129)), - }, - "huffman_tables_hex": ["00ff", "abcd12"], - "scan_count": 10, - "restart_interval": 4, - "precision_bits": 8, - "adobe_transform": 1, - "jfif": {"version": "1.1"}, - }, - "pil": {"width": 2000, "height": 1000}, - "content_format": "jpeg", - } - - -def test_v1_feature_schema_is_fixed_for_sparse_records() -> None: - assert len(ai_score.feature_names("v1")) == 97 - assert len(ai_score.features_of({}, schema="v1")) == 97 - - -def test_v2_feature_schema_includes_existing_forensic_data() -> None: - record = _complete_record() - - names = ai_score.feature_names("v2") - values = ai_score.features_of(record, schema="v2") - by_name = dict(zip(names, values, strict=True)) - - assert len(names) == len(values) == 406 - assert by_name["cfa_peak_0"] == 2.5 - assert by_name["cfa_peak_1"] == 3.0 - assert by_name["dct_ac_0_0"] == 1 / 21 - assert by_name["dct_ac_7_20"] == 1 / 21 - assert by_name["jpeg_quant_0_0"] == 1.0 - assert by_name["jpeg_quant_1_63"] == 128.0 - assert by_name["jpeg_quant_table_count"] == 2.0 - assert by_name["jpeg_huffman_table_count"] == 2.0 - assert by_name["jpeg_huffman_total_bytes"] == 5.0 - assert by_name["jpeg_scan_count"] == 10.0 - assert by_name["jpeg_jfif_present"] == 1.0 - assert by_name["format_webp"] == 0.0 - assert by_name["format_isobmff"] == 0.0 - assert by_name["format_other"] == 0.0 - - -def test_v2_feature_schema_is_fixed_when_forensics_are_missing() -> None: - names = ai_score.feature_names("v2") - values = ai_score.features_of({}, schema="v2") - - assert len(names) == len(values) == 406 - assert np.isnan(values[names.index("dct_ac_0_0")]) - assert np.isnan(values[names.index("jpeg_quant_0_0")]) - assert np.isnan(values[names.index("jpeg_scan_count")]) - - -def test_grouped_stratified_split_keeps_hashes_on_one_side() -> None: - labels = np.asarray([1, 1, 1, 0, 0, 0, 1, 0]) - hashes = np.asarray(["a", "a", "b", "c", "c", "d", "e", "f"]) - - train, test = ai_score.grouped_stratified_split(labels, hashes, test_size=0.5, random_state=7) - - assert set(hashes[train]).isdisjoint(set(hashes[test])) - assert set(labels[train]) == {0, 1} - assert set(labels[test]) == {0, 1} - assert sorted(np.concatenate([train, test]).tolist()) == list(range(len(labels))) - - -def test_grouped_stratified_split_rejects_conflicting_labels() -> None: - labels = np.asarray([0, 1, 0, 1]) - hashes = np.asarray(["same", "same", "negative", "positive"]) - - with np.testing.assert_raises_regex(ValueError, "conflicting labels"): - ai_score.grouped_stratified_split(labels, hashes) - - -def test_temporal_holdout_excludes_hashes_seen_during_training() -> None: - dates = np.asarray(["2026-01-01", "2026-01-01", "2026-01-02", "2026-01-03", "2026-01-04", "2026-01-04"]) - hashes = np.asarray(["repeated", "old", "middle", "new-a", "repeated", "new-b"]) - - train, test, cutoff = ai_score.temporal_holdout_split(dates, hashes, train_fraction=0.5) - - assert cutoff == "2026-01-03" - assert set(hashes[train]).isdisjoint(set(hashes[test])) - assert set(hashes[test]) == {"new-a", "new-b"} - - -def test_legacy_model_bundle_defaults_to_v1_schema() -> None: - assert ai_score.model_schema({}) == "v1" - assert ai_score.model_schema({"feature_schema": "v2"}) == "v2" diff --git a/tests/test_forensic_metadata.py b/tests/test_forensic_metadata.py new file mode 100644 index 0000000..cf6335e --- /dev/null +++ b/tests/test_forensic_metadata.py @@ -0,0 +1,236 @@ +"""Tests for the metadata-only forensic collector.""" + +from __future__ import annotations + +import base64 +import json +import zlib +from typing import TYPE_CHECKING + +import piexif +import pytest +from PIL import Image +from PIL.PngImagePlugin import PngInfo + +from remove_ai_watermarks.forensic_metadata import ( + FORENSIC_METADATA_RECORD_TYPE, + FORENSIC_METADATA_SCHEMA_VERSION, + SUPPORTED_EXTENSIONS, + _b64, + _decode_exif_value, + _jpeg_forensics_bytes, + _png_text_decode, + _safe_str, + apple_live_photo_id, + collect_forensic_metadata, + read_full_exif, + read_isobmff_inventory, + read_isobmff_provenance_path, + read_jpeg_segments, + read_pil_info, + read_png_chunks, + read_png_late_metadata_path, + read_webp_chunks, + sha256_of, + sniff_format, + xattr_quarantine, + xattr_where_from, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _jpeg(path: Path) -> Path: + Image.new("RGB", (48, 32), (20, 80, 160)).save(path, "JPEG", quality=87) + return path + + +def _png_chunk(chunk_type: bytes, payload: bytes) -> bytes: + crc = zlib.crc32(chunk_type + payload).to_bytes(4, "big") + return len(payload).to_bytes(4, "big") + chunk_type + payload + crc + + +def test_supported_extensions_are_media_not_documents(): + assert {".jpg", ".png", ".webp", ".heic", ".mp4"}.issubset(SUPPORTED_EXTENSIONS) + assert ".pdf" not in SUPPORTED_EXTENSIONS + + +def test_json_helpers_and_format_sniffer(): + class BadString: + def __str__(self): + raise RuntimeError("no string") + + assert _safe_str("ok") == "ok" + assert "BadString" in _safe_str(BadString()) + assert _b64(b"abc") == base64.b64encode(b"abc").decode("ascii") + assert _b64(b"x" * 20, cap=4) == "eHh4eA==...TRUNCATED(20 bytes total)" + assert _decode_exif_value(b"ascii") == "ascii" + assert _decode_exif_value(b"\xff").startswith("hex:") + assert _decode_exif_value((1, b"two")) == [1, "two"] + assert sniff_format(b"\x89PNG\r\n\x1a\n") == "png" + assert sniff_format(b"\xff\xd8\xff\xe0") == "jpeg" + assert sniff_format(b"RIFF....WEBP") == "webp" + assert sniff_format(b"....ftypheic").startswith("isobmff:") + assert sniff_format(b"unknown").startswith("unknown:") + + +def test_png_text_and_container_metadata_are_preserved(tmp_path: Path): + info = PngInfo() + info.add_text("parameters", "Steps: 20, Model: SDXL", zip=True) + path = tmp_path / "workflow.png" + Image.new("RGB", (32, 32)).save(path, pnginfo=info) + trailer = b'{"Label":"1"}' + path.write_bytes(path.read_bytes() + trailer) + + record = collect_forensic_metadata(path) + + assert record["schema_version"] == FORENSIC_METADATA_SCHEMA_VERSION == 1 + assert record["record_type"] == FORENSIC_METADATA_RECORD_TYPE == "forensic_metadata" + assert record["content_format"] == "png" + assert any(chunk.get("type") == "zTXt" for chunk in record["png_chunks"]) + assert record["png_post_iend_bytes"] == len(trailer) + assert base64.b64decode(record["png_post_iend_base64"]) == trailer + assert "Steps: 20" in json.dumps(record) + assert json.loads(json.dumps(record, allow_nan=False)) == record + + +def test_png_text_decoders_and_direct_chunk_reader(tmp_path: Path): + assert "hello" in _png_text_decode("tEXt", b"key\x00hello") + compressed = b"prompt\x00\x00" + zlib.compress(b"workflow") + assert "workflow" in _png_text_decode("zTXt", compressed) + assert "value" in _png_text_decode("iTXt", b"key\x00\x00\x00\x00\x00value") + + path = tmp_path / "plain.png" + Image.new("RGB", (8, 8)).save(path) + chunks, trailer = read_png_chunks(path.read_bytes()) + + assert chunks[0]["type"] == "IHDR" + assert trailer == b"" + + +def test_jpeg_exif_segments_encoder_and_trailer(tmp_path: Path): + path = tmp_path / "camera.jpg" + exif = piexif.dump( + { + "0th": { + piexif.ImageIFD.Make: b"Camera Corp", + piexif.ImageIFD.Software: b"Camera Firmware", + }, + "Exif": {}, + "GPS": {}, + "1st": {}, + } + ) + Image.new("RGB", (64, 48)).save(path, "JPEG", exif=exif, quality=82) + trailer = b'PhotoEditor_Re_Edit_Data{"genAIType":1}' + path.write_bytes(path.read_bytes() + trailer) + + record = collect_forensic_metadata(path) + + assert record["exif"]["0th"]["Make"] == "Camera Corp" + assert record["jpeg"]["post_eoi_bytes"] == len(trailer) + assert base64.b64decode(record["jpeg"]["post_eoi_base64"]) == trailer + assert record["jpeg_forensics"]["quant_tables"] + assert sha256_of(path.read_bytes()) == record["sha256"] + + +def test_direct_exif_pil_and_jpeg_readers(tmp_path: Path): + path = _jpeg(tmp_path / "plain.jpg") + + exif, thumbnail = read_full_exif(path) + pil, iptc, exif_blob = read_pil_info(path) + segments = read_jpeg_segments(path.read_bytes()) + + assert isinstance(exif, dict) + assert thumbnail is None + assert pil["width"] == 48 + assert pil["height"] == 32 + assert isinstance(iptc, dict) + assert exif_blob is None or isinstance(exif_blob, bytes) + assert isinstance(segments["segments"], list) + assert _jpeg_forensics_bytes(path.read_bytes())["quant_tables"] + assert _jpeg_forensics_bytes(b"not a jpeg") == {} + + +def test_webp_inventory_keeps_metadata_but_not_frame_pixels(tmp_path: Path): + path = tmp_path / "image.webp" + xmp = b"metadata" + Image.new("RGB", (32, 32), (30, 40, 50)).save(path, "WEBP", xmp=xmp) + + chunks = read_webp_chunks(path.read_bytes()) + + xmp_chunk = next(chunk for chunk in chunks if chunk["type"] == "XMP ") + assert xmp_chunk["text"] == xmp.decode() + assert all("base64" not in chunk for chunk in chunks if chunk["type"] in {"VP8 ", "VP8L", "ANMF"}) + + +def test_isobmff_inventory_and_streaming_provenance(tmp_path: Path): + path = tmp_path / "signed.mp4" + ftyp = b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom" + payload = b"jumb c2pa trainedAlgorithmicMedia" + uuid_box = (8 + len(payload)).to_bytes(4, "big") + b"uuid" + payload + path.write_bytes(ftyp + b"\x00\x00\x00\x08mdat" + uuid_box) + + inventory = read_isobmff_inventory(path.read_bytes()) + streamed = read_isobmff_provenance_path(path) + + assert "ftyp" in inventory["boxes"] + assert base64.b64decode(inventory["provenance_boxes"][0]["base64"]) == payload + assert base64.b64decode(streamed["provenance_boxes"][0]["base64"]) == payload + + +def test_oversized_path_keeps_bounded_windows_and_late_png_metadata(tmp_path: Path, monkeypatch): + path = tmp_path / "late.png" + Image.new("RGB", (16, 16)).save(path) + source = path.read_bytes() + iend = source.rfind(b"\x00\x00\x00\x00IEND") + padding = _png_chunk(b"vpAg", b"\x00" * ((1 << 20) + 1)) + metadata = b'AIGC\x00{"Label":"1"}' + path.write_bytes(source[:iend] + padding + _png_chunk(b"tEXt", metadata) + source[iend:]) + monkeypatch.setattr("remove_ai_watermarks.forensic_metadata._MAX_FULL_READ", 1) + + record = collect_forensic_metadata(path) + + assert record["oversized"]["head_scanned_bytes"] == path.stat().st_size + assert base64.b64decode(record["raw_metadata_windows"]["head_base64"]) + assert base64.b64decode(record["png_late_metadata_chunks"][0]["base64"]) == metadata + + +def test_collection_registers_optional_heif_and_missing_file_raises(tmp_path: Path, monkeypatch): + registered = False + + def mark_registered(): + nonlocal registered + registered = True + + monkeypatch.setattr("remove_ai_watermarks.image_io._register_heif", mark_registered) + collect_forensic_metadata(_jpeg(tmp_path / "plain.jpg")) + + assert registered is True + with pytest.raises(FileNotFoundError): + collect_forensic_metadata(tmp_path / "missing.jpg") + + +@pytest.mark.parametrize("schema_version", [2, True, 1.0]) +def test_collection_rejects_unsupported_output_schema_before_reading(tmp_path: Path, schema_version: object): + with pytest.raises(ValueError, match="Unsupported forensic metadata schema"): + collect_forensic_metadata( + tmp_path / "missing.jpg", + schema_version=schema_version, # type: ignore[arg-type] + ) + + +def test_xattrs_and_live_photo_probe_are_safe_on_plain_file(tmp_path: Path): + path = _jpeg(tmp_path / "plain.jpg") + + assert xattr_where_from(path) == [] or isinstance(xattr_where_from(path), list) + assert xattr_quarantine(path) is None or isinstance(xattr_quarantine(path), str) + assert apple_live_photo_id(path.read_bytes()) is None + + +def test_late_png_reader_soft_fails_on_non_png(tmp_path: Path): + path = tmp_path / "plain.bin" + path.write_bytes(b"not png") + + assert read_png_late_metadata_path(path) == [] diff --git a/tests/test_identify.py b/tests/test_identify.py index ee68391..330a9e5 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -112,6 +112,36 @@ class TestProvenanceEvidence: assert evidence.exif_generator == "NovelAI" + @pytest.mark.parametrize( + "record", + [ + {"name": "trainedAlgorithmicMedia.jpg"}, + {"sha256": "jumb-c2pa-OpenAI-trainedAlgorithmicMedia"}, + {"pil": {"trainedAlgorithmicMedia": "plain"}}, + {"signals": {"provenance": {"is_ai_generated": True}}}, + {"pixel": {"error": "trainedAlgorithmicMedia"}}, + ], + ) + def test_external_diagnostics_and_arbitrary_keys_are_not_evidence(self, tmp_path: Path, record: dict): + report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "plain.jpg")) + + assert report.is_ai_generated is None + assert report.signals == [] + + def test_external_metadata_value_is_evidence(self, tmp_path: Path): + record = { + "exif": { + "0th": { + "ImageDescription": "digitalSourceType=trainedAlgorithmicMedia", + } + } + } + + report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "generated.jpg")) + + assert report.is_ai_generated is True + assert report.ai_source_kind == "generated" + @pytest.mark.parametrize( "filename", [ @@ -443,6 +473,18 @@ class TestIdentifyRealSamples: r = identify(p, check_visible=False, check_invisible=False) assert r.is_ai_generated is True assert r.platform == "Apple Photos (Clean Up AI edit)" + assert r.ai_source_kind == "enhanced" + + def test_standalone_iptc_composite_synthetic_is_enhanced(self, tmp_path: Path): + p = tmp_path / "composite.jpg" + p.write_bytes( + b'\xff\xd8\xff\xe1\xff\xd9' + ) + + r = identify(p, check_visible=False, check_invisible=False) + + assert r.is_ai_generated is True + assert r.ai_source_kind == "enhanced" def test_flux_bfl_c2pa_png(self): # flux-1.png: real Black Forest Labs FLUX.2 Playground output (signed C2PA). @@ -1353,3 +1395,106 @@ class TestSharedPixelDecode: report = identify(self.SAMPLE, check_visible=True, check_invisible=False) assert not any(s.name.startswith("visible_") for s in report.signals) assert report.is_ai_generated is True # the C2PA verdict survives the decode failure + + +class TestSynthIdProxyIsDecidedInTheVerdict: + """The SynthID byte scan belongs to the verdict, not to extraction. + + Extraction has two implementations -- one reading a file, one reading a portable + record -- so a rule that lives in only one of them is a rule the other silently + lacks. This one did: 74 corpus images reported SynthID through ``identify`` and + not through the record.""" + + # A JUMBF-wrapped manifest from a SynthID-pairing signer on AI-generated content: + # the exact shape `synthid_source`'s byte scan is gated on. Spliced into a real + # JPEG as a well-formed APP11 segment, because a malformed one is skipped by the + # record's structural walk and the test would compare two different inputs. + MANIFEST = b"jumb c2pa Google LLC trainedAlgorithmicMedia" + + def _jpeg_with_manifest(self, path: Path) -> Path: + import numpy as np + from PIL import Image + + Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG") + data = path.read_bytes() + segment = b"\xff\xeb" + (len(self.MANIFEST) + 2).to_bytes(2, "big") + self.MANIFEST + path.write_bytes(data[:2] + segment + data[2:]) + return path + + def test_both_paths_infer_it_from_the_same_bytes(self, tmp_path: Path): + from remove_ai_watermarks.identify import identify_metadata_record + from remove_ai_watermarks.metadata_record import collect_metadata_record + + path = self._jpeg_with_manifest(tmp_path / "gemini.jpg") + + via_file = identify(path, check_visible=False, check_invisible=False) + via_record = identify_metadata_record(collect_metadata_record(path), path=path) + + assert any("SynthID" in mark for mark in via_file.watermarks) + assert via_record.watermarks == via_file.watermarks + + def test_it_needs_a_manifest_and_an_ai_source_type(self, tmp_path: Path): + """The vendor name alone is not evidence: an ordinary photo mentioning + "Google LLC" in EXIF must not acquire a SynthID verdict.""" + import numpy as np + from PIL import Image + + path = tmp_path / "photo.jpg" + Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG") + data = path.read_bytes() + note = b"Google LLC Pixel" + path.write_bytes(data[:2] + b"\xff\xeb" + (len(note) + 2).to_bytes(2, "big") + note + data[2:]) + + report = identify(path, check_visible=False, check_invisible=False) + + assert not any("SynthID" in mark for mark in report.watermarks) + + +class TestRegistryScansSkipTheCodedPixels: + """The vendor registries match short raw substrings -- the shortest are four and + five bytes. Over a megabyte of compressed pixel data such a sequence turns up by + chance: `Bria` matched inside the entropy-coded scan of 4 of 14,707 corpus JPEGs, + and that entry asserts AI, so a chance match can declare an image AI-generated. + `c2pa_marker_in` already refuses a bare `c2pa` substring for the same reason.""" + + def _jpeg(self, path: Path, *, in_segment: bytes = b"", in_scan: bytes = b"") -> Path: + import numpy as np + from PIL import Image + + Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG") + data = path.read_bytes() + if in_segment: + payload = b"jumb c2pa trainedAlgorithmicMedia " + in_segment + data = data[:2] + b"\xff\xeb" + (len(payload) + 2).to_bytes(2, "big") + payload + data[2:] + if in_scan: + # After SOS, i.e. inside the entropy-coded scan the walk skips. + sos = data.index(b"\xff\xda") + data = data[: sos + 16] + in_scan + data[sos + 16 :] + path.write_bytes(data) + return path + + def test_a_token_in_a_marker_segment_is_attributed(self, tmp_path: Path): + from remove_ai_watermarks.identify import _issuers_in, _metadata_region + from remove_ai_watermarks.metadata import scan_head + + path = self._jpeg(tmp_path / "signed.jpg", in_segment=b"Bria") + + assert _issuers_in(_metadata_region(scan_head(path))) == ["Bria Artificial Intelligence"] + + def test_a_token_in_the_coded_scan_is_not(self, tmp_path: Path): + from remove_ai_watermarks.identify import _issuers_in, _metadata_region + from remove_ai_watermarks.metadata import scan_head + + path = self._jpeg(tmp_path / "chance.jpg", in_segment=b"OpenAI", in_scan=b"Bria") + region = _metadata_region(scan_head(path)) + + assert _issuers_in(region) == ["OpenAI"] + + def test_a_container_that_does_not_parse_is_left_whole(self, tmp_path: Path): + """Cutting a buffer the walk did not understand would drop real evidence to + avoid a chance match, which is the wrong way round.""" + from remove_ai_watermarks.identify import _metadata_region + + blob = b"\xff\xd8" + b"not really a jpeg, no valid marker chain here" * 4 + + assert _metadata_region(blob) == blob diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 4c863c6..9753b23 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1316,6 +1316,30 @@ class TestAIGCLabel: assert b"ContentProducer" in _png_late_metadata(p, 8) assert b"ContentProducer" in scan_head(p, 8) + def test_scan_head_collects_webp_metadata_past_window(self, tmp_path: Path): + """WebP stores ``XMP `` AFTER the pixels, so on any WebP above the window a + fixed read can stop short of an IPTC "Made with AI" tag.""" + from remove_ai_watermarks.metadata import _riff_late_metadata, scan_head + + p = tmp_path / "late.webp" + xmp = ( + b"trainedAlgorithmicMedia" + ) + Image.new("RGB", (16, 16)).save(p, "WEBP", xmp=xmp) + + assert b"trainedAlgorithmicMedia" in _riff_late_metadata(p, 12) + assert b"trainedAlgorithmicMedia" in scan_head(p, 12) + + def test_riff_late_metadata_ignores_the_coded_image(self, tmp_path: Path): + """The point of stepping chunk by chunk rather than reading through: the + pixel payload never enters the scan buffer.""" + from remove_ai_watermarks.metadata import _riff_late_metadata + + p = tmp_path / "plain.webp" + Image.new("RGB", (64, 64), (200, 30, 30)).save(p, "WEBP") + + assert _riff_late_metadata(p, 12) == b"" + class TestHuggingFaceJob: """HuggingFace-hosted job marker (``hf-job-id`` PNG text chunk).""" diff --git a/tests/test_metadata_internals.py b/tests/test_metadata_internals.py index 0218d0c..89afe0c 100644 --- a/tests/test_metadata_internals.py +++ b/tests/test_metadata_internals.py @@ -3,10 +3,12 @@ consolidated metadata strip (formerly legacy metadata helper).""" from __future__ import annotations +import logging import struct from pathlib import Path import pytest +from PIL import Image from remove_ai_watermarks._internal.c2pa import ( _parse_c2pa_chunk, @@ -835,3 +837,38 @@ class TestTc260ContainerRouting: readers = _tc260_container_readers() assert [r.__module__.rsplit(".", 1)[-1] for r in readers] == ["isobmff", "ebml", "riff", "flv"] + + +class TestC2paReaderFailureIsVisible: + """A reader failure and a file with no manifest both return None, so the caller + cannot tell them apart -- and the consequence is not symmetric. A file with no + manifest is a normal verdict; a reader that could not read a file it was handed + can silently downgrade one, so the log level must make the failure observable.""" + + def _records(self, caplog, path: str) -> list[str]: + from remove_ai_watermarks._internal import c2pa + + with caplog.at_level(logging.DEBUG, logger="remove_ai_watermarks._internal.c2pa"): + assert c2pa._manifest_json_uncached(path) is None + return [f"{r.levelname} {r.getMessage()}" for r in caplog.records] + + def test_an_unreadable_file_warns(self, caplog): + records = self._records(caplog, "/nonexistent/definitely-not-here.png") + + assert any(r.startswith("WARNING") for r in records), records + + def test_an_unsupported_container_stays_quiet(self, caplog, tmp_path: Path): + target = tmp_path / "notes.txt" + target.write_text("plain text, not a container the reader handles") + + records = self._records(caplog, str(target)) + + assert not any(r.startswith("WARNING") for r in records), records + + def test_a_plain_image_without_a_manifest_logs_nothing(self, caplog, tmp_path: Path): + target = tmp_path / "plain.png" + Image.new("RGB", (8, 8)).save(target) + + records = self._records(caplog, str(target)) + + assert records == [] diff --git a/tests/test_metadata_record.py b/tests/test_metadata_record.py new file mode 100644 index 0000000..f4b7315 --- /dev/null +++ b/tests/test_metadata_record.py @@ -0,0 +1,333 @@ +"""Tests for the portable metadata record. + +The contract is one property: a verdict built from the record equals the verdict +built from the file. Everything here exists to pin that, or to pin a placement the +record could plausibly drop. +""" + +from __future__ import annotations + +import json +import struct +import zlib +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from remove_ai_watermarks.identify import ( + PROVENANCE_REPORT_SCHEMA_VERSION, + ProvenanceReport, + evidence_from_metadata_record, + identify, + identify_from_evidence, + identify_metadata_record, +) +from remove_ai_watermarks.metadata_record import ( + HEAD_WINDOW, + METADATA_RECORD_SCHEMA_VERSION, + METADATA_RECORD_TYPE, + collect_metadata_record, +) + +FIXTURES = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance" +COMPARED = ("is_ai_generated", "platform", "confidence", "ai_source_kind", "ai_from_metadata") + + +def _verdict_via_record(path: Path) -> ProvenanceReport: + """The contractor's path: collect, serialize, judge -- without the file.""" + record = json.loads(json.dumps(collect_metadata_record(path))) + return identify_metadata_record(record, path=path) + + +def _assert_same_verdict(path: Path) -> None: + via_record = _verdict_via_record(path) + via_file = identify(path, check_visible=False, check_invisible=False) + + for field in COMPARED: + assert getattr(via_record, field) == getattr(via_file, field), field + assert sorted(s.name for s in via_record.signals) == sorted(s.name for s in via_file.signals) + assert sorted(via_record.watermarks) == sorted(via_file.watermarks) + + +def _noise_png(path: Path, size: tuple[int, int] = (700, 700)) -> Path: + """A PNG whose IDAT is incompressible, so the file exceeds the head window.""" + rng = np.random.default_rng(0) + Image.fromarray(rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8)).save(path) + return path + + +def _insert_png_chunk(path: Path, chunk_type: bytes, payload: bytes) -> None: + """Splice a chunk in just before IEND, i.e. AFTER the whole pixel stream.""" + data = path.read_bytes() + end = data.rindex(b"IEND") - 4 + chunk = struct.pack(">I", len(payload)) + chunk_type + payload + chunk += struct.pack(">I", zlib.crc32(chunk_type + payload) & 0xFFFFFFFF) + path.write_bytes(data[:end] + chunk + data[end:]) + + +class TestRecordReproducesTheFileVerdict: + """The whole point of the record. Every tracked provenance fixture is compared, + which covers C2PA, the China TC260 label, the xAI EXIF pair and the IPTC tag.""" + + @pytest.mark.skipif(not FIXTURES.is_dir(), reason="provenance fixtures not present") + @pytest.mark.parametrize("name", sorted(p.name for p in FIXTURES.iterdir()) if FIXTURES.is_dir() else []) + def test_fixture(self, name: str): + _assert_same_verdict(FIXTURES / name) + + @pytest.mark.skipif(not FIXTURES.is_dir(), reason="provenance fixtures not present") + def test_the_fixtures_actually_exercise_several_signals(self): + """Guards the test above: if the fixture set ever narrows to one signal + family, equality across it stops meaning much.""" + found = { + signal.name + for path in FIXTURES.iterdir() + for signal in identify(path, check_visible=False, check_invisible=False).signals + } + assert len(found) >= 4, found + + +class TestPlacementsTheRecordCouldDrop: + def test_a_trailer_after_eoi_survives(self, tmp_path: Path): + """Samsung Galaxy AI appends its marker past the JPEG EOI, and the value it is + gated on can sit further back still. A record that stopped at the last + structural marker would report no signal at all.""" + path = tmp_path / "edited.jpg" + Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG") + with path.open("ab") as handle: + handle.write(b'PhotoEditor_Re_Edit_Data{"genAIType":17}') + + assert identify(path, check_visible=False, check_invisible=False).confidence == "medium" + _assert_same_verdict(path) + + def test_a_metadata_chunk_past_the_head_window_survives(self, tmp_path: Path): + """A PNG encoder may put the label chunk after the pixels. The file is larger + than the record's head window, so only the seek-past-IDAT reader finds it.""" + path = _noise_png(tmp_path / "late.png") + assert path.stat().st_size > HEAD_WINDOW + # An XMP packet in the namespaced TC260 form, which is how the label travels + # when it is not a bare ``AIGC`` keyword chunk. + label = json.dumps({"Label": "1", "ContentProducer": "001191110102MACQD9K64010000"}) + xmp = ( + b'' + + label.encode() + + b"" + ) + _insert_png_chunk(path, b"iTXt", b"XML:com.adobe.xmp\x00\x00\x00\x00\x00" + xmp) + + assert "aigc" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals} + _assert_same_verdict(path) + + def test_an_exif_pair_survives(self, tmp_path: Path): + """xAI is recognized from an (ImageDescription, Artist) PAIR, and both are read + by tag NAME. A record carrying only raw bytes loses it.""" + import piexif + + path = tmp_path / "grok.jpg" + Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG") + exif = { + "0th": { + piexif.ImageIFD.ImageDescription: b"Signature: " + b"A" * 80, + piexif.ImageIFD.Artist: b"3f2504e0-4f89-11d3-9a0c-0305e82c3301", + } + } + piexif.insert(piexif.dump(exif), str(path)) + + assert "xai_signature" in {s.name for s in identify(path, check_visible=False, check_invisible=False).signals} + _assert_same_verdict(path) + + +class TestRecordShape: + def test_the_record_survives_json(self, tmp_path: Path): + record = collect_metadata_record(_noise_png(tmp_path / "plain.png")) + + assert json.loads(json.dumps(record, allow_nan=False))["container"] == "png" + assert record["schema_version"] == METADATA_RECORD_SCHEMA_VERSION == 1 + assert record["record_type"] == METADATA_RECORD_TYPE == "provenance_metadata" + + @pytest.mark.parametrize("keep_version", [True, False]) + def test_transport_filename_is_not_evidence(self, tmp_path: Path, keep_version: bool): + """The path labels a record; detector tokens in it do not describe pixels.""" + path = tmp_path / "jumb-c2pa-OpenAI-trainedAlgorithmicMedia.jpg" + Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "JPEG") + record = collect_metadata_record(path) + if not keep_version: + record.pop("schema_version") + record.pop("record_type") + + report = identify_metadata_record(record, path=path) + + assert report.is_ai_generated is None + assert report.platform is None + assert report.confidence == "none" + + def test_pixels_are_not_carried(self, tmp_path: Path): + """The reason the record walks regions instead of shipping the head: a record + that carried the pixel stream would be the size of the image.""" + path = _noise_png(tmp_path / "big.png", size=(1200, 1200)) + record = collect_metadata_record(path) + + idat = path.read_bytes() + start = idat.index(b"IDAT") + 4 + assert idat[start : start + 512] not in json.dumps(record).encode() + assert len(json.dumps(record)) < path.stat().st_size // 10 + + def test_an_unreadable_container_still_yields_a_record(self, tmp_path: Path): + path = tmp_path / "junk.bin" + path.write_bytes(b"\x00\x01\x02not an image at all" * 100) + + record = collect_metadata_record(path) + + assert record["container"] == "unknown" + assert json.dumps(record) # serializable, and no exception on the way here + _assert_same_verdict(path) + + def test_a_missing_file_does_not_raise(self, tmp_path: Path): + """Collection runs over whatever a caller hands it, including a path that + vanished between listing and reading.""" + record = collect_metadata_record(tmp_path / "gone.png") + + assert record["container"] == "unknown" + assert record["metadata_base64"] == "" + assert record["status"] == "error" + assert record["issues"] == [{"stage": "source", "code": "unavailable"}] + + def test_unknown_record_schema_is_rejected(self, tmp_path: Path): + path = _noise_png(tmp_path / "plain.png") + record = collect_metadata_record(path) + record["schema_version"] = 2 + + with pytest.raises(ValueError, match="Unsupported provenance metadata schema"): + identify_metadata_record(record, path=path) + + @pytest.mark.parametrize("schema_version", [True, 1.0, "1", None]) + def test_native_record_schema_requires_the_integer_one(self, tmp_path: Path, schema_version: object): + path = _noise_png(tmp_path / "plain.png") + record = collect_metadata_record(path) + record["schema_version"] = schema_version + + with pytest.raises(ValueError, match="Unsupported provenance metadata schema"): + identify_metadata_record(record, path=path) + + @pytest.mark.parametrize("status", [None, "partial", "unknown", True]) + def test_native_record_requires_complete_collection_status(self, tmp_path: Path, status: object): + path = _noise_png(tmp_path / "plain.png") + record = collect_metadata_record(path) + if status is None: + record.pop("status") + else: + record["status"] = status + + with pytest.raises(ValueError, match="collection status"): + identify_metadata_record(record, path=path) + + @pytest.mark.parametrize("schema_version", [2, True, 1.0]) + def test_collection_rejects_unsupported_output_schema_before_reading(self, tmp_path: Path, schema_version: object): + with pytest.raises(ValueError, match="Unsupported provenance metadata schema"): + collect_metadata_record( + tmp_path / "missing.png", + schema_version=schema_version, # type: ignore[arg-type] + ) + + def test_broad_forensic_record_is_not_detector_input(self, tmp_path: Path): + path = _noise_png(tmp_path / "plain.png") + + with pytest.raises(ValueError, match="Unsupported metadata record type"): + identify_metadata_record( + {"record_type": "forensic_metadata", "schema_version": 1}, + path=path, + ) + + def test_failed_collection_cannot_be_judged_as_an_unknown_image(self, tmp_path: Path): + path = tmp_path / "missing.png" + + with pytest.raises(ValueError, match="collection failed"): + identify_metadata_record(collect_metadata_record(path), path=path) + + +class TestReportTransport: + def test_report_contract_is_versioned_json_and_omits_local_path(self, tmp_path: Path): + path = _noise_png(tmp_path / "plain.png") + + payload = identify_metadata_record(collect_metadata_record(path), path=path).to_dict() + + assert payload["schema_version"] == PROVENANCE_REPORT_SCHEMA_VERSION == 1 + assert "path" not in payload + assert json.loads(json.dumps(payload, allow_nan=False)) == payload + + @pytest.mark.parametrize("schema_version", [2, True, 1.0]) + def test_report_rejects_unsupported_output_schema(self, tmp_path: Path, schema_version: object): + path = _noise_png(tmp_path / "plain.png") + report = identify_metadata_record(collect_metadata_record(path), path=path) + + with pytest.raises(ValueError, match="Unsupported provenance report schema"): + report.to_dict(schema_version=schema_version) # type: ignore[arg-type] + + def test_convenience_entry_point_matches_explicit_sequence(self, tmp_path: Path): + path = _noise_png(tmp_path / "plain.png") + record = collect_metadata_record(path) + + explicit = identify_from_evidence(evidence_from_metadata_record(record, path=path)) + convenience = identify_metadata_record(record, path=path) + + assert convenience == explicit + + +def test_a_webp_record_matches(tmp_path: Path): + """RIFF has its own walk; a chunk kept or dropped wrongly shows up here.""" + path = tmp_path / "image.webp" + Image.fromarray(np.zeros((64, 64, 3), dtype=np.uint8)).save(path, "WEBP", xmp=b"plain") + + assert collect_metadata_record(path)["container"] == "webp" + _assert_same_verdict(path) + + +def test_a_webp_record_collects_metadata_after_a_large_frame(tmp_path: Path): + """The RIFF walker seeks past coded pixels instead of stopping at the head window.""" + path = tmp_path / "late.webp" + rng = np.random.default_rng(7) + pixels = rng.integers(0, 255, (900, 900, 3), dtype=np.uint8) + xmp = b"trainedAlgorithmicMedia" + Image.fromarray(pixels).save(path, "WEBP", lossless=True, xmp=xmp) + + assert path.stat().st_size > HEAD_WINDOW + _assert_same_verdict(path) + + +def test_a_webp_record_does_not_carry_animation_frame_pixels(tmp_path: Path): + """ANMF is a coded-frame container, not a metadata chunk.""" + from remove_ai_watermarks.metadata_record import _riff_regions + + frame = b"jumb c2pa OpenAI trainedAlgorithmicMedia" * 20 + riff = b"RIFF" + (4 + 8 + len(frame)).to_bytes(4, "little") + b"WEBP" + riff += b"ANMF" + len(frame).to_bytes(4, "little") + frame + + assert frame not in _riff_regions(riff) + + +def test_an_invalid_short_riff_size_does_not_turn_the_container_into_a_trailer(tmp_path: Path): + from remove_ai_watermarks.metadata_record import _trailer + + path = tmp_path / "invalid.webp" + path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPjumb c2pa trainedAlgorithmicMedia") + + assert _trailer(path, "webp") == b"" + + +def test_the_record_never_reopens_the_source(tmp_path: Path, monkeypatch): + """The reason the record exists: the verdict must run where the file is not.""" + path = FIXTURES / "chatgpt-1.png" if (FIXTURES / "chatgpt-1.png").exists() else _noise_png(tmp_path / "x.png") + record = json.loads(json.dumps(collect_metadata_record(path))) + + def fail_if_called(*args, **kwargs): + raise AssertionError("the record path must not open the source file") + + monkeypatch.setattr("builtins.open", fail_if_called) + monkeypatch.setattr(Path, "open", fail_if_called) + monkeypatch.setattr(Image, "open", fail_if_called) + + report = identify_from_evidence(evidence_from_metadata_record(record, path=path)) + + assert isinstance(report, ProvenanceReport) diff --git a/tests/test_pixel_evidence.py b/tests/test_pixel_evidence.py new file mode 100644 index 0000000..5ff96c0 --- /dev/null +++ b/tests/test_pixel_evidence.py @@ -0,0 +1,253 @@ +"""Tests for the pixel-forensics collector. + +These pin the service contract and the edge cases a consumer cannot infer safely: +a family is empty rather than wrong when the image is too small for it, one failing +family does not lose the other five, and artifacts that identify the source image +stay behind their opt-in. +""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +import numpy as np +import pytest +from PIL import Image + +from remove_ai_watermarks.pixel_evidence import ( + AC_POSITIONS, + PIXEL_EVIDENCE_SCHEMA_VERSION, + PixelEvidence, + _dct_matrix, + dct_features, + extract_pixel_evidence, + is_available, + perceptual_hash, +) + +if TYPE_CHECKING: + from pathlib import Path + +FAMILIES = ("dct", "fft", "noise", "ela", "gradient", "color") + + +def _textured(path: Path, size: tuple[int, int] = (192, 160), *, seed: int = 0) -> Path: + """A textured image. Flat color would make several families degenerate (zero + residual, empty gradient histogram) and hide a real break.""" + rng = np.random.default_rng(seed) + base = rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8) + ramp = np.linspace(0, 255, size[0], dtype=np.uint8)[None, :, None] + Image.fromarray(np.clip(base // 2 + ramp // 2, 0, 255).astype(np.uint8)).save(path) + return path + + +class TestFamilies: + def test_every_family_is_measured_on_a_textured_image(self, tmp_path: Path): + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png")) + + assert evidence.decoded + for family in FAMILIES: + assert getattr(evidence, family), family + + def test_the_same_image_measures_the_same_twice(self, tmp_path: Path): + """Determinism is what makes these comparable across runs and machines; the + residual is computed in row chunks, which is exactly the kind of optimization + that can perturb the last bits.""" + path = _textured(tmp_path / "textured.png") + + first, second = extract_pixel_evidence(path), extract_pixel_evidence(path) + + assert dataclasses.asdict(first) == dataclasses.asdict(second) + + def test_source_dimensions_survive_the_downscale(self, tmp_path: Path): + """The recorded size is the SOURCE size, read before the 2048px cap. Recording + the analysed size instead would still pass every statistic check, because + those run on the downscaled array either way.""" + path = tmp_path / "oversize.png" + Image.fromarray(np.zeros((80, 3000, 3), dtype=np.uint8)).save(path) + + assert extract_pixel_evidence(path).decode == {"width": 3000, "height": 80} + + def test_selected_dct_coefficients_match_the_full_transform(self): + gray = np.random.default_rng(7).uniform(0, 255, (24, 32)).astype(np.float32) + basis = _dct_matrix(np) + blocks = gray.reshape(3, 8, 4, 8).swapaxes(1, 2) + full = np.einsum("ij,abjk,lk->abil", basis, blocks, basis) + bins = np.linspace(-20.5, 20.5, 22) + expected = [ + np.histogram(full[:, :, row, column].ravel(), bins=bins)[0].tolist() for row, column in AC_POSITIONS + ] + + assert dct_features(gray)["dct_ac_hist"] == expected + + def test_perceptual_hash_matches_the_full_transform(self): + gray = np.random.default_rng(8).uniform(0, 255, (32, 32)).astype(np.float32) + basis = _dct_matrix(np, 32) + coefficients = basis @ gray @ basis.T + low = coefficients[:8, :8].ravel()[1:] + bits = low > np.median(low) + expected = f"{int(''.join('1' if bit else '0' for bit in bits), 2):016x}" + + assert perceptual_hash(gray) == expected + + +class TestDegenerateInputs: + def test_undecodable_file_reports_the_error_and_stays_empty(self, tmp_path: Path): + path = tmp_path / "broken.png" + path.write_bytes(b"not an image") + + evidence = extract_pixel_evidence(path, artifacts=True) + + assert evidence.decoded is False + assert "error" in evidence.decode + assert all(getattr(evidence, family) == {} for family in FAMILIES) + assert evidence.artifacts == {} + + def test_image_too_small_for_a_family_leaves_it_empty(self, tmp_path: Path): + """8x8 is below the FFT's 32px floor but at the block DCT's. A consumer must + not assume a fixed feature width, so the narrow case is pinned.""" + path = tmp_path / "tiny.png" + Image.fromarray(np.arange(8 * 8 * 3, dtype=np.uint8).reshape(8, 8, 3)).save(path) + + evidence = extract_pixel_evidence(path) + + assert evidence.decoded is True + assert evidence.fft == {} + assert evidence.color != {} + + def test_a_failing_family_does_not_lose_the_others(self, tmp_path: Path, monkeypatch): + path = _textured(tmp_path / "textured.png") + + def boom(*args, **kwargs): + raise ValueError(f"family failed for {path}") + + monkeypatch.setattr("remove_ai_watermarks.pixel_evidence.color_features", boom) + evidence = extract_pixel_evidence(path) + + assert evidence.color == {"error": "ValueError"} + assert evidence.status == "partial" + assert evidence.to_dict()["status"] == "partial" + assert evidence.dct != {} + assert evidence.gradient != {} + + +class TestArtifactsAreOptIn: + """The artifacts identify the source image -- a thumbnail is a picture, a + perceptual hash matches one. Everything else is a scalar or a fixed-length + histogram. That difference in kind is the reason for the flag, so the flag is + what these guard.""" + + def test_off_by_default(self, tmp_path: Path): + assert extract_pixel_evidence(_textured(tmp_path / "textured.png")).artifacts == {} + + def test_on_request_it_returns_the_spatial_layer(self, tmp_path: Path): + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), artifacts=True) + + assert set(evidence.artifacts) == {"phash", "thumbnail_jpeg_b64", "ela_map", "noise_residual", "fft_phase"} + assert len(evidence.artifacts["phash"]) == 16 + + def test_the_thumbnail_is_a_readable_image_of_the_source(self, tmp_path: Path): + """Stated plainly because it is the privacy claim: this field reconstructs + the picture, at 128px.""" + import base64 + import io + + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), artifacts=True) + + with Image.open(io.BytesIO(base64.b64decode(evidence.artifacts["thumbnail_jpeg_b64"]))) as thumb: + assert max(thumb.size) <= 128 + + def test_a_different_image_hashes_differently(self, tmp_path: Path): + one = extract_pixel_evidence(_textured(tmp_path / "a.png", seed=1), artifacts=True) + two = extract_pixel_evidence(_textured(tmp_path / "b.png", seed=2), artifacts=True) + + assert one.artifacts["phash"] != two.artifacts["phash"] + + def test_the_statistics_carry_no_array_payloads(self, tmp_path: Path): + """Without the flag nothing array-shaped may appear: that is what makes the + default set aggregates rather than content.""" + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png")) + + for family in FAMILIES: + for key, value in getattr(evidence, family).items(): + assert isinstance(value, (int, float, str, list)), (family, key) + if isinstance(value, list): + for item in value: + assert isinstance(item, (int, float, list)), (family, key) + + +def test_is_available_reports_the_optional_dependency(): + assert is_available() is True # the test environment installs the pixels extra + + +def test_evidence_is_frozen(tmp_path: Path): + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png")) + assert isinstance(evidence, PixelEvidence) + with pytest.raises(dataclasses.FrozenInstanceError): + evidence.decode = {} # type: ignore[misc] + + +def test_transport_contract_is_versioned_json_and_omits_path(tmp_path: Path): + import json + + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png"), timings=True) + payload = evidence.to_dict() + + assert payload["schema_version"] == PIXEL_EVIDENCE_SCHEMA_VERSION == 1 + assert payload["status"] == "complete" + assert "path" not in payload + assert payload["timing_ms"]["total"] >= 0 + assert json.loads(json.dumps(payload, allow_nan=False)) == payload + + +@pytest.mark.parametrize("schema_version", [2, True, 1.0]) +def test_transport_rejects_unsupported_output_schema(tmp_path: Path, schema_version: object): + evidence = extract_pixel_evidence(_textured(tmp_path / "textured.png")) + + with pytest.raises(ValueError, match="Unsupported pixel evidence schema"): + evidence.to_dict(schema_version=schema_version) # type: ignore[arg-type] + + +def test_decode_error_transport_does_not_leak_the_local_path(tmp_path: Path): + import json + + path = tmp_path / "broken.png" + path.write_bytes(b"not an image") + + payload = extract_pixel_evidence(path).to_dict() + + assert payload["status"] == "error" + assert payload["decode"]["error"] == "UnidentifiedImageError" + assert str(path) not in json.dumps(payload) + + +def test_timings_are_opt_in(tmp_path: Path): + path = _textured(tmp_path / "textured.png") + + assert extract_pixel_evidence(path).timing_ms == {} + assert set(extract_pixel_evidence(path, artifacts=True, timings=True).timing_ms) == { + "decode", + "noise", + "fft", + "ela", + "dct", + "gradient", + "color", + "full_artifacts", + "total", + } + + +def test_pixel_decode_registers_optional_heif_opener(tmp_path: Path, monkeypatch): + registered = False + + def mark_registered(): + nonlocal registered + registered = True + + monkeypatch.setattr("remove_ai_watermarks.image_io._register_heif", mark_registered) + + extract_pixel_evidence(_textured(tmp_path / "textured.png")) + + assert registered is True diff --git a/tests/test_security_clamp.py b/tests/test_security_clamp.py index ffdaae5..4e2a30f 100644 --- a/tests/test_security_clamp.py +++ b/tests/test_security_clamp.py @@ -18,10 +18,14 @@ from __future__ import annotations import struct import tracemalloc +from typing import TYPE_CHECKING from remove_ai_watermarks import metadata from remove_ai_watermarks._internal import c2pa, isobmff +if TYPE_CHECKING: + from pathlib import Path + PNG_SIG = b"\x89PNG\r\n\x1a\n" _HUGE = 0x7FFFFFFF # ~2 GiB declared length on a tiny file @@ -128,3 +132,28 @@ class TestIsobmffStripFailSafe: assert stripped == 0 assert cleaned == data assert len(cleaned) == len(data) + + +class TestRiffLateMetadataIsBounded: + """A metadata scan must not become a near-full-file read because one chunk lied + about its length. This runs on the memoized verdict path, over images from + arbitrary sources.""" + + def _webp_with_declared_length(self, path: Path, declared: int, payload: bytes) -> Path: + chunk = b"XMP " + declared.to_bytes(4, "little") + payload + body = b"WEBP" + b"VP8 " + (4).to_bytes(4, "little") + b"\x00\x00\x00\x00" + chunk + path.write_bytes(b"RIFF" + len(body).to_bytes(4, "little") + body) + return path + + def test_a_chunk_claiming_the_whole_file_is_clamped_to_what_remains(self, tmp_path: Path): + target = self._webp_with_declared_length(tmp_path / "liar.webp", 1 << 30, b"AI" * 64) + + collected = metadata._riff_late_metadata(target, 12) + + assert collected == b"AI" * 64 + + def test_the_total_is_capped(self, tmp_path: Path): + payload = b"x" * 4096 + target = self._webp_with_declared_length(tmp_path / "big.webp", len(payload), payload) + + assert len(metadata._riff_late_metadata(target, 12, max_total=512)) == 512 diff --git a/uv.lock b/uv.lock index d59fbf8..ad9ee33 100644 --- a/uv.lock +++ b/uv.lock @@ -615,7 +615,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly", marker = "python_full_version < '3.11'" }, + { name = "humanfriendly" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -787,7 +787,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -822,43 +822,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufft = [ - { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufile = [ - { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] curand = [ - { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] [[package]] @@ -974,8 +974,8 @@ name = "email-validator" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dnspython", marker = "python_full_version >= '3.12'" }, - { name = "idna", marker = "python_full_version >= '3.12'" }, + { name = "dnspython" }, + { name = "idna" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } wheels = [ @@ -987,7 +987,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1253,7 +1253,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -1328,8 +1328,8 @@ name = "inflect" version = "7.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools", marker = "python_full_version >= '3.12'" }, - { name = "typeguard", marker = "python_full_version >= '3.12'" }, + { name = "more-itertools" }, + { name = "typeguard" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" } wheels = [ @@ -1810,7 +1810,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "nvidia-cuda-nvrtc" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -1849,7 +1849,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -1861,7 +1861,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -1891,9 +1891,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, - { name = "nvidia-cusparse", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -1905,7 +1905,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.12' or sys_platform != 'darwin'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -1980,12 +1980,12 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "coloredlogs", marker = "python_full_version < '3.11'" }, - { name = "flatbuffers", marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "protobuf", marker = "python_full_version < '3.11'" }, - { name = "sympy", marker = "python_full_version < '3.11'" }, + { name = "coloredlogs" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, @@ -2034,10 +2034,10 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "flatbuffers", marker = "python_full_version >= '3.11'" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "protobuf", marker = "python_full_version >= '3.11'" }, + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/17/4d/5014667e2a3a77d6e1b74cc3d88948d06163b8e0a33a84c85073322b5dec/onnxruntime-1.28.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f5c5daabd28aad610f83fdcf32acec8fb57e6adc6c6a39fe2a3c755db957b410", size = 19130506, upload-time = "2026-07-25T01:22:34.489Z" }, @@ -2205,10 +2205,10 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "pytz", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, - { name = "tzdata", marker = "python_full_version < '3.11' or python_full_version >= '3.14'" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2278,9 +2278,9 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'win32')" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } wheels = [ @@ -2759,10 +2759,10 @@ name = "pydantic" version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-types", marker = "python_full_version >= '3.12'" }, - { name = "pydantic-core", marker = "python_full_version >= '3.12'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, - { name = "typing-inspection", marker = "python_full_version >= '3.12'" }, + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ @@ -2771,7 +2771,7 @@ wheels = [ [package.optional-dependencies] email = [ - { name = "email-validator", marker = "python_full_version >= '3.12'" }, + { name = "email-validator" }, ] [[package]] @@ -2779,7 +2779,7 @@ name = "pydantic-core" version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ @@ -3025,7 +3025,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/45/bfaaab38545a33a9f06c61211fc3bea2e23e8a8e00fedeb8e57feda722ff/pywavelets-1.8.0.tar.gz", hash = "sha256:f3800245754840adc143cbc29534a1b8fc4b8cff6e9d403326bd52b7bb5c35aa", size = 3935274, upload-time = "2024-12-04T19:54:20.593Z" } wheels = [ @@ -3090,7 +3090,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/75/50581633d199812205ea8cdd0f6d52f12a624886b74bf1486335b67f01ff/pywavelets-1.9.0.tar.gz", hash = "sha256:148d12203377772bea452a59211d98649c8ee4a05eff019a9021853a36babdc8", size = 3938340, upload-time = "2025-08-04T16:20:04.978Z" } wheels = [ @@ -3331,7 +3331,7 @@ wheels = [ [[package]] name = "remove-ai-watermarks" -version = "0.25.0" +version = "0.26.0" source = { editable = "." } dependencies = [ { name = "c2pa-python" }, @@ -3659,7 +3659,7 @@ name = "stamina" version = "26.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tenacity", marker = "python_full_version >= '3.12'" }, + { name = "tenacity" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/bd/b2f71ae14368a066f103d182f25bbc6c3bf4aa695889f3ed3cba026d6f36/stamina-26.1.0.tar.gz", hash = "sha256:0214d05fdf5102c518194a4aac7520ce53cf660550ae3b940701aad88cf50c17", size = 568171, upload-time = "2026-04-13T17:44:31.012Z" } wheels = [ @@ -3959,7 +3959,7 @@ name = "typeguard" version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b4/de/4420db493fa8fc0856d5e5c1b159c63a323d2de2317babe36b01568928e8/typeguard-4.6.0.tar.gz", hash = "sha256:e7414f09111317de3e335de92cd397c5c0ca00b1cc1676de12e1d444a79b3f21", size = 82330, upload-time = "2026-07-26T08:40:23.207Z" } wheels = [ @@ -3995,7 +3995,7 @@ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ @@ -4025,10 +4025,10 @@ name = "uv-outdated" version = "1.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", marker = "python_full_version >= '3.12'" }, - { name = "rich", marker = "python_full_version >= '3.12'" }, - { name = "typer", marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "rich" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/84/78736b81c0e6ebefd3810b04a3bc6cb82bf7ea63474821b02d5cd9040439/uv_outdated-1.0.4.tar.gz", hash = "sha256:126745028823d8d452a82faaf53ea1d4ab5cdea7bba3159fc2ce7e5d0443146c", size = 19176, upload-time = "2025-12-25T10:54:22.77Z" } wheels = [ @@ -4040,18 +4040,18 @@ name = "uv-secure" version = "0.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "python_full_version >= '3.12'" }, - { name = "cvss", marker = "python_full_version >= '3.12'" }, - { name = "httpx", marker = "python_full_version >= '3.12'" }, - { name = "humanize", marker = "python_full_version >= '3.12'" }, - { name = "inflect", marker = "python_full_version >= '3.12'" }, - { name = "orjson", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pydantic", extra = ["email"], marker = "python_full_version >= '3.12'" }, - { name = "rich", marker = "python_full_version >= '3.12'" }, - { name = "stamina", marker = "python_full_version >= '3.12'" }, - { name = "tomlkit", marker = "python_full_version >= '3.12'" }, - { name = "typer", marker = "python_full_version >= '3.12'" }, + { name = "anyio" }, + { name = "cvss" }, + { name = "httpx" }, + { name = "humanize" }, + { name = "inflect" }, + { name = "orjson" }, + { name = "packaging" }, + { name = "pydantic", extra = ["email"] }, + { name = "rich" }, + { name = "stamina" }, + { name = "tomlkit" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/99/29318cedfc5583cf2d503f0eedb9c4e96829541c356ce5d2aacfe09ef67f/uv_secure-0.17.2.tar.gz", hash = "sha256:e394939e0872df392d8f650d15ac1571b9267fc2f3671a183aa73c0977f0f402", size = 47240, upload-time = "2026-04-18T08:45:38.185Z" } wheels = [