From bebff368fc2abdc86601d00dc1b29d61576fbf1b Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Wed, 5 Aug 2026 15:09:33 -0700 Subject: [PATCH] Decide the SynthID proxy in the verdict, where both extractors meet A full-corpus audit of the record path against the file path found 75 of 48,905 images disagreeing, and 74 were one gap: the SynthID byte scan for containers whose manifest no parser reaches lived in `get_ai_metadata`, an extractor the record path does not run. The record silently reported no SynthID for images `identify` flagged. Moving the scan into `identify_from_evidence` fixes it by construction rather than by copying the rule into a second extractor -- the same shape `soft_binding` already uses. Its byte checks mirror `metadata.synthid_source` literally instead of reusing the broader `has_c2pa` / `c2pa_source_kind` derived above, so the file path's answers do not move: verdicts over a 4,000-image sample are byte-identical. `scripts/record_parity_audit.py` is the audit itself, now repeatable. It walks a dataset, judges every image through both seams with the record round-tripped through JSON, and reports disagreements by field and by signal. The rule in `.claude/rules/development.md` says to re-run both sides of this seam after changing either; this is what to run. Both timing and audit scripts now put the package's OWN `src` on the path. From a worktree an editable install resolves to the main checkout, so the audit imported a different tree than the one under test -- the failure the same rules file warns about, reproduced within an hour of writing it down. Co-Authored-By: Claude Opus 5 --- docs/module-internals.md | 8 +- scripts/detection_timing.py | 5 +- scripts/record_parity_audit.py | 216 +++++++++++++++++++++++++++ src/remove_ai_watermarks/identify.py | 21 ++- tests/test_identify.py | 53 +++++++ 5 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 scripts/record_parity_audit.py diff --git a/docs/module-internals.md b/docs/module-internals.md index c167325..980045d 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -391,7 +391,13 @@ metadata extraction from verdict logic: metadata record into the same evidence type without file access. Diagnostic values under `error` and `kind` are excluded from evidence while nested raw bytes remain available through encoded binary fields. -- `identify_from_evidence` evaluates that evidence without reopening the source. +- `identify_from_evidence` evaluates that evidence without reopening the source. Rules + that decide a verdict live here, not in extraction: extraction has two + implementations, and a rule in only one of them is a rule the other lacks. The + SynthID proxy is the worked example — its structured form comes from the manifest, + and the byte-scan fallback for containers no parser reaches runs in the verdict, so + both extractors reach the same answer. It did not, and the record path silently + reported no SynthID for images the file path flagged. - `identify` preserves the path-based API and adds the optional registered visible-mark and open invisible-watermark decoders after extraction. diff --git a/scripts/detection_timing.py b/scripts/detection_timing.py index bb3bfb2..2d1838a 100644 --- a/scripts/detection_timing.py +++ b/scripts/detection_timing.py @@ -55,7 +55,10 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Callable, Iterator -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +# The package's OWN tree, not the repository root: from a worktree, an editable +# install resolves `remove_ai_watermarks` to the MAIN checkout, so a script measuring +# this tree would silently import a different one. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from remove_ai_watermarks import identify as identify_mod from remove_ai_watermarks import metadata as metadata_mod diff --git a/scripts/record_parity_audit.py b/scripts/record_parity_audit.py new file mode 100644 index 0000000..554401f --- /dev/null +++ b/scripts/record_parity_audit.py @@ -0,0 +1,216 @@ +"""Audit the record path against the file path over a whole dataset. + +WHY THIS EXISTS + +Two seams reach the same provenance verdict: + + identify(path, check_visible=False, check_invisible=False) + + identify_metadata_record(collect_metadata_record(path), path=path) + +Their equality is the record's entire contract, and it can break from either side -- +a region the collector stops walking, or a placement the file path learns to read and +the record does not. ``tests/test_metadata_record.py`` pins it over the tracked +fixtures; those cover the signal families we already know about. This covers the ones +we do not: every real placement in a real corpus, which is where all three defects +found so far actually came from. + +The record is round-tripped through ``json.dumps``/``loads`` before it is judged, so +a value that only survives in memory fails here rather than at a customer. + +WHAT IT REPORTS + +One JSONL row per image: both verdicts, whether they agree, the record size, and any +exception from either side. The summary counts disagreements by field and by signal, +so "the record lost samsung_genai on 13 files" reads directly off the output instead +of being reconstructed. + +Pass ``--baseline`` with an earlier run to also diff against it. That answers the +other question a detection change raises: which files changed verdict, and are they +exactly the ones that were meant to. + +DATA SAFETY + +Read-only over a local dataset. Writes only the given output path, which belongs +outside the repository. Resumable: rerunning skips files already recorded. + + uv run python scripts/record_parity_audit.py data/spaces/originals .local-eval/parity.jsonl + uv run python scripts/record_parity_audit.py --baseline .local-eval/previous.jsonl +""" + +from __future__ import annotations + +import argparse +import collections +import json +import logging +import sys +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + +# The package's OWN tree, not the repository root: from a worktree, an editable +# install resolves `remove_ai_watermarks` to the MAIN checkout, so a script measuring +# this tree would silently import a different one. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from remove_ai_watermarks.identify import identify, identify_metadata_record +from remove_ai_watermarks.metadata_record import collect_metadata_record + +log = logging.getLogger(__name__) + +SUPPORTED = frozenset({".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif", ".avif"}) +# Every field of the verdict a caller can act on. `path` is excluded: it is extraction +# context, and the two paths are handed the same one by construction. +COMPARED = ("is_ai_generated", "platform", "confidence", "ai_source_kind", "ai_from_metadata") + + +def _verdict(report: Any) -> dict[str, Any]: + return { + **{field: getattr(report, field) for field in COMPARED}, + "signals": sorted(signal.name for signal in report.signals), + "watermarks": sorted(report.watermarks), + } + + +def _audit(path: Path) -> dict[str, Any]: + row: dict[str, Any] = {"path": str(path)} + try: + row["bytes"] = path.stat().st_size + except OSError as exc: + return {**row, "error": f"stat: {exc}"} + + try: + started = time.perf_counter() + record = json.loads(json.dumps(collect_metadata_record(path))) + row["collect_ms"] = (time.perf_counter() - started) * 1000 + row["record_bytes"] = len(json.dumps(record)) + row["container"] = record.get("container") + via_record = _verdict(identify_metadata_record(record, path=path)) + except Exception as exc: + return {**row, "error": f"record path: {type(exc).__name__}: {exc}"} + + try: + via_file = _verdict(identify(path, check_visible=False, check_invisible=False)) + except Exception as exc: + return {**row, "error": f"file path: {type(exc).__name__}: {exc}"} + + row["record"] = via_record + row["file"] = via_file + row["agree"] = via_record == via_file + return row + + +def _iter_images(root: Path) -> Iterator[Path]: + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix.lower() in SUPPORTED: + yield path + + +def _done(out_path: Path) -> set[str]: + if not out_path.exists(): + return set() + done: set[str] = set() + with out_path.open(encoding="utf-8") as handle: + for line in handle: + try: + done.add(json.loads(line)["path"]) + except (ValueError, KeyError): + continue + return done + + +def _summarize(rows: list[dict[str, Any]], baseline: Path | None) -> None: + failed = [r for r in rows if "error" in r] + usable = [r for r in rows if "error" not in r] + disagreed = [r for r in usable if not r["agree"]] + + print(f"\nimages: {len(rows)} errors: {len(failed)} compared: {len(usable)}") + print(f"record path disagrees with file path: {len(disagreed)}") + for row in failed[:10]: + print(f" ERROR {Path(row['path']).name}: {row['error']}") + + fields: collections.Counter[str] = collections.Counter() + for row in disagreed: + fields.update(field for field in COMPARED if row["record"][field] != row["file"][field]) + for name in set(row["file"]["signals"]) - set(row["record"]["signals"]): + fields[f"signal missing from record: {name}"] += 1 + for name in set(row["record"]["signals"]) - set(row["file"]["signals"]): + fields[f"signal only in record: {name}"] += 1 + for label, count in fields.most_common(): + print(f" {label}: {count}") + for row in disagreed[:10]: + print(f" {Path(row['path']).name}\n record: {row['record']}\n file: {row['file']}") + + if baseline is None: + return + previous = {} + with baseline.open(encoding="utf-8") as handle: + for line in handle: + try: + item = json.loads(line) + except ValueError: + continue + if "error" not in item: + previous[Path(item["path"]).name] = item + + changed = [] + for row in usable: + was = previous.get(Path(row["path"]).name) + if was is None: + continue + before = was.get("file") or {"confidence": was.get("confidence"), "signals": was.get("signals")} + if before.get("confidence") != row["file"]["confidence"] or before.get("signals") != row["file"]["signals"]: + changed.append((row["path"], before, row["file"])) + print(f"\nverdicts changed against the baseline: {len(changed)}") + gained: collections.Counter[str] = collections.Counter() + for _, before, after in changed: + for name in set(after["signals"]) - set(before.get("signals") or []): + gained[f"gained {name}"] += 1 + for name in set(before.get("signals") or []) - set(after["signals"]): + gained[f"LOST {name}"] += 1 + for label, count in gained.most_common(): + print(f" {label}: {count}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("dataset", type=Path) + parser.add_argument("out", type=Path) + parser.add_argument("--baseline", type=Path, default=None, help="an earlier run to diff verdicts against") + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--progress-every", type=int, default=2000) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + args.out.parent.mkdir(parents=True, exist_ok=True) + done = _done(args.out) + if done: + log.info("resuming: %d images already audited", len(done)) + + processed = 0 + started = time.monotonic() + with args.out.open("a", encoding="utf-8") as handle: + for path in _iter_images(args.dataset): + if str(path) in done: + continue + handle.write(json.dumps(_audit(path), ensure_ascii=False, default=str) + "\n") + handle.flush() + processed += 1 + if processed % args.progress_every == 0: + log.info("%d images, %.1f/s", processed, processed / (time.monotonic() - started)) + if args.limit and processed >= args.limit: + break + + log.info("audited %d images in %.1f s", processed, time.monotonic() - started) + with args.out.open(encoding="utf-8") as handle: + rows = [json.loads(line) for line in handle] + _summarize(rows, args.baseline) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index ec92a76..b6054bc 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -30,6 +30,8 @@ from remove_ai_watermarks._internal.c2pa import ( cbor_text_after, extract_c2pa_info, soft_binding_vendors_in, + synthid_vendors_in, + synthid_verdict, ) from remove_ai_watermarks._internal.constants import ( C2PA_AI_TOOLS, @@ -980,9 +982,24 @@ def _identify_from_evidence( platform = f"C2PA signer: {cloud_vendor} (cloud manifest)" # ── SynthID metadata proxy ────────────────────────────────────── - # get_ai_metadata already sets synthid_watermark for both PNG (caBX parser) - # and non-PNG (its own synthid_source fallback), so no extra scan is needed. + # Structured first (the PNG caBX parser and the manifest store both fill + # `synthid_watermark`), then the byte scan for the containers that keep the + # manifest where no parser reaches it. + # + # The scan lives HERE, in the verdict, and not in extraction, for the same reason + # `soft_binding` below does: extraction has two implementations -- one reading a + # file, one reading a portable record -- and a rule that lives in only one of them + # is a rule the other silently lacks. It did: 74 corpus images reported SynthID + # through `identify` and not through the record, because `get_ai_metadata`'s own + # fallback has no counterpart on the record side. `get_ai_metadata` keeps its copy + # for its own callers; the verdict no longer depends on which extractor ran. synthid = meta.get("synthid_watermark") + # The literal byte checks mirror `metadata.synthid_source` exactly rather than + # reusing the derived `has_c2pa` / `c2pa_source_kind` above, which are broader: + # the file path's answer must not move. + trained_source = b"trainedAlgorithmicMedia" in head or b"TrainedAlgorithmicMedia" in head + if not synthid and trained_source and c2pa_marker_in(head) and (vendors := synthid_vendors_in(head)): + synthid = synthid_verdict(", ".join(vendors)) if synthid: watermarks.append(f"SynthID watermark, inferred from C2PA metadata ({synthid})") caveats.append(_SYNTHID_CAVEAT) diff --git a/tests/test_identify.py b/tests/test_identify.py index ee68391..26a7933 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -1353,3 +1353,56 @@ class TestSharedPixelDecode: report = identify(self.SAMPLE, check_visible=True, check_invisible=False) assert not any(s.name.startswith("visible_") for s in report.signals) assert report.is_ai_generated is True # the C2PA verdict survives the decode failure + + +class TestSynthIdProxyIsDecidedInTheVerdict: + """The SynthID byte scan belongs to the verdict, not to extraction. + + Extraction has two implementations -- one reading a file, one reading a portable + record -- so a rule that lives in only one of them is a rule the other silently + lacks. This one did: 74 corpus images reported SynthID through ``identify`` and + not through the record.""" + + # A JUMBF-wrapped manifest from a SynthID-pairing signer on AI-generated content: + # the exact shape `synthid_source`'s byte scan is gated on. Spliced into a real + # JPEG as a well-formed APP11 segment, because a malformed one is skipped by the + # record's structural walk and the test would compare two different inputs. + MANIFEST = b"jumb c2pa Google LLC trainedAlgorithmicMedia" + + def _jpeg_with_manifest(self, path: Path) -> Path: + import numpy as np + from PIL import Image + + Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG") + data = path.read_bytes() + segment = b"\xff\xeb" + (len(self.MANIFEST) + 2).to_bytes(2, "big") + self.MANIFEST + path.write_bytes(data[:2] + segment + data[2:]) + return path + + def test_both_paths_infer_it_from_the_same_bytes(self, tmp_path: Path): + from remove_ai_watermarks.identify import identify_metadata_record + from remove_ai_watermarks.metadata_record import collect_metadata_record + + path = self._jpeg_with_manifest(tmp_path / "gemini.jpg") + + via_file = identify(path, check_visible=False, check_invisible=False) + via_record = identify_metadata_record(collect_metadata_record(path), path=path) + + assert any("SynthID" in mark for mark in via_file.watermarks) + assert via_record.watermarks == via_file.watermarks + + def test_it_needs_a_manifest_and_an_ai_source_type(self, tmp_path: Path): + """The vendor name alone is not evidence: an ordinary photo mentioning + "Google LLC" in EXIF must not acquire a SynthID verdict.""" + import numpy as np + from PIL import Image + + path = tmp_path / "photo.jpg" + Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(path, "JPEG") + data = path.read_bytes() + note = b"Google LLC Pixel" + path.write_bytes(data[:2] + b"\xff\xeb" + (len(note) + 2).to_bytes(2, "big") + note + data[2:]) + + report = identify(path, check_visible=False, check_invisible=False) + + assert not any("SynthID" in mark for mark in report.watermarks)