diff --git a/CLAUDE.md b/CLAUDE.md index d05f00c..09640cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,6 @@ You are a **principal Python engineer** maintaining a CLI tool and library for r - Pyright first run is slow (2-3 min) due to ML deps (torch/diffusers/transformers stubs); full-project `uv run pyright` can stall for many minutes — scope it to changed files. - `ultralytics` monkey-patches `PIL.Image.open` and tries to autoload `pi_heif`. When `pi_heif` is missing, opening files raises `ModuleNotFoundError`, not `UnidentifiedImageError`. Code that opens user-supplied or unknown-format files should `except Exception`, not just `OSError`/`UnidentifiedImageError`. - Metadata detection for AVIF/HEIF/JPEG-XL relies on a binary scan for `C2PA_UUID` + `IPTC_AI_MARKERS`. C2PA removal in those containers is implemented via `noai/isobmff.py` (top-level ``uuid`` / ``jumb`` box stripper, no re-encoding). EXIF/XMP boxes inside those containers are not yet scrubbed. -- **SynthID detection is metadata-only.** There is no reliable *local* detector of the SynthID *pixel* watermark — Google's decoder is proprietary, no public spec or API (only a waitlisted portal). We detect SynthID by its C2PA companion (`synthid_source` / `SYNTHID_C2PA_ISSUERS`), which is reliable while the manifest is intact but says nothing once C2PA is stripped. **Surface-dependent blind spot (verified 2026-05-24):** the same Google model emits different metadata per surface -- the Gemini *app* wraps outputs in Google C2PA, but the *API/playground* (AI Studio, Nano Banana / gemini-2.5-flash-image) emits the SynthID *pixel* watermark (confirmed via the Gemini-app oracle) + the visible sparkle but **no C2PA/IPTC at all**, so `synthid_source` returns None despite SynthID being present. Only the pixel oracle or the visible-sparkle detector catches those. (Meta AI is another surface mismatch: it writes the IPTC `digitalSourceType=trainedAlgorithmicMedia` marker, not C2PA and not SynthID.) Google→SynthID is long-standing; OpenAI→SynthID is confirmed by OpenAI's Help Center (ChatGPT/Codex/API "include both C2PA metadata and SynthID watermarks", updated 2026-05-21) but time-gated (pre-rollout OpenAI images carry C2PA without SynthID), so the OpenAI verdict is hedged "likely". Oracles: Gemini app "Verify with SynthID" (Google), openai.com/verify (OpenAI). The spectral phase-coherence approach from `github.com/aloshdenny/reverse-SynthID` was evaluated (May 2026) and **does not work for real-content detection**: on its own shipped codebook + validation set, watermarked and cleaned images were indistinguishable (conf within noise, cleaned often higher); it only fires on pure-black 1024x1024 reference images at exact resolution (the controlled case it was calibrated on). The README's "90% / conf=0.91" reproduces only in that lab condition. Do not build a production detector on it; if revisited, it is experimental/diagnostic only and needs a per-resolution, per-model reference corpus. A from-scratch gpt-image pilot (2026-05-24) confirmed this independently: 5 independent solid-black gpt-image outputs share a near-identical fixed signature (pairwise residual correlation **0.92**, avg-template retains 97% energy), so the watermark/carrier IS strongly present and consistent on flat content — but the carrier frequencies extracted from it do NOT discriminate real content (carrier-to-random ratio: cleaned 1.86 > watermarked 1.53; a non-gpt-image image scored highest at 3.67). The signature drowns in content texture. Net: a perfectly consistent solid-color signature still yields no real-content pixel detector with magnitude/carrier methods. +- **SynthID detection is metadata-only.** There is no reliable *local* detector of the SynthID *pixel* watermark — Google's decoder is proprietary, no public spec or API (only a waitlisted portal). We detect SynthID by its C2PA companion (`synthid_source` / `SYNTHID_C2PA_ISSUERS`), which is reliable while the manifest is intact but says nothing once C2PA is stripped. **Surface-dependent blind spot (verified 2026-05-24):** the same Google model emits different metadata per surface -- the Gemini *app* wraps outputs in Google C2PA, but the *API/playground* (AI Studio, Nano Banana / gemini-2.5-flash-image) emits the SynthID *pixel* watermark (confirmed via the Gemini-app oracle) + the visible sparkle but **no C2PA/IPTC at all**, so `synthid_source` returns None despite SynthID being present. Only the pixel oracle or the visible-sparkle detector catches those. (Meta AI is another surface mismatch: it writes the IPTC `digitalSourceType=trainedAlgorithmicMedia` marker, not C2PA and not SynthID.) Google→SynthID is long-standing; OpenAI→SynthID is confirmed by OpenAI's Help Center (ChatGPT/Codex/API "include both C2PA metadata and SynthID watermarks", updated 2026-05-21) but time-gated (pre-rollout OpenAI images carry C2PA without SynthID), so the OpenAI verdict is hedged "likely". Oracles: Gemini app "Verify with SynthID" (Google), openai.com/verify (OpenAI). The spectral phase-coherence approach from `github.com/aloshdenny/reverse-SynthID` was evaluated (May 2026) and **does not work for real-content detection**: on its own shipped codebook + validation set, watermarked and cleaned images were indistinguishable (conf within noise, cleaned often higher); it only fires on pure-black 1024x1024 reference images at exact resolution (the controlled case it was calibrated on). The README's "90% / conf=0.91" reproduces only in that lab condition. Do not build a production detector on it; if revisited, it is experimental/diagnostic only and needs a per-resolution, per-model reference corpus. A from-scratch gpt-image pilot (2026-05-24) confirmed this independently: 5 independent solid-black gpt-image outputs share a near-identical fixed signature (pairwise residual correlation **0.92**, avg-template retains 97% energy), so the watermark/carrier IS strongly present and consistent on flat content — but the carrier frequencies extracted from it do NOT discriminate real content (carrier-to-random ratio: cleaned 1.86 > watermarked 1.53; a non-gpt-image image scored highest at 3.67). The signature drowns in content texture. Net: a perfectly consistent solid-color signature still yields no real-content pixel detector with magnitude/carrier methods. A corpus discrimination test (2026-05-24, `scripts/synthid_pixel_probe.py`, raw zero-mean residual NCC) independently re-confirms this: at matched resolution, SynthID positives do NOT cluster apart from negatives (within-Gemini 0.07; at 1024 px pos-vs-neg >= pos-vs-pos). The only high correlations were near-duplicate *content* (5 ChatGPT renders of one prompt at ~0.92, while a distinct ChatGPT image scored ~0 against them) — content, not a carrier. The probe is solid-fills-only and EXPERIMENTAL/DIAGNOSTIC; do not use it on real content. - **External AI-vs-real classifier models are out of scope (decided 2026-05-24).** Generic HuggingFace detectors (`Organika/sdxl-detector` Swin Transformer, `umm-maybe/AI-image-detector`, and fine-tunes) exist and report ~0.98 on their *own* SDXL-vs-real validation sets, but they are per-generator and the model cards themselves note degraded accuracy off-distribution; they are untested on gpt-image / Gemini Nano Banana (the metadata-stripped surfaces we care about), and our own light SDXL pass would likely defeat them the same way it defeats SynthID. Detection here stays local + signal-based (metadata + visible sparkle); do not add a bundled classifier dependency. - **SynthID v2 vs default pipeline:** the SDXL-based default profile (since May 2026) defeats SynthID v2. **Verified end-to-end (May 2026):** local SDXL run on a Gemini 3 Pro output, checked via the Gemini app's "Verify with SynthID" feature, returned "no SynthID watermark detected". Also confirmed against **OpenAI's** SynthID (2026-05-23): a fresh ChatGPT/gpt-image output read "SynthID detected" on openai.com/verify before the local SDXL run and "SynthID not detected" after (corpus regression chain: pos `4ef377bd` -> cleaned `47188e88`). The same configuration is used in raiw-app production (`fal-ai/fast-sdxl` at native ~1024 px, strength 0.05, steps 50). SD-1.5 dreamshaper at 768 px was previously the default and does NOT defeat v2 — verified empirically against the same feature (strength 0.04, 0.10, and elastic warp α∈{5,8} all flagged positive). That SD-1.5 path was removed; only `default` (SDXL) and `ctrlregen` profiles remain. diff --git a/scripts/synthid_pixel_probe.py b/scripts/synthid_pixel_probe.py new file mode 100644 index 0000000..1cefa6f --- /dev/null +++ b/scripts/synthid_pixel_probe.py @@ -0,0 +1,149 @@ +"""SynthID pixel-carrier probe -- EXPERIMENTAL / DIAGNOSTIC ONLY. + +There is no local detector of the SynthID pixel watermark on real content: the +carrier drowns in scene texture (see CLAUDE.md, confirmed repeatedly). This +probe is meaningful ONLY on **solid-color fills**, where the per-pixel deviation +from the image mean is essentially the watermark carrier (almost all the +variance). It answers two controlled questions, neither of which is a +real-content detector: + + consistency IMAGES... + Mean pairwise normalized cross-correlation (NCC) of the carriers across + independent solid fills from one model, vs a random baseline. Genuine + SynthID positives share a fixed carrier, so they correlate well above + random (the pilot saw ~0.92 on gpt-image black fills); clean fills don't. + + removal --pos P... --cleaned C... + Build a carrier template from the positive fills, then compare how the + positives and the pipeline-cleaned fills correlate to it. If removal + worked, the cleaned correlation collapses toward the random baseline -- + pixel-domain evidence that the pipeline destroys the carrier, not just the + C2PA metadata. + +Do NOT run this on real-content images; the numbers are uninformative there. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import click +import numpy as np +from PIL import Image +from rich.console import Console + +if TYPE_CHECKING: + from numpy.typing import NDArray + +log = logging.getLogger(__name__) +console = Console() + + +def load_gray(path: str) -> NDArray[np.float64]: + """Load an image as a float64 grayscale array (mean of RGB channels).""" + with Image.open(path) as img: + return np.asarray(img.convert("RGB"), dtype=np.float64).mean(axis=2) + + +def carrier(gray: NDArray[np.float64]) -> NDArray[np.float64]: + """Zero-mean, unit-norm residual of a solid-fill image -- its carrier. + + Returns a flattened unit-norm vector for NCC comparison. A perfectly flat + image (std 0, e.g. a synthetic #000000 reference) has no carrier and yields + an all-zero vector, which correlates to 0 with everything. + """ + residual = gray - float(gray.mean()) + norm = float(np.linalg.norm(residual)) + if norm == 0.0: + return residual.ravel() + return (residual / norm).ravel() + + +def ncc(a: NDArray[np.float64], b: NDArray[np.float64]) -> float: + """Normalized cross-correlation of two carriers (unit-norm zero-mean vectors).""" + if a.shape != b.shape or a.size == 0: + return 0.0 + return float(np.dot(a, b)) + + +def mean_pairwise_ncc(carriers: list[NDArray[np.float64]]) -> float: + """Average NCC over all distinct carrier pairs; 0.0 if fewer than two.""" + scores = [ncc(carriers[i], carriers[j]) for i in range(len(carriers)) for j in range(i + 1, len(carriers))] + return float(np.mean(scores)) if scores else 0.0 + + +def template(carriers: list[NDArray[np.float64]]) -> NDArray[np.float64]: + """Average carrier, renormalized to unit norm (the shared-pattern estimate).""" + avg = np.mean(carriers, axis=0) + norm = float(np.linalg.norm(avg)) + return avg / norm if norm else avg + + +def random_baseline(shape: tuple[int, ...], n: int, *, seed: int = 0) -> float: + """Mean pairwise NCC of ``n`` random-noise carriers of ``shape`` (~0).""" + rng = np.random.default_rng(seed) + noise = [carrier(rng.standard_normal(shape)) for _ in range(max(n, 2))] + return mean_pairwise_ncc(noise) + + +def _load_carriers(paths: tuple[str, ...]) -> list[NDArray[np.float64]]: + """Load carriers for same-shaped images; warn and skip mismatched shapes.""" + grays = [(p, load_gray(p)) for p in paths] + shape = grays[0][1].shape + carriers: list[NDArray[np.float64]] = [] + for p, g in grays: + if g.shape != shape: + console.print(f" [yellow]skip[/] {p}: shape {g.shape} != {shape}") + continue + carriers.append(carrier(g)) + return carriers + + +@click.group() +def cli() -> None: + """SynthID pixel-carrier probe (solid-color fills only).""" + + +@cli.command() +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True)) +def consistency(images: tuple[str, ...]) -> None: + """Mean pairwise carrier NCC across solid fills, vs the random baseline.""" + carriers = _load_carriers(images) + if len(carriers) < 2: + console.print("[red]Need at least two same-shaped images.[/]") + raise SystemExit(1) + observed = mean_pairwise_ncc(carriers) + baseline = random_baseline(carriers[0].shape, len(carriers)) + console.print(f" carriers: {len(carriers)}") + console.print(f" mean pairwise NCC: [bold]{observed:.3f}[/]") + console.print(f" random baseline: {baseline:.3f}") + verdict = "shared carrier present" if observed > 0.3 else "no shared carrier (within noise)" + console.print(f" verdict: [bold]{verdict}[/]") + + +@cli.command() +@click.option("--pos", "pos", multiple=True, required=True, type=click.Path(exists=True), help="Positive solid fills.") +@click.option( + "--cleaned", "cleaned", multiple=True, required=True, type=click.Path(exists=True), help="Pipeline-cleaned fills." +) +def removal(pos: tuple[str, ...], cleaned: tuple[str, ...]) -> None: + """Does the pipeline drop the carrier correlation toward the random baseline?""" + pos_carriers = _load_carriers(pos) + cleaned_carriers = _load_carriers(cleaned) + if not pos_carriers or not cleaned_carriers: + console.print("[red]Need at least one positive and one cleaned fill of matching shape.[/]") + raise SystemExit(1) + tmpl = template(pos_carriers) + pos_corr = float(np.mean([ncc(c, tmpl) for c in pos_carriers])) + cleaned_corr = float(np.mean([ncc(c, tmpl) for c in cleaned_carriers])) + baseline = random_baseline(tmpl.shape, max(len(cleaned_carriers), 2)) + console.print(f" positive->template NCC: [bold]{pos_corr:.3f}[/]") + console.print(f" cleaned->template NCC: [bold]{cleaned_corr:.3f}[/]") + console.print(f" random baseline: {baseline:.3f}") + effective = cleaned_corr < pos_corr / 2 + console.print(f" verdict: [bold]{'carrier attenuated' if effective else 'carrier survives'}[/]") + + +if __name__ == "__main__": + cli() diff --git a/tests/test_synthid_pixel_probe.py b/tests/test_synthid_pixel_probe.py new file mode 100644 index 0000000..dea6141 --- /dev/null +++ b/tests/test_synthid_pixel_probe.py @@ -0,0 +1,104 @@ +"""Tests for the SynthID pixel-carrier probe. + +The probe's logic is validated synthetically: a fixed carrier injected into +uniform fills must correlate strongly, random noise must not, and simulated +removal (dropping the carrier) must collapse the correlation. No real SynthID +fills or GPU are needed. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +from click.testing import CliRunner +from PIL import Image + +# scripts/ is not an installed package; add it to the path for import. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_pixel_probe as probe + +_SHAPE = (64, 64) + + +def _fixed_carrier(seed: int = 7) -> np.ndarray: + """A reproducible 2-D carrier pattern (the stand-in for the SynthID signal).""" + rng = np.random.default_rng(seed) + return rng.standard_normal(_SHAPE) + + +def _fill_with_carrier(base: float, pattern: np.ndarray, noise_seed: int, noise: float = 0.05) -> np.ndarray: + """A solid fill at brightness ``base`` carrying ``pattern`` plus light noise.""" + rng = np.random.default_rng(noise_seed) + return base + pattern + noise * rng.standard_normal(_SHAPE) + + +class TestCarrier: + def test_flat_image_has_zero_carrier(self): + # A perfectly uniform fill (std 0, like the synthetic refs) carries nothing. + flat = np.zeros(_SHAPE) + assert not np.any(probe.carrier(flat)) + + def test_carrier_is_unit_norm(self): + c = probe.carrier(_fixed_carrier() + 100.0) + assert np.isclose(np.linalg.norm(c), 1.0) + + def test_ncc_self_is_one(self): + c = probe.carrier(_fixed_carrier()) + assert np.isclose(probe.ncc(c, c), 1.0) + + def test_ncc_mismatched_shape_is_zero(self): + a = probe.carrier(np.random.default_rng(1).standard_normal((8, 8))) + b = probe.carrier(np.random.default_rng(2).standard_normal((16, 16))) + assert probe.ncc(a, b) == 0.0 + + +class TestConsistency: + def test_shared_carrier_correlates_high(self): + pattern = _fixed_carrier() + carriers = [probe.carrier(_fill_with_carrier(b, pattern, s)) for s, b in enumerate((10, 60, 120, 200))] + assert probe.mean_pairwise_ncc(carriers) > 0.8 + + def test_random_fills_near_zero(self): + rng = np.random.default_rng(0) + carriers = [probe.carrier(rng.standard_normal(_SHAPE)) for _ in range(5)] + assert abs(probe.mean_pairwise_ncc(carriers)) < 0.2 + + def test_random_baseline_near_zero(self): + assert abs(probe.random_baseline(_SHAPE, 6)) < 0.2 + + +class TestRemoval: + def test_removed_carrier_collapses_correlation(self): + pattern = _fixed_carrier() + pos = [probe.carrier(_fill_with_carrier(b, pattern, s)) for s, b in enumerate((20, 90, 160))] + tmpl = probe.template(pos) + # "Cleaned" fills keep the base + noise but lose the shared pattern. + rng = np.random.default_rng(99) + cleaned = [probe.carrier(b + 0.05 * rng.standard_normal(_SHAPE)) for b in (20, 90, 160)] + pos_corr = float(np.mean([probe.ncc(c, tmpl) for c in pos])) + cleaned_corr = float(np.mean([probe.ncc(c, tmpl) for c in cleaned])) + assert pos_corr > 0.8 + assert cleaned_corr < 0.2 + + +class TestCli: + def _solid_fill_png(self, tmp_path: Path, name: str, base: int, pattern: np.ndarray, seed: int) -> Path: + arr = np.clip(_fill_with_carrier(base, pattern, seed), 0, 255).astype(np.uint8) + path = tmp_path / name + Image.fromarray(np.stack([arr] * 3, axis=2)).save(path) + return path + + def test_consistency_command_runs(self, tmp_path: Path): + pattern = _fixed_carrier() + paths = [str(self._solid_fill_png(tmp_path, f"f{i}.png", b, pattern, i)) for i, b in enumerate((40, 120, 200))] + result = CliRunner().invoke(probe.cli, ["consistency", *paths]) + assert result.exit_code == 0, result.output + assert "mean pairwise NCC" in result.output + + def test_consistency_needs_two_images(self, tmp_path: Path): + path = str(self._solid_fill_png(tmp_path, "only.png", 100, _fixed_carrier(), 0)) + result = CliRunner().invoke(probe.cli, ["consistency", path]) + assert result.exit_code != 0