From 8a648794ad49843bd971fe33e9c871873712ce54 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Tue, 11 Aug 2026 11:09:55 -0700 Subject: [PATCH] Add calibrated SynthID pixel detector --- README.md | 18 + docs/cli.md | 22 +- docs/controlnet-removal-pipeline-research.md | 19 +- docs/installation.md | 2 +- docs/known-limitations.md | 17 +- docs/module-internals.md | 51 +- docs/python-api.md | 21 + docs/supported-signals.md | 24 +- docs/synthid-detector-removal-plan.md | 326 +++++++++++- docs/synthid.md | 113 ++++- docs/watermarking-landscape.md | 7 +- scripts/synthid_oracle_batch.py | 467 ++++++++++++++++++ scripts/synthid_periodic_tile.py | 44 +- scripts/synthid_periodic_tile_ablation.py | 288 +++++++++++ scripts/synthid_periodic_tile_probe.py | 22 +- scripts/synthid_phase_carrier.py | 37 +- src/remove_ai_watermarks/__init__.py | 8 + src/remove_ai_watermarks/assets/__init__.py | 2 +- .../assets/synthid_periodic_tile_2048_v1.npz | Bin 0 -> 7501 bytes src/remove_ai_watermarks/cli.py | 70 ++- src/remove_ai_watermarks/identify.py | 60 ++- src/remove_ai_watermarks/metadata.py | 5 +- src/remove_ai_watermarks/synthid_detector.py | 258 ++++++++++ tests/test_api.py | 4 + tests/test_cli.py | 22 + tests/test_identify.py | 26 + tests/test_synthid_detector.py | 217 ++++++++ tests/test_synthid_oracle_batch.py | 212 ++++++++ tests/test_synthid_periodic_tile_ablation.py | 122 +++++ tests/test_synthid_periodic_tile_probe.py | 19 + tests/test_synthid_phase_carrier.py | 18 + tests/test_synthid_tile_attack.py | 19 +- 32 files changed, 2394 insertions(+), 146 deletions(-) create mode 100644 scripts/synthid_oracle_batch.py create mode 100644 scripts/synthid_periodic_tile_ablation.py create mode 100644 src/remove_ai_watermarks/assets/synthid_periodic_tile_2048_v1.npz create mode 100644 src/remove_ai_watermarks/synthid_detector.py create mode 100644 tests/test_synthid_detector.py create mode 100644 tests/test_synthid_oracle_batch.py create mode 100644 tests/test_synthid_periodic_tile_ablation.py diff --git a/README.md b/README.md index 03030ac..4918ea1 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ removal. | Goal | Command | GPU | | --- | --- | --- | | Find provenance signals and watermarks | `identify` | No | +| Detect the SynthID pixel carrier in the calibrated image-size range | `detect-synthid` | No | | Remove known visible AI marks | `visible` | No | | Erase a region you select | `erase` | No | | Strip AI metadata | `metadata` | No | @@ -48,6 +49,7 @@ removal. | Need | Install | | --- | --- | | Metadata inspection and stripping | `remove-ai-watermarks` | +| Local SynthID carrier detection in the calibrated size range | `remove-ai-watermarks[pixels]` | | Visible detection and removal | `remove-ai-watermarks[visible]` | | Visible video processing | `remove-ai-watermarks[video]` | | Video SynthID removal | `remove-ai-watermarks[video,diffusion]` | @@ -74,6 +76,19 @@ Inspect an image: remove-ai-watermarks identify image.png ``` +Detect the supported SynthID pixel carrier after installing the pixel runtime: + +```bash +uv tool install --force "remove-ai-watermarks[pixels]" +remove-ai-watermarks detect-synthid image.png +``` + +This detector is positive-only and limited to one measured carrier family in +the [calibrated image-size range](docs/synthid.md#32-how-our-tool-detects-the-supported-carrier). +It expects the recovered carrier at its measured 16-pixel sampling scale; +arbitrary spatial resampling is not registered. `not_detected` or `unsupported` +is not a clean-image guarantee. + For visible watermark removal, install the pixel dependencies: ```bash @@ -333,6 +348,9 @@ import remove_ai_watermarks as raiw result, removed = raiw.remove_visible("watermarked.png", "clean.png") print(removed) +synthid = raiw.detect_synthid("image.png") +print(synthid.status, synthid.score) + provenance = raiw.identify_video("input.mp4") report = raiw.inspect_video_metadata("input.mp4") complete = raiw.remove_video_all("input.mp4", "clean.mp4") diff --git a/docs/cli.md b/docs/cli.md index 270ba39..072cdca 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -14,6 +14,7 @@ defaults. This page focuses on choosing the right command. | Command or signal | Required installation | | --- | --- | | `metadata` and metadata-only `identify` | Default package | +| `detect-synthid` and the calibrated-size SynthID pixel signal in `identify` | `remove-ai-watermarks[pixels]` | | Visible signals in `identify` | `remove-ai-watermarks[visible]` (`pixels` is the minimal runtime) | | Open DWT-DCT signals in `identify` | `remove-ai-watermarks[detect]` | | Adobe TrustMark signals in `identify` | `remove-ai-watermarks[trustmark]` | @@ -54,8 +55,25 @@ Metadata only inspection: remove-ai-watermarks identify image.png --no-visible ``` -Despite the historical option name, `--no-visible` skips both visible and open -invisible pixel detectors. Metadata inspection still runs. +Despite the historical option name, `--no-visible` skips all pixel detectors, +including the supported SynthID carrier, visible marks, open DWT-DCT, and +TrustMark. Metadata inspection still runs. + +## Detect the supported SynthID pixel carrier + +```bash +remove-ai-watermarks detect-synthid image.png +remove-ai-watermarks detect-synthid image.png --json +``` + +The command returns one of `detected`, `not_detected`, or `unsupported`. The +runtime detector covers one frozen periodic carrier family in the +[calibrated image-size range](synthid.md#32-how-our-tool-detects-the-supported-carrier) +and needs the `pixels` extra. It never resizes the input and does not register +a carrier whose sampling period changed through arbitrary spatial resampling. +It is positive-only: `not_detected` means the score stayed below this detector's +threshold, while `unsupported` means the image geometry is outside its scope. +Neither result proves that another SynthID epoch or payload is absent. ## Remove known visible marks diff --git a/docs/controlnet-removal-pipeline-research.md b/docs/controlnet-removal-pipeline-research.md index d750310..afe9ea9 100644 --- a/docs/controlnet-removal-pipeline-research.md +++ b/docs/controlnet-removal-pipeline-research.md @@ -79,8 +79,9 @@ enough yet barely touches flat fills. So the survivors FLIP by content type — choice alone does not guarantee removal. **2. Seed non-determinism near threshold.** img2img uses a random seed unless `--seed` -is passed, and there is no local SynthID detector to self-verify. The bracelet survived -controlnet @0.15 in one run and CLEARED @0.15 in another (same pipeline+strength+res). +is passed, and these geometries are outside the current local detector's scope. +The bracelet survived controlnet @0.15 in one run and CLEARED @0.15 in another +(same pipeline+strength+res). So a single clean run does NOT establish a strength as safe — characterizing a reliable floor needs a seed-repeatability sweep (N runs, varied seed), not one pass. @@ -144,8 +145,8 @@ Gemini app; the two payloads are vendor-specific and never cross-checked): but never recovered original identity precisely — every setting traded one problem for another. See `docs/synthid-robust-identity-research-2026-06-08.md` "Empirical follow-up" for the full sweep. -- **No local SynthID detector exists** → the service can't self-verify; bake in strength - margin and periodic oracle spot-checks. +- **No applicable local detector exists for these geometries** → the service + can't self-verify; bake in strength margin and periodic oracle spot-checks. - **Lesson:** visual-quality / face-identity recovery does NOT prove removal — only the oracle does, across MULTIPLE content types; never conclude from a partial result (the photoreal-only data first read as "controlnet shields, default removes"; the flat-graphic @@ -298,8 +299,9 @@ attention-slicing; ~1-2 min/image, so a coarse sweep is a sub-hour background ru is needed ONLY for the separate native-large-Gemini (2816 px) case, which OOMs even without a ControlNet (that requires a GPU task). The genuine external dependency is NOT compute but the **manual SynthID oracle**: -there is no local SynthID detector, so removal is verified by hand in the Gemini app -("Verify with SynthID") per image, regardless of where the diffusion runs. +these geometries are outside the current local detector's scope, so removal is +verified by hand in the Gemini app ("Verify with SynthID") per image, regardless +of where the diffusion runs. Runner: **`scripts/controlnet_sweep.py`** (built 2026-06-02) implements exactly this sweep — SDXL base 1.0 + an SDXL-native ControlNet img2img, one output per (control x strength x scale) @@ -441,8 +443,9 @@ shielding risk; defer to a v2 after the single-canny path is dialed in. **Hard caveat:** every change that increases preservation (higher scale, denser canny, fuller window, softer edges) marginally REDUCES effective regeneration and so raises the chance the watermark -survives -- exactly the shielding failure mode. There is no local SynthID detector, so each tuning -change must be re-confirmed on the oracle. These are img2img-context recommendations derived from +survives -- exactly the shielding failure mode. These geometries are outside the +current local detector's scope, so each tuning change must be re-confirmed on +the oracle. These are img2img-context recommendations derived from generation-context sources plus our own measurements; treat the playbook as hypotheses to verify, not settled defaults. diff --git a/docs/installation.md b/docs/installation.md index f709c3d..1669e38 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -94,7 +94,7 @@ application actually uses: | Extra | Capability | Automatically includes | Torch or model download | | --- | --- | --- | --- | -| `pixels` | Shared BGR array and image-processing runtime | NumPy, headless OpenCV | No | +| `pixels` | Shared BGR runtime and calibrated-size SynthID carrier detection | NumPy, headless OpenCV | No | | `heif` | HEIC, HEIF, and AVIF pixel decoding | pillow-heif | No | | `visible` | Visible mark detection, OpenCV inpainting, and manual erasing | `pixels` | No | | `video` | Visible video identification/removal and timestamp preservation | `visible`, PyAV | No | diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 4d48ace..71e1cef 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -64,12 +64,19 @@ identity or exact texture, and each then runs the same face stage. `qwen-zimage` is the higher fidelity of the two. Both are large, slow, and may still alter small text or difficult faces. -### Removal cannot be verified locally for proprietary SynthID +### Local SynthID detection covers one image-carrier family -The project has no public local SynthID pixel decoder. It recognizes presence -from supported provenance metadata: Google AI C2PA under Google's all-media -policy, or current OpenAI C2PA with an explicit watermark action. A missing -provenance signal is not a negative pixel verdict. +Google does not publish the proprietary SynthID payload decoder. The project +ships a positive-only detector for one measured periodic image carrier in a +calibrated image-size range, plus provenance-based recognition +from Google AI C2PA or current OpenAI C2PA with an explicit watermark action. +It does not cover images outside that size range, crop, strong JPEG compression, +video, or future carrier epochs. Arbitrary dimensions inside the range are +supported only while the recovered carrier retains its measured 16-pixel +sampling lattice. A spatial resize changes that period; the runtime does not +yet search fractional periods or infer the pre-resize geometry. A `not_detected` +or `unsupported` result is not a negative universal verdict, and removal still +requires the matching provider oracle for confirmation. For important outputs: diff --git a/docs/module-internals.md b/docs/module-internals.md index 4ceb242..4044b87 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -451,7 +451,56 @@ metadata extraction from verdict logic: both extractors reach the same answer. It did not, and the record path silently reported no SynthID for images the file path flagged. - `identify` preserves the path-based API and adds the optional registered - visible-mark and open invisible-watermark decoders after extraction. + visible-mark, open invisible-watermark, and supported SynthID carrier + detectors after extraction. + +### SynthID periodic carrier detector + +[`synthid_detector.py`](../src/remove_ai_watermarks/synthid_detector.py) is the +runtime form of the frozen 2048x2048 periodic-tile experiment. It folds a +Gaussian high-pass residual modulo 16x16 within a calibrated pixel-count range +and compares the normalized RGB tile with the bundled float64 template +`assets/synthid_periodic_tile_2048_v1.npz`. Exact multiples use the original +reshape-and-mean path; other sizes use count-correct modulo folding, +without resize. Channels are filtered and folded sequentially, and partial edge +blocks are accumulated without a full-frame padding buffer so the 18-megapixel +ceiling does not require multiple three-channel float workspaces. The model hash +is pinned by a test, and the unchanged operating threshold is +`0.17357069773071196`. + +The direct API returns `detected`, `not_detected`, or `unsupported`; the last is +distinct because no resize is performed. Support is based on a calibrated range +of 1,000,000 through 18,000,000 decoded pixels. The frozen threshold accepted +none of 5,000 public COCO views balanced across every observed target geometry, +and none of a separate 5,000-view challenge over 256 generated geometries +covering every pair of modulo-16 edge remainders. The original 2048x2048 +verdicts and exact scores remain unchanged. Runtime matches do not attribute a +provider. `identify` adds only positive matches as high-confidence +evidence and never turns a local negative into a clean verdict. + +Arbitrary geometry is not the same as arbitrary spatial resampling. On a +stratified 80-image fixed-positive sample, one-step resizes at seven nonidentity +scales from 0.5 through 1.5 reduced the unchanged 16x16 detector from 80 accepted +sources to zero at every scale. Restoring the original geometry recovered 58-80 +sources, showing that the carrier period scaled with the pixels. A discovery +bank that scaled the template to integer periods 8, 10, 12, 14, 18, 20, and 24 +was promising: a threshold frozen above 3,000 resized COCO controls accepted no +view in a 2,000-control final partition and accepted 672 of 800 source-disjoint +provider positives. It also stayed below threshold on the tracked OpenAI and +Adobe controls. The branch remains research-only because noninteger periods at +scales 0.8, 0.9, 1.1, 1.2, and 1.333 collapsed, while separate per-period +thresholds accepted five final controls. The runtime therefore keeps only the +fixed 16-pixel lattice. + +A follow-up fractional-period probe sampled the 30 strongest template harmonics +over a continuous 7.5-24.5 period range. The correct period appeared within +0.05 pixels among the top three candidates for 58 of 60 transformed positives. +Testing nine neighboring reconstructed geometries then recovered 44 of 60 at +the native threshold, compared with an upper bound of 48 when the true source +geometry was supplied. The complete search still failed its small frozen +control split: a threshold above 250 development controls accepted two of 150 +final controls. Multiplying the canonical score by spectral-period confidence +also accepted two. This branch is not a calibrated runtime fallback. ### Portable metadata record diff --git a/docs/python-api.md b/docs/python-api.md index 8839c7d..a30dbb2 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -12,6 +12,27 @@ path that still runs on CPU and combines `video` and `diffusion`. Add `heif` independently when path-based pixel APIs must decode HEIC, HEIF, or AVIF. See the complete [feature-extra matrix](installation.md#feature-extras). +## Detect the supported SynthID carrier + +Install `remove-ai-watermarks[pixels]`, then call the lazy top-level API: + +```python +import remove_ai_watermarks as raiw + +result = raiw.detect_synthid("input.png") +print(result.status) # "detected" | "not_detected" | "unsupported" +print(result.score) # float for a supported image size, otherwise None +print(result.threshold) # frozen operating point +``` + +The detector is positive-only and covers one measured periodic carrier family +in the [calibrated image-size range](synthid.md#32-how-our-tool-detects-the-supported-carrier). +Arbitrary dimensions are accepted inside that range, but arbitrary spatial +resampling can change the carrier period and is not registered. `not_detected` +means only that this model did not find its carrier; `unsupported` is kept +separate from a negative result. Neither is proof that the image contains no +SynthID watermark. + ## Remove visible marks Install `remove-ai-watermarks[visible]` before using the visible-removal API. diff --git a/docs/supported-signals.md b/docs/supported-signals.md index 752fb40..c84214a 100644 --- a/docs/supported-signals.md +++ b/docs/supported-signals.md @@ -79,6 +79,7 @@ The inspection and stripping code handles signals in these groups: - xAI and Grok EXIF signature fields; - Samsung AI editing markers; - Hugging Face job metadata; +- one positive-only SynthID periodic pixel carrier in a calibrated image-size range; - open Stable Diffusion style DWT-DCT watermarks with the `detect` extra; - Adobe TrustMark with the `trustmark` extra. @@ -125,12 +126,21 @@ Current pipeline values, both CUDA-only: The `controlnet`, `sdxl`, `qwen` and `default` values were removed. A retired name is rejected at parse time rather than remapped onto a surviving profile. -SynthID does not have a public local pixel decoder in this project. The tool -recognizes presence from supported provenance: Google AI C2PA under Google's -all-media watermark policy, and current OpenAI C2PA carrying an explicit -`c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action does not -assert SynthID. After provenance metadata is removed, a local negative result -is still inconclusive. +Google does not publish the SynthID payload decoder. This project ships a +positive-only detector for one measured periodic image-carrier family in a +calibrated image-size range, available through `detect-synthid` +and the default pixel pass in `identify` when the `pixels` extra is installed. +The unchanged fixed threshold accepted none of the public COCO views in both +an observed-geometry challenge and a generated-geometry challenge covering all +modulo-16 edge cases. Arbitrary dimensions in the calibrated range are accepted, +but the input must retain the measured 16-pixel carrier scale: arbitrary spatial +resampling is not registered. The detector does not attribute a provider locally. + +The tool also recognizes presence from supported provenance: Google AI C2PA +under Google's all-media watermark policy, and current OpenAI C2PA carrying an +explicit `c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action +does not assert SynthID. A pixel result of `not_detected` or `unsupported` +remains inconclusive for other sizes, epochs, codecs, and payloads. For MP4, MOV, and M4V, `video invisible` or the explicit `video all --invisible` option can regenerate the video through a VAE and strip @@ -149,7 +159,7 @@ not a universal clean verdict. | Provider or family | Visible | Invisible path | Metadata or provenance | | --- | --- | --- | --- | -| Google Gemini | Sparkle | Diffusion regeneration for SynthID | C2PA and related source signals | +| Google Gemini | Sparkle | Local positive-only calibrated-size detector; diffusion regeneration | C2PA and related source signals | | Google Veo video | Veo diamond and legacy text | Oracle-certified VAE removal for SynthID | C2PA and related source signals | | OpenAI image generators | None registered | Diffusion regeneration for supported invisible signals | C2PA and generator provenance | | Stable Diffusion and SDXL | None registered | Diffusion regeneration; optional open decoder | Embedded parameters and text metadata | diff --git a/docs/synthid-detector-removal-plan.md b/docs/synthid-detector-removal-plan.md index ccb08ee..1e9f377 100644 --- a/docs/synthid-detector-removal-plan.md +++ b/docs/synthid-detector-removal-plan.md @@ -1074,10 +1074,10 @@ provider-specific expert for the supported 1536x2816 carrier epoch. Identity and bounded translation views use the frozen phase and support thresholds; unsupported geometry, insufficient carrier magnitude, and ambiguous phase return `abstain`. Vendor attribution may select the expert that supplied -accepted evidence, but it must not turn an abstention into a provider label. The next -calibration gate still requires at least 3,000 native-support negatives, -same-provider oracle negatives, matched non-target solid outputs, and a new -temporal positive that influenced neither profile nor threshold. +accepted evidence, but it must not turn an abstention into a provider label. +The next calibration gate still requires at least 3,000 native-support +negatives, same-provider oracle negatives, matched non-target solid outputs, +and a new temporal positive that influenced neither profile nor threshold. ### 2026-08-10: 2048 periodic-tile detector @@ -1151,6 +1151,323 @@ family, but the two cross-source carrier matches prohibit a stronger vendor claim until an oracle distinguishes direct provider output from shared-backend output. +A pixel-space ablation then tested whether the frozen tile merely predicted the +local scores or controlled them. At twice the train-median tile norm, aligned +subtraction changed the fixed-tile decision from 29 of 30 accepted originals to +zero and the independently fitted sparse-phase decision from 27 of 30 to zero. +The median fidelity was 53.74 dB PSNR and 0.99681 SSIM. Cyclically shifting the +same tile by one row and column left one phase acceptance, while a seeded +zero-mean random tile orthogonal to the learned template left 13; their median +PSNRs were within 0.13 dB of the aligned edit. For both local representations, +the aligned edit reduced the score more than either control on all 30 paired +images, with a two-sided sign-test p-value of 1.86e-9 for each comparison. +The same aligned edit reversed both local decisions on each of the two disputed +cross-source matches, while the orthogonal control left both phase decisions +accepted. That two-item result is descriptive but makes an accidental threshold +crossing less likely; it still cannot distinguish a shared carrier from direct +provider output. + +This is the strongest local causal evidence for the 16x16 mechanism, but the +strength was selected after inspecting this locked test and is therefore +discovery-only. The shifted control also suppressed the phase representation +substantially, so local score reversal cannot certify signal removal. A matching +provider oracle must still compare untouched sources, aligned candidates, and +norm-matched controls before any removal claim. The reproducible local harness +is `scripts/synthid_periodic_tile_ablation.py`; its report records input and +model hashes without writing derivative images. + +### 2026-08-10: confirmatory oracle-batch readiness + +The corpus audit found no new 2048x2048 positive collected after the tile rule +and subtraction strength were frozen. Existing test images cannot be renamed as +a temporal holdout, so the confirmatory oracle batch remains blocked on new +provider output at the supported native geometry. + +The batch mechanics are ready in `scripts/synthid_oracle_batch.py`. For each new +source, it preregisters five requests in a fixed order: untouched source, +lossless re-encode, aligned subtraction, cyclic-shifted subtraction, and an +orthogonal norm-matched tile. It refuses to write media inside the repository, +records artifact and decoded-pixel hashes, pins both model artifacts and the +matching provider, and detects any post-registration byte change. Oracle results +live in a separate manifest-bound file so the preregistration remains immutable; +the evaluator requires complete ordered results, verbatim responses, +timezone-aware timestamps, and separate SynthID and C2PA verdicts. + +A causal success requires the source, re-encode, shifted, and orthogonal views +all to remain `detected` while only the aligned view becomes `not_detected`. +`Indeterminate` and refusal remain indeterminate. A one-source dry run on an old +test positive verified all five artifacts and the local scoring path, but is +pipeline validation only and contributes no confirmatory evidence. + +### 2026-08-10: first confirmatory oracle attempt + +Four new 2048x2048 Gemini outputs were generated after the local rule and +strength were frozen, then registered as a 20-request batch. The manifest and +all 20 artifacts passed hash verification before submission. The first source +group produced these ordered results: + +| Role | Local tile/phase decision | Provider result | +| --- | --- | --- | +| source | detected / detected | SynthID indeterminate; Google C2PA present | +| re-encode control | detected / detected | Google AI signal detected | +| aligned subtraction | detected / not detected | Google AI signal detected | +| shifted control | detected / detected | Google AI signal detected | +| orthogonal control | detected / detected | refused: verification quota exceeded | + +The quota response instructed the account to wait 24 hours. No adaptive retry +was made, and the remaining 15 preregistered requests were not submitted. The +batch cannot receive its preregistered aggregate verdict until complete ordered +results exist. + +Even incomplete, this attempt rejects the frozen removal recipe for the first +temporal source: aligned subtraction crossed the phase threshold but remained +above the tile threshold and did not clear the provider oracle. The remaining +sources can still measure transfer and disagreement between the two local +experts, but they cannot turn this first aligned result into a universal +pixel-only removal success. + +### 2026-08-10: positive-only runtime detector + +Removal is deferred while the transferred tile signal is exposed as a bounded +detector. `src/remove_ai_watermarks/synthid_detector.py` loads the frozen model +as a bundled pickle-free runtime asset and returns `detected`, `not_detected`, +or `unsupported` without resizing input. The direct API and +`detect-synthid` CLI initially covered only native 2048x2048 images; `identify` +consumes a positive match as high-confidence evidence but never treats a +negative or unsupported result as proof of absence. + +The operating point and model are unchanged from the locked experiment. The +detector accepted all four post-freeze Gemini outputs, including the source +whose frozen phase score missed, while retaining the prior 29-of-30 locked-test +sensitivity and zero accepted validation/test negatives. The unresolved two +external-source matches remain the reason a runtime positive names the carrier +but does not attribute a provider. + +### 2026-08-10: calibrated native-geometry extension + +The frozen 16x16 template and threshold transfer across the common native +geometries in the provider-positive evidence. Non-divisible image dimensions +use direct modulo folding with per-cell sample counts; divisible dimensions, +including 2048x2048, retain the original numerical path. All prior 2048x2048 +records were replayed and matched exactly, including floating-point scores. + +The runtime registry was expanded only after the unchanged threshold accepted +none of 60,000 public COCO challenge views, 3,000 at each of 20 target +geometries. This is a geometry extension of the same positive-only carrier +expert, not an OpenAI pixel detector or a proprietary payload decoder. + +### 2026-08-11: calibrated image-size range + +The fixed template was then evaluated on every provider-positive image in the +local evidence set rather than only its common geometries. It accepted 3,928 of +4,698 images across 757 exact geometries. Sensitivity separated by pixel count: +1,987 of 2,021 images at or above three megapixels crossed the threshold, while +1,940 of 2,672 images from one through two megapixels did. This establishes a +carrier-family boundary, not universal SynthID recall: explicit C2PA watermark +actions also occur below threshold, and three strong carriers use a different +cyclic phase. + +Two public-image geometry challenges tested whether geometry itself creates +false matches. The first balanced 5,000 COCO images across all 757 observed +geometries, with every geometry present in both development and final partitions; +the maximum fixed score was 0.12549 and none crossed the unchanged 0.17357 +threshold. The second transformed the same 5,000 source images at 256 generated +geometries from one through 18 megapixels, covering every width/height remainder +pair modulo 16. Its maximum was 0.16387 and again none crossed the threshold. +Runtime support therefore uses that challenged pixel-count interval instead of +an enumerated geometry registry, still without resizing. + +A cyclic-registration branch was rejected after the public-image challenge. +At a low threshold it recovered 130 fixed-phase misses but false-fired on +tracked OpenAI and Adobe controls. Raising the threshold above all development +controls recovered only 23 in-range provider positives while adding 84 +cross-source matches beyond the fixed detector. Three strong shifted Google +carriers remain a documented miss rather than weakening the runtime rule. + +Top-level source labels were not treated as watermark-negative truth. Of 237 +non-Google-labeled corpus images crossing the fixed threshold, 81 retained +SynthID provenance. High-scoring Adobe manifests explicitly named Gemini Flash +as the remote provider, and perceptual matching found direct Google derivatives +under editing or relabeling categories. The remaining source labels cannot prove +absence of the carrier, so they constrain provider attribution but do not justify +raising the pixel threshold. + +### 2026-08-11: spatial-resampling boundary + +Arbitrary decoded geometry did not imply arbitrary resize robustness. A +stratified sample of 80 fixed-positive provider images was rescaled in memory at +0.5, 0.625, 0.75, 0.875, 1.125, 1.25, and 1.5. The unchanged 16x16 detector +accepted none of the 80 images at every nonidentity scale. Restoring each view +to its source dimensions recovered 58-80, which isolates the failure to the +scaled sampling lattice rather than simple information destruction. + +Scaling the template and modulo fold to the corresponding integer periods 8, +10, 12, 14, 18, 20, and 24 recovered 73-80 of the discovery images at the +native threshold. Searching all eight periods raised control scores, so a +single threshold was frozen above the maximum of 3,000 deterministically +resized COCO development views. It accepted none of 2,000 final controls and +672 of 800 source-disjoint provider positives, with 77-88 accepted at each +scale. Six tracked OpenAI and Adobe controls remained below the frozen threshold. + +This did not generalize to arbitrary scaling. Factors 0.8, 0.9, 1.1, 1.2, and +1.333 create noninteger carrier periods; rounding the period accepted zero of +80 images in the fixed-phase branch at every factor, and cyclic registration +accepted only 1-16. Period-specific thresholds improved sensitivity but accepted +five of the 2,000 final controls. The integer-period bank is therefore retained +as a research result, not added to the runtime. A universal resize branch needs +fractional-period registration or reliable reconstruction of the pre-resize +geometry, followed by a new multiple-search calibration. + +A discovery follow-up tested that fractional-period direction with the 30 +strongest template harmonics. A continuous 7.5-24.5 spectral search placed the +true period within 0.05 pixels among its top three candidates for 58 of 60 +transformed positives. Searching nine neighboring integer reconstruction +geometries recovered 44 of 60 at the native threshold; even the true source +geometry recovered only 48, so resampling damage sets a separate upper bound. +The full search then failed a frozen 250-development/150-final COCO challenge: +the calibration threshold accepted two final controls. A preselected product +of canonical correlation and spectral confidence increased positive +acceptance from 49 to 52 of 60 at its development threshold but accepted the +same two final controls. This fractional FFT baseline is rejected for runtime +use; better period estimation alone does not solve multiple-search specificity. + +### 2026-08-10: OpenAI periodic-carrier challenge + +The OpenAI track repeated the Google carrier method without runtime provenance +features. C2PA selected the research labels, but every candidate score consumed +only decoded RGB pixels. The temporally split corpus contained 4,615 OpenAI +training positives, 1,280 validation positives, 1,364 development-test +positives, 8,668 difficult external-source controls, and 3,000 COCO controls. +The development test is not called a locked holdout because earlier +representations had already exposed it. + +High-pass residuals were folded modulo 8x8, 16x16, and 32x32 and normalized as +in the Google fixed-tile expert. A single 16x16 OpenAI template accepted only +one of 1,280 validation positives at the threshold above all 2,287 validation +controls. A 16-template geometry bank improved that to 12 positives but +accepted one of 3,372 development-test controls. The corresponding +development-test sensitivity was 9 of 1,364. The frozen Google template did +not transfer: at its unchanged threshold it accepted 14 validation positives +and 572 validation controls. + +The apparent native-size signal was a confound. At 1024x1536, 91.6% of the +learned template energy projected onto a 2x2 periodic lattice, compared with +5.6% for the frozen Google template. Firefly, China AIGC-labeled, local +diffusion, and unidentified hosted-generator controls reached correlations of +0.92-0.98 against the uncorrected OpenAI template. This is operational evidence +for a shared decoder or upscaler lattice, not an OpenAI watermark carrier. + +Projecting the 2x2 nuisance lattice out of every 16x16 fold isolated a weaker +epoch-specific residual. The 1024x1536 expert accepted 5 of 55 validation and +10 of 51 development-test positives while accepting none of the full +validation or development-test controls. Nine of the ten development-test +matches occurred from July 12 through July 14; later dates contributed only +one of 27. The 1536x1024 expert accepted 6 of 37 validation and 2 of 51 +development-test positives with no development-test controls. Amplitude, +active-support, channel-consensus, payload-invariant power, and spherical +multi-template branches did not improve this frontier without false positives. + +The OpenAI branch therefore remains research-only and is not added to the +runtime detector. The next valid training epoch requires same-content +OpenAI-oracle `detected` and `not_detected` pairs, or provider-authorized decoder +labels, followed by a fresh temporal holdout. Source provenance alone cannot +distinguish a true watermark carrier from the measured generator lattice. + +### 2026-08-10: OpenAI content-dependent decoder challenge + +The periodic result did not exhaust the neural-watermark hypothesis. OpenAI's +ChatGPT Images 2.0 system card describes its watermark as content-specific, and +the SynthID-Image paper says the encoder is content-dependent by design. The +paper also separates watermark detection from payload recovery and evaluates +the external SynthID-O model at its preferred 512x512 input size. These facts +support a learned full-field decoder, but they do not establish that OpenAI and +Google deploy identical model weights, payloads, or pixel spectra. + +A second corpus audit found 261 same-provider candidate controls. Their signed +provenance contained `created` and `converted` actions without the +`c2pa.watermarked.unbound` action, but a subsequent software-agent audit showed +that they do not all come from the positive generation epoch. Only 118 name +`gpt-image 2.0`: 97 training, 16 validation, and five development-test images. +Another 108 name an earlier version and 35 do not expose a usable version. The +full 261-image split remains 217/26/18. These are useful confound controls, but +even the exact-version subset is action-derived rather than a provider-oracle +negative label. + +Two pixel-only native-resolution experts were trained with the mixed-version +control set. A +local residual CNN accepted 132 of 1,280 validation positives and 149 of 1,364 +development-test positives, with zero same-provider or COCO controls in both +splits and two of 1,373 difficult development-test controls. A separate +spectral, autocorrelation, bit-plane, and residual-error expert accepted 175 +validation and 167 development-test positives with no accepted controls. A +validation-selected linear combination accepted 243 positives in each split, +with no accepted validation or development-test control. The zero-error +development-test bound across 3,391 controls is 0.0883% at one-sided 95% +confidence, but the same-provider subset contains only 18 images and therefore +does not independently establish provider specificity. + +The exact-version audit did not invalidate the native score counts, but it +weakened their attribution. None of the 16 validation or five development-test +`gpt-image 2.0` controls crossed the residual, forensic, ensemble, canonical, +or JPEG-equalized thresholds. That is directionally useful, but five temporal +test controls cannot establish deployment specificity. Perceptual-hash and +local CLIP searches compared all 118 exact-version controls with 7,259 +positives. They recovered a few near-content matches but no clean before/after +watermark pair. Sub-pixel affine alignment left content, edge, and photometric +changes in the residual, while 8x8, 16x16, and 32x32 folded residual +correlations varied in sign and magnitude. Those candidates cannot be used as +causal watermark differences. + +Transformation challenges rejected the combined native expert as a SynthID +decoder. Of its 243 accepted development-test positives, 121 remained accepted +after a 5% crop round trip, six after a 75% resize round trip, and none after a +JPEG-95 round trip. No transformed same-provider control was accepted. The +codec fragility is inconsistent with treating the score as the documented +robust watermark without further evidence. + +The 512x512 branch also failed the held-out specificity gate. Its selected +model accepted 238 validation and 194 development-test positives, but also +accepted two of 18 same-provider, four of 1,373 difficult, and nine of 2,000 +COCO development-test controls. Finally, a CNN trained and evaluated only +after applying the same JPEG-95 round trip to every positive and control +accepted eight validation positives and six development-test positives while +accepting five of 2,000 development-test COCO controls. Equalizing the codec +therefore removed nearly all of the apparent signal instead of revealing a +robust hidden decoder feature. + +An independent three-CNN OpenAI surrogate was also evaluated from its +published weights, without adopting its code or claims. At the published 0.5 +threshold its ensemble marked 82.0% of COCO validation controls and 83.4% of +COCO development-test controls as watermarked. It also marked 56.3% of +difficult development-test controls and 15 of 16 exact-version OpenAI +validation controls, compared with 87.3% of OpenAI validation positives. The +repository does not publish its training corpus or logs, and its implementation +applies supposedly shared pair augmentation independently. This result rejects +the model as a SynthID decoder and demonstrates why same-content-looking +regenerations are not sufficient negative labels. + +A final transform-augmented full-field branch tested the closest practical +approximation to the documented neural decoder. A 512x512 CNN consumed both +RGB and a fixed high-pass residual and trained with random crop/resize, +JPEG 70-100, horizontal flip, and mild color transforms. Training sampling +balanced watermarked positives, difficult controls, and the exact-version +OpenAI candidate controls. Selection required a score above every control for +both native and JPEG-95 validation views. All four fixed training epochs +accepted zero of 500 selection positives. On the full validation set the +frozen model accepted zero of 1,280 OpenAI positives in native, JPEG-95, +75%-resize, and 5%-crop views, while accepting up to two unrelated controls. +On development test it accepted zero of 1,364 OpenAI positives in native, +JPEG-95, and resize views. The crop view accepted one OpenAI positive and one +of 1,373 difficult controls. Transform augmentation therefore did not recover +a usable weaker OpenAI signal. + +The measured OpenAI-native separation is best classified as an export or +rendering noiseprint, not a validated SynthID detector. None of these models is +added to runtime. Transform-augmented training has now also failed. A +defensible next epoch needs provider-authorized labels or clean same-content +before/after watermark pairs, plus a fresh temporal holdout that has not +influenced representation or threshold selection. + ## Decision record The program has four possible honest outcomes per provider: @@ -1184,5 +1501,6 @@ Only after this milestone should implementation of D3 and D4 begin. ## Primary sources - OpenAI, [Content provenance](https://developers.openai.com/api/docs/guides/content-provenance). +- OpenAI, [ChatGPT Images 2.0 system card](https://deploymentsafety.openai.com/chatgpt-images-2-0/automated-evaluations-and-adversarial-testing). - Google, [Verify AI-generated images, videos, and audio](https://support.google.com/gemini/answer/16722517?hl=en). - Gowal et al., [SynthID-Image: Image watermarking at internet scale](https://arxiv.org/abs/2510.09263). diff --git a/docs/synthid.md b/docs/synthid.md index b92cc16..90af5e1 100644 --- a/docs/synthid.md +++ b/docs/synthid.md @@ -308,6 +308,30 @@ limited JPEG and crop robustness. The pickle-free research implementation is `scripts/synthid_periodic_tile_probe.py`; exact evidence and caveats are in the [`2048 periodic-tile experiment`](synthid-detector-removal-plan.md#2026-08-10-2048-periodic-tile-detector). +An aligned-subtraction ablation strengthened the mechanism finding without +clearing the oracle gate. At a discovery-selected amplitude, it reversed both +the fixed-tile and independently fitted phase decisions on all 30 test images +at a median 53.74 dB PSNR and 0.99681 SSIM. The aligned edit reduced both scores +more than cyclic-shifted and orthogonal random tile controls on every paired +image. A shifted tile nevertheless suppressed the phase score enough to leave +only one accepted image, so these local reversals remain surrogate evidence, +not verified SynthID removal. The exact controls and caveats are recorded in +the linked experiment section. + +The immutable oracle-batch and result evaluator are implemented. On 2026-08-10, +four new 2048x2048 Gemini images were generated after the rule was frozen and +registered as a 20-request confirmatory batch. The first source group exhausted +the account's verification quota after five requests. The untouched source +returned Google C2PA Content Credentials without a separate SynthID verdict; +the lossless re-encode, aligned subtraction, and cyclic-shifted control all +still returned a Google AI signal. The orthogonal control was refused because +the quota had been exceeded, and the remaining 15 requests were not submitted. +This incomplete run is already negative evidence for the frozen pixel-only +recipe: the aligned candidate cleared the phase detector but remained positive +under the tile detector and the provider oracle. The local carrier expert +therefore ships only as a positive-only, exact-geometry detector; it is not a +universal SynthID detector or a remover. + A controlled study (June 2026, clean v0.8.6 with text/face protection OFF, native resolution on this repo's default SDXL pipeline) measured the minimum img2img strength that removes the SynthID pixel watermark, verified per image on @@ -402,7 +426,7 @@ diffusion prior." ## 3. Detectability and verifier access -### 3.1 No public local decoder +### 3.1 No public payload decoder The SynthID decoder is proprietary and not released: @@ -411,8 +435,8 @@ The SynthID decoder is proprietary and not released: > available to trusted testers." > -- Gowal et al., arXiv:2510.09263 -There are no released decoder weights and no reproducible algorithm for local -detection. Google provides verification in Gemini and a limited SynthID Detector +There are no released payload-decoder weights or public algorithm. Google +provides verification in Gemini and a limited SynthID Detector portal. OpenAI now documents a synchronous Content Provenance API whose image response contains separate C2PA and SynthID outcomes. That API is a remote, OpenAI-scoped verifier, not a local decoder. Its documentation also says not to @@ -425,12 +449,73 @@ Google's SynthID Detector service is: > professionals" on a waitlist > -- deepmind.google/models/synthid/ -The external variant SynthID-O is available "through partnerships" only. Our -tool does not currently detect SynthID pixels locally. The gated research path -for determining whether that can change is documented in +The external variant SynthID-O is available "through partnerships" only. This +project instead detects one empirically recovered periodic carrier family in a +calibrated image-size range. It does not decode the proprietary payload or +generalize that local signal to unsupported sizes, +codecs, video, or future epochs. The evidence and gates are documented in [`synthid-detector-removal-plan.md`](synthid-detector-removal-plan.md). -### 3.2 How our tool recognizes SynthID from provenance +### 3.2 How our tool detects the supported carrier + +`remove-ai-watermarks detect-synthid image.png` folds the image residual modulo +16x16 and compares it with a frozen float64 template. It evaluates only native +input, without resize. Exact-multiple dimensions retain the original folding +path; non-divisible dimensions use count-correct modulo folding. The model and +threshold remain frozen from the 2048x2048 experiment. The fixed threshold +accepted none of 5,000 public COCO views balanced across every observed target +geometry. A separate 5,000-view challenge used 256 generated geometries from one +through 18 megapixels and covered every pair of width/height remainders modulo +16; it also produced no accepted view. Runtime support is therefore the +challenged interval of 1,000,000 through 18,000,000 decoded pixels rather than +an enumerated width-by-height registry. The original 2048x2048 scores remain +exactly unchanged. + +This geometry support does not imply arbitrary resize robustness. The fixed +carrier has a 16-pixel sampling lattice. In a stratified 80-image positive +sample, direct detection fell from 80 accepted originals to zero after each of +seven nonidentity resizes from 0.5 through 1.5. Scaling the template and folding +period to matching integers recovered the signal, and a conservative threshold +above 3,000 resized COCO development controls accepted 672 of 800 source-disjoint +provider positives with no acceptance in 2,000 final controls. That branch is +not shipped: noninteger periods from ordinary scale factors collapsed, and +less conservative per-period thresholds accepted five final controls. The +runtime therefore detects arbitrary decoded dimensions only when the carrier +retains its measured 16-pixel scale. + +A positive result identifies the carrier but does not attribute a provider. +Provider identity still comes from provenance. + +The command reports `not_detected` separately from `unsupported`. Both are +inconclusive outside the measured carrier family and calibrated image-size range. + +The same modulo-folding method has been tested separately on a large, +temporally split OpenAI-labeled corpus. Its strongest native-size template was +dominated by a generic 2x2 generator lattice that also appeared in multiple +non-OpenAI controls. Removing that nuisance component left a sparse, +time-limited signal with inadequate sensitivity. OpenAI pixel detection is +therefore not part of the runtime expert. + +Learned residual, forensic, ensemble, and canonical 512x512 representations +were also tested against 261 same-provider candidate controls. A later +software-agent audit found that only 118 of those controls explicitly name +`gpt-image 2.0`; the rest come from earlier or unknown versions. The strongest +native ensemble accepted 243 of 1,364 development-test positives with no +accepted controls, but accepted none of those positives after a JPEG-95 round +trip. A model trained after equalizing every image through JPEG-95 accepted +only six development-test positives and five COCO controls. The apparent +native signal is therefore treated as an export noiseprint rather than a +validated robust watermark. A local CLIP search over the exact-version subset +found no clean same-content before/after pair, and a published third-party CNN +surrogate mislabeled 83.4% of held-out COCO controls at its stated threshold. +A transform-augmented 512x512 RGB-plus-residual CNN then accepted zero of 1,364 +development-test positives in native, JPEG-95, and resize views; its crop view +accepted one positive and one difficult control. Detailed counts and rejected +alternatives are in +the [`OpenAI periodic-carrier challenge`](synthid-detector-removal-plan.md#2026-08-10-openai-periodic-carrier-challenge) +and [`OpenAI content-dependent decoder challenge`](synthid-detector-removal-plan.md#2026-08-10-openai-content-dependent-decoder-challenge). + +### 3.3 How our tool recognizes SynthID from provenance We recognize SynthID indirectly from supported C2PA evidence; this is not a pixel watermark decode. Google states that all media generated by its tools is @@ -451,7 +536,7 @@ This is why: - A quiet `identify` output is not proof that SynthID was removed -- it only means the metadata signal is gone. -### 3.3 Oracle scope: each vendor detects only their own +### 3.4 Oracle scope: each vendor detects only their own OpenAI's current Content Provenance API documentation says it checks supported OpenAI signals and is not a general-purpose AI detector. Google's current Gemini @@ -470,7 +555,7 @@ A Google-SynthID image reads clean on openai.com/verify. An OpenAI image reads clean in the Gemini oracle. They are different payloads within the same framework. -### 3.4 Video verification and attack harness +### 3.5 Video verification and attack harness Gemini's built-in verification flow reports whether and where it detects Google SynthID in a video. This remains a proprietary oracle: invoke `@synthid`, use @@ -699,7 +784,8 @@ photoreal, `sdxl` on flat graphics, the §5.1 content-x-pipeline table), BUT on hard case (flat fills) `sdxl` is the WEAKER remover (plain img2img barely perturbs a flat region at low strength), so it needs AT LEAST controlnet's strength -- the certified floor is therefore the right floor for `sdxl` too. This is a MARGIN argument -for `sdxl`, not a separate certification (no local SynthID detector to self-verify). +for `sdxl`, not a separate certification (the tested geometries are outside the +current local detector's scope). The higher strength costs little quality where it matters, because `controlnet` is now the default pipeline, so `sdxl` is reached only via an explicit `--pipeline sdxl` (a deliberate opt-down), where over-regeneration has no faces/text to damage. @@ -810,7 +896,7 @@ random (unset) seed differed between runs. So **0.15 is the borderline floor for controlnet photoreal, not a robust guarantee**: at the threshold the same image+settings can pass or fail run-to-run. img2img runs with `seed=None` (random) unless `--seed` is passed, so a removal SERVICE gets a coin-flip near threshold and -has no local SynthID detector to self-verify. +has no applicable local SynthID detector at these geometries to self-verify. **Controlnet strength ladder on the two photoreal images (oracle, `--auto`, `--max-resolution 1536`):** @@ -825,7 +911,7 @@ has no local SynthID detector to self-verify. non-deterministic borderline); both photoreal survivors cleared at 0.20. Honest caveat: 0.20 is one confirming run WITH margin, not an N-run repeatability proof -- for a removal service, add a little more margin or validate repeatability, since -there is no local SynthID detector to self-check. **Implications:** (1) the +these geometries are outside the current local detector's scope. **Implications:** (1) the content×pipeline table above conflates a borderline/non-deterministic 0.15 result with deterministic content behavior -- the photoreal-survives-controlnet effect is solid at 0.10 but at 0.15 it is near-threshold noise; (2) for reliable removal pick @@ -910,3 +996,6 @@ reproducible verification requires a fixed seed. 8. Jiang et al. (2025). **VideoMarkBench: Benchmarking Robustness of Video Watermarking.** arXiv:2505.21620. https://arxiv.org/abs/2505.21620 + +9. OpenAI. **ChatGPT Images 2.0 system card.** + https://deploymentsafety.openai.com/chatgpt-images-2-0/automated-evaluations-and-adversarial-testing diff --git a/docs/watermarking-landscape.md b/docs/watermarking-landscape.md index 86ff339..e74f0fe 100644 --- a/docs/watermarking-landscape.md +++ b/docs/watermarking-landscape.md @@ -64,7 +64,12 @@ payloads. Removal remuxes either container through ffmpeg with stream copy. may carry IPTC metadata but no registered C2PA or pixel watermark. The open DWT-DCT decoder only applies when the producing pipeline actually ran its encoder and the carrier remains decodable. -- **Invisible but NOT locally detectable (proprietary, API/oracle only — same wall as SynthID):** Amazon Titan Image Generator + Nova Canvas (Bedrock `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), NVIDIA Cosmos (SynthID video). No local detector possible; treat like SynthID. +- **Invisible but NOT locally detectable (proprietary, API/oracle only):** + Amazon Titan Image Generator + Nova Canvas (Bedrock + `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), + and NVIDIA Cosmos (SynthID video). No public payload decoder is available; + unlike the project's calibrated-size SynthID carrier expert, these signals + have no measured local detector here. - **C2PA 2.4 "Durable Content Credentials" (April 2026; verified against the spec) raise the bar for metadata stripping.** 2.4 defines soft bindings (an invisible watermark or a content fingerprint) plus a server-side manifest repository and a new `c2pa.repository-receipt` assertion. Per the spec: "if a C2PA manifest is removed from an asset, but a copy of that manifest remains in a provenance store elsewhere, the manifest and asset may be matched using available soft bindings." So our local `metadata --remove` deletes the *embedded* manifest, but a fingerprint/watermark soft binding can still re-link the image to its manifest in a repository server-side. Stripping the file is becoming necessary-but-not-sufficient against durable provenance. (Our parsers target the stable embedded-manifest format documented in C2PA 2.1 §11; that format is unchanged in 2.4 -- the new pieces are repository/soft-binding infra, not the on-file box layout, so no parser change is implied.) Spec: https://spec.c2pa.org/specifications/specifications/2.4/specs/C2PA_Specification.html We now READ the soft-binding `alg` (`C2PA_SOFT_BINDINGS` / `soft_binding_vendors_in`) to name the forensic-watermark vendor, and locally DECODE the one open scheme, Adobe TrustMark (`trustmark_detector`); the rest (Digimarc/Imatag/Steg.AI/...) stay name-only (proprietary decoders). - **Built in the dated batch:** soft-binding vendor detection, IPTC Photo Metadata AI-disclosure fields, C2PA detection and stripping for supported diff --git a/scripts/synthid_oracle_batch.py b/scripts/synthid_oracle_batch.py new file mode 100644 index 0000000..34c5d78 --- /dev/null +++ b/scripts/synthid_oracle_batch.py @@ -0,0 +1,467 @@ +"""Build and verify an immutable confirmatory batch for a SynthID oracle. + +The batch contains a referenced untouched source plus four lossless PNG views: +an exact-pixel re-encode, aligned periodic-tile subtraction, a cyclically +shifted tile, and an orthogonal random tile. Building the batch performs no +network requests. Oracle results remain empty until a separately authorized +submission records them. +""" + +from __future__ import annotations + +import json +import logging +import math +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import click +from PIL import Image +from synthid_periodic_tile_ablation import ( + candidate_quality, + control_templates, + phase_score_pixels, + tile_score_pixels, +) +from synthid_periodic_tile_probe import PeriodicTileModel +from synthid_periodic_tile_probe import load_model as load_tile_model +from synthid_phase_carrier import PhaseCarrierModel +from synthid_phase_carrier import load_model as load_phase_model +from synthid_pixel_attack import load_rgb +from synthid_research_manifest import artifact_sha256, pixel_fingerprint +from synthid_tile_attack import subtract_tiled_template + +if TYPE_CHECKING: + import numpy as np + +log = logging.getLogger(__name__) + +ROLE_ORDER = ("source", "reencode_control", "aligned", "shifted", "orthogonal_random") +DERIVATIVE_ROLES = ROLE_ORDER[1:] +FORMAT_VERSION = 1 +SYNTHID_RESULTS = {"detected", "not_detected", "indeterminate", "refused"} +C2PA_RESULTS = {"present", "absent", "indeterminate", "unavailable"} + + +def _inside(path: Path, parent: Path) -> bool: + """Return whether resolved PATH is inside resolved PARENT.""" + try: + path.resolve().relative_to(parent.resolve()) + except ValueError: + return False + return True + + +def _json_float(value: float) -> float | None: + """Return finite VALUE or None for JSON interoperability.""" + return value if math.isfinite(value) else None + + +def _write_png(path: Path, pixels: np.ndarray) -> None: + """Write exact RGB PIXELS as a deterministic lossless PNG.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + raise ValueError(f"refusing to overwrite oracle artifact: {path}") + Image.fromarray(pixels, mode="RGB").save(path, format="PNG", compress_level=9) + + +def _score_row( + pixels: np.ndarray, + *, + source: np.ndarray, + tile_model: PeriodicTileModel, + phase_model: PhaseCarrierModel, + tile_threshold: float, + phase_threshold: float, + active_threshold: float, +) -> dict[str, object]: + """Return local frozen scores and paired fidelity for PIXELS.""" + tile_score = tile_score_pixels(pixels, tile_model) + phase_score, active_support = phase_score_pixels(pixels, phase_model) + quality = candidate_quality(source, pixels) + return { + "tile_score": tile_score, + "tile_accepted": tile_score >= tile_threshold, + "phase_score": phase_score, + "active_support": active_support, + "phase_accepted": phase_score >= phase_threshold and active_support >= active_threshold, + "residual_rms": quality["residual_rms"], + "psnr_db": _json_float(quality["psnr_db"]), + "ssim": quality["ssim"], + "changed_pixel_fraction": quality["changed_pixel_fraction"], + } + + +def _artifact_row( + path: Path, + *, + role: str, + source_id: str, + in_batch: bool, + batch_root: Path, + scores: dict[str, object], +) -> dict[str, object]: + """Return one hash-frozen manifest row for PATH.""" + pixel_sha256, width, height = pixel_fingerprint(path) + return { + "source_id": source_id, + "role": role, + "path": str(path.relative_to(batch_root) if in_batch else path.resolve()), + "in_batch": in_batch, + "artifact_sha256": artifact_sha256(path), + "pixel_sha256": pixel_sha256, + "width": width, + "height": height, + "synthid_result": None, + "c2pa_result": None, + "submitted_at": None, + **scores, + } + + +def build_batch( + sources: list[Path], + *, + output_dir: Path, + tile_model_path: Path, + phase_model_path: Path, + tile_threshold: float, + phase_threshold: float, + active_threshold: float, + strength: float, + seed: int, + provider: str, + repository_root: Path, +) -> Path: + """Build a preregistered, unsubmitted oracle batch and return its manifest.""" + if not sources: + raise ValueError("at least one source is required") + if strength <= 0.0 or not math.isfinite(strength): + raise ValueError("strength must be finite and positive") + if provider not in {"google", "openai"}: + raise ValueError("provider must be google or openai") + if _inside(output_dir, repository_root): + raise ValueError("oracle batches must be written outside the repository") + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError("output directory must not already contain files") + output_dir.mkdir(parents=True, exist_ok=True) + + tile_model = load_tile_model(tile_model_path) + phase_model = load_phase_model(phase_model_path) + if (tile_model.height, tile_model.width) != (phase_model.height, phase_model.width): + raise ValueError("tile and phase model geometries differ") + templates = control_templates(tile_model.template, seed=seed) + + rows: list[dict[str, object]] = [] + seen_source_hashes: set[str] = set() + for source_path in sources: + source_hash = artifact_sha256(source_path) + if source_hash in seen_source_hashes: + raise ValueError("duplicate source artifact") + seen_source_hashes.add(source_hash) + source = load_rgb(source_path) + if source.shape != (tile_model.height, tile_model.width, 3): + raise ValueError(f"{source_path}: geometry does not match the frozen models") + source_id = source_hash[:16] + score_args = { + "source": source, + "tile_model": tile_model, + "phase_model": phase_model, + "tile_threshold": tile_threshold, + "phase_threshold": phase_threshold, + "active_threshold": active_threshold, + } + source_scores = _score_row(source, **score_args) + rows.append( + _artifact_row( + source_path, + role="source", + source_id=source_id, + in_batch=False, + batch_root=output_dir, + scores=source_scores, + ) + ) + for role in DERIVATIVE_ROLES: + variant = ( + source + if role == "reencode_control" + else subtract_tiled_template( + source, + templates[role] * tile_model.expected_norm, + strength=strength, + ) + ) + output_path = output_dir / source_id / f"{role}.png" + _write_png(output_path, variant) + rows.append( + _artifact_row( + output_path, + role=role, + source_id=source_id, + in_batch=True, + batch_root=output_dir, + scores=(source_scores if role == "reencode_control" else _score_row(variant, **score_args)), + ) + ) + + manifest = { + "format_version": FORMAT_VERSION, + "status": "preregistered_unsubmitted", + "source_count": len(sources), + "request_count": len(rows), + "request_order": ROLE_ORDER, + "provider": provider, + "decision_rule": ( + "Count causal success only when source, reencode_control, shifted, and " + "orthogonal_random are detected in the matching provider SynthID oracle, " + "aligned is not_detected, and indeterminate is never treated as negative." + ), + "strength": strength, + "seed": seed, + "tile_threshold": tile_threshold, + "phase_threshold": phase_threshold, + "active_threshold": active_threshold, + "tile_model": str(tile_model_path.resolve()), + "tile_model_sha256": artifact_sha256(tile_model_path), + "phase_model": str(phase_model_path.resolve()), + "phase_model_sha256": artifact_sha256(phase_model_path), + "rows": rows, + } + manifest_path = output_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, allow_nan=False) + "\n", encoding="utf-8") + manifest_hash = artifact_sha256(manifest_path) + (output_dir / "manifest.sha256").write_text(manifest_hash + " manifest.json\n", encoding="utf-8") + results_template = { + "format_version": FORMAT_VERSION, + "manifest_sha256": manifest_hash, + "provider": provider, + "rows": [ + { + "source_id": row["source_id"], + "role": row["role"], + "artifact_sha256": row["artifact_sha256"], + "synthid_result": None, + "c2pa_result": None, + "raw_response": None, + "submitted_at": None, + } + for row in rows + ], + } + (output_dir / "results-template.json").write_text( + json.dumps(results_template, indent=2) + "\n", + encoding="utf-8", + ) + verify_batch(manifest_path, repository_root=repository_root) + return manifest_path + + +def verify_batch(manifest_path: Path, *, repository_root: Path) -> dict[str, object]: + """Verify manifest, model, source, and derivative hashes without mutation.""" + batch_root = manifest_path.parent + if _inside(batch_root, repository_root): + raise ValueError("oracle batches must remain outside the repository") + digest_path = batch_root / "manifest.sha256" + expected_manifest_hash = digest_path.read_text(encoding="utf-8").split()[0] + if artifact_sha256(manifest_path) != expected_manifest_hash: + raise ValueError("manifest hash mismatch") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("format_version") != FORMAT_VERSION: + raise ValueError("unsupported oracle-batch manifest version") + rows = manifest.get("rows") + if not isinstance(rows, list) or len(rows) != manifest.get("request_count"): + raise ValueError("manifest request count mismatch") + if artifact_sha256(Path(manifest["tile_model"])) != manifest["tile_model_sha256"]: + raise ValueError("tile model hash mismatch") + if artifact_sha256(Path(manifest["phase_model"])) != manifest["phase_model_sha256"]: + raise ValueError("phase model hash mismatch") + + groups: dict[str, list[str]] = {} + artifact_hashes: set[str] = set() + for row in rows: + source_id = str(row["source_id"]) + groups.setdefault(source_id, []).append(str(row["role"])) + path = batch_root / str(row["path"]) if row["in_batch"] else Path(str(row["path"])) + if artifact_sha256(path) != row["artifact_sha256"]: + raise ValueError(f"artifact hash mismatch for {source_id}/{row['role']}") + pixel_sha256, width, height = pixel_fingerprint(path) + if (pixel_sha256, width, height) != (row["pixel_sha256"], row["width"], row["height"]): + raise ValueError(f"pixel fingerprint mismatch for {source_id}/{row['role']}") + artifact_hashes.add(str(row["artifact_sha256"])) + if any(tuple(roles) != ROLE_ORDER for roles in groups.values()): + raise ValueError("each source must have the fixed request-role order") + if len(groups) != manifest.get("source_count"): + raise ValueError("manifest source count mismatch") + if len(artifact_hashes) < len(rows) - len(groups): + raise ValueError("unexpected duplicate derivative artifacts") + template = json.loads((batch_root / "results-template.json").read_text(encoding="utf-8")) + expected_identities = [(row["source_id"], row["role"], row["artifact_sha256"]) for row in rows] + template_identities = [(row["source_id"], row["role"], row["artifact_sha256"]) for row in template.get("rows", [])] + if template.get("manifest_sha256") != expected_manifest_hash or template.get("provider") != manifest["provider"]: + raise ValueError("results template does not identify the preregistered batch") + if template_identities != expected_identities: + raise ValueError("results template row identities differ from the manifest") + return manifest + + +def _parse_submitted_at(value: object) -> None: + """Reject timestamps that are absent or not timezone-aware ISO-8601.""" + if not isinstance(value, str): + raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp") from error + if parsed.tzinfo is None: + raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp") + + +def evaluate_results( + manifest_path: Path, + results_path: Path, + *, + repository_root: Path, +) -> dict[str, object]: + """Validate a complete result file and apply the preregistered decision rule.""" + manifest = verify_batch(manifest_path, repository_root=repository_root) + results = json.loads(results_path.read_text(encoding="utf-8")) + manifest_hash = artifact_sha256(manifest_path) + if results.get("format_version") != FORMAT_VERSION: + raise ValueError("unsupported oracle-results version") + if results.get("manifest_sha256") != manifest_hash or results.get("provider") != manifest["provider"]: + raise ValueError("results do not identify the preregistered batch") + manifest_rows = manifest["rows"] + result_rows = results.get("rows") + if not isinstance(result_rows, list) or len(result_rows) != len(manifest_rows): + raise ValueError("oracle results must cover every preregistered request") + + grouped: dict[str, dict[str, str]] = {} + for expected, result in zip(manifest_rows, result_rows, strict=True): + identity = (result.get("source_id"), result.get("role"), result.get("artifact_sha256")) + expected_identity = (expected["source_id"], expected["role"], expected["artifact_sha256"]) + if identity != expected_identity: + raise ValueError("oracle result order or artifact identity differs from the manifest") + synthid_result = result.get("synthid_result") + c2pa_result = result.get("c2pa_result") + if synthid_result not in SYNTHID_RESULTS: + raise ValueError("invalid or missing SynthID result") + if c2pa_result not in C2PA_RESULTS: + raise ValueError("invalid or missing C2PA result") + if not isinstance(result.get("raw_response"), str) or not result["raw_response"].strip(): + raise ValueError("raw_response must preserve the nonempty verbatim oracle result") + _parse_submitted_at(result.get("submitted_at")) + grouped.setdefault(str(expected["source_id"]), {})[str(expected["role"])] = str(synthid_result) + + source_results: list[dict[str, str]] = [] + for source_id, roles in grouped.items(): + if tuple(roles) != ROLE_ORDER: + raise ValueError("oracle results do not preserve the fixed role order") + values = set(roles.values()) + if values & {"indeterminate", "refused"}: + verdict = "indeterminate" + elif ( + roles["source"] == "detected" + and roles["reencode_control"] == "detected" + and roles["shifted"] == "detected" + and roles["orthogonal_random"] == "detected" + and roles["aligned"] == "not_detected" + ): + verdict = "causal_success" + elif any(roles[role] != "detected" for role in ("source", "reencode_control", "shifted", "orthogonal_random")): + verdict = "control_failed" + else: + verdict = "aligned_still_detected" + source_results.append({"source_id": source_id, "verdict": verdict}) + counts = { + verdict: sum(row["verdict"] == verdict for row in source_results) + for verdict in ("causal_success", "aligned_still_detected", "control_failed", "indeterminate") + } + return { + "format_version": FORMAT_VERSION, + "manifest_sha256": manifest_hash, + "provider": manifest["provider"], + "source_count": manifest["source_count"], + "counts": counts, + "sources": source_results, + } + + +@click.group() +def main() -> None: + """Build and verify an immutable confirmatory oracle batch.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + + +@main.command("build") +@click.argument("tile_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("phase_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("sources", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--output-dir", type=click.Path(file_okay=False, path_type=Path), required=True) +@click.option("--tile-threshold", type=float, required=True) +@click.option("--phase-threshold", type=float, required=True) +@click.option("--active-threshold", type=float, required=True) +@click.option("--strength", type=click.FloatRange(min=0.0, min_open=True), default=2.0, show_default=True) +@click.option("--seed", type=int, default=20260810, show_default=True) +@click.option("--provider", type=click.Choice(["google", "openai"]), required=True) +def build_command( + tile_model_path: Path, + phase_model_path: Path, + sources: tuple[Path, ...], + output_dir: Path, + tile_threshold: float, + phase_threshold: float, + active_threshold: float, + strength: float, + seed: int, + provider: str, +) -> None: + """Build a frozen batch from exact-geometry SOURCES.""" + manifest = build_batch( + list(sources), + output_dir=output_dir, + tile_model_path=tile_model_path, + phase_model_path=phase_model_path, + tile_threshold=tile_threshold, + phase_threshold=phase_threshold, + active_threshold=active_threshold, + strength=strength, + seed=seed, + provider=provider, + repository_root=Path(__file__).resolve().parent.parent, + ) + log.info("Wrote preregistered oracle batch: %s", manifest) + + +@main.command("verify") +@click.argument("manifest_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +def verify_command(manifest_path: Path) -> None: + """Verify an existing batch without changing it.""" + manifest = verify_batch(manifest_path, repository_root=Path(__file__).resolve().parent.parent) + log.info("Verified %d immutable oracle requests", manifest["request_count"]) + + +@main.command("evaluate") +@click.argument("manifest_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("results_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def evaluate_command(manifest_path: Path, results_path: Path, report_out: Path) -> None: + """Validate RESULTS_PATH and write the preregistered batch verdict.""" + repository_root = Path(__file__).resolve().parent.parent + if _inside(report_out, repository_root): + raise click.BadParameter("oracle result reports must be written outside the repository") + report = evaluate_results( + manifest_path, + results_path, + repository_root=repository_root, + ) + report_out.parent.mkdir(parents=True, exist_ok=True) + if report_out.exists(): + raise click.BadParameter("refusing to overwrite an oracle result report") + report_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + log.info("Wrote oracle batch verdict: %s", report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_periodic_tile.py b/scripts/synthid_periodic_tile.py index 1e77b42..147d922 100644 --- a/scripts/synthid_periodic_tile.py +++ b/scripts/synthid_periodic_tile.py @@ -1,50 +1,12 @@ -"""Shared periodic-residual helpers for SynthID research probes.""" +"""Compatibility imports for shared periodic-residual helpers.""" from __future__ import annotations -import cv2 import numpy as np +from remove_ai_watermarks.synthid_detector import fold_residual_template, unit_tile -def fold_residual_template( - pixels: np.ndarray, - *, - tile_height: int, - tile_width: int, - denoise_sigma: float, -) -> np.ndarray: - """Estimate a zero-mean periodic residual template by modulo folding.""" - if pixels.ndim != 3 or pixels.shape[2] != 3: - raise ValueError("pixels must have shape (height, width, 3)") - if tile_height < 1 or tile_width < 1 or denoise_sigma <= 0.0: - raise ValueError("tile dimensions and denoise sigma must be positive") - height, width = pixels.shape[:2] - if height % tile_height != 0 or width % tile_width != 0: - raise ValueError("image geometry must be divisible by the tile geometry") - source = pixels.astype(np.float32) - denoised = cv2.GaussianBlur( - source, - (0, 0), - sigmaX=denoise_sigma, - sigmaY=denoise_sigma, - borderType=cv2.BORDER_REFLECT_101, - ) - residual = source - denoised - repeats_y = height // tile_height - repeats_x = width // tile_width - folded = residual.reshape(repeats_y, tile_height, repeats_x, tile_width, 3).mean( - axis=(0, 2), - dtype=np.float64, - ) - return folded - np.mean(folded, axis=(0, 1), keepdims=True) - - -def unit_tile(tile: np.ndarray) -> tuple[np.ndarray, float]: - """Return TILE normalized by its L2 norm and the original norm.""" - norm = float(np.linalg.norm(tile)) - if norm == 0.0: - return np.zeros_like(tile, dtype=np.float64), 0.0 - return np.asarray(tile, dtype=np.float64) / norm, norm +__all__ = ["cyclic_tile_correlations", "fold_residual_template", "unit_tile"] def cyclic_tile_correlations(template: np.ndarray, tile: np.ndarray) -> np.ndarray: diff --git a/scripts/synthid_periodic_tile_ablation.py b/scripts/synthid_periodic_tile_ablation.py new file mode 100644 index 0000000..15a47b6 --- /dev/null +++ b/scripts/synthid_periodic_tile_ablation.py @@ -0,0 +1,288 @@ +"""Measure whether a frozen periodic tile causally controls local carrier scores. + +This harness compares subtraction of the learned tile with cyclically shifted +and orthogonal random tiles of the same norm. It measures local research +detectors only. A score reversal is not evidence that a provider oracle would +stop detecting SynthID. +""" + +from __future__ import annotations + +import json +import logging +import math +from collections import defaultdict +from pathlib import Path + +import click +import numpy as np +from synthid_periodic_tile import unit_tile +from synthid_periodic_tile_probe import PeriodicTileModel +from synthid_periodic_tile_probe import load_model as load_tile_model +from synthid_periodic_tile_probe import score_pixels as score_tile_pixels +from synthid_phase_carrier import PhaseCarrierModel, score_pixels +from synthid_phase_carrier import load_model as load_phase_model +from synthid_pixel_attack import load_rgb, measure +from synthid_research_manifest import artifact_sha256 +from synthid_tile_attack import parse_positive_floats, subtract_tiled_template + +log = logging.getLogger(__name__) + + +def exact_sign_test(negative: int, positive: int) -> float: + """Return an exact two-sided sign-test p-value after excluding ties.""" + count = negative + positive + if count == 0: + return 1.0 + tail = sum(math.comb(count, index) for index in range(min(negative, positive) + 1)) / 2**count + return min(1.0, 2.0 * tail) + + +def control_templates(template: np.ndarray, *, seed: int) -> dict[str, np.ndarray]: + """Return aligned, shifted, and norm-matched orthogonal control tiles.""" + rng = np.random.default_rng(seed) + random_tile = rng.normal(size=template.shape) + random_tile -= np.mean(random_tile, axis=(0, 1), keepdims=True) + random_tile -= np.sum(random_tile * template) * template + random_tile, norm = unit_tile(random_tile) + if norm == 0.0 or abs(float(np.sum(random_tile * template))) > 1e-12: + raise ValueError("could not construct an orthogonal random control") + return { + "aligned": template, + "shifted": np.roll(template, shift=(1, 1), axis=(0, 1)), + "orthogonal_random": random_tile, + } + + +def phase_score_pixels(pixels: np.ndarray, model: PhaseCarrierModel) -> tuple[float, float]: + """Return the unregistered phase score and active support for PIXELS.""" + result = score_pixels(pixels, model) + return result.score, result.active_weight_fraction + + +def tile_score_pixels(pixels: np.ndarray, model: PeriodicTileModel) -> float: + """Return the fixed-phase periodic-tile score for PIXELS.""" + return score_tile_pixels(pixels, model).score + + +def summarize(values: list[float]) -> dict[str, float]: + """Return bounded descriptive statistics for VALUES.""" + return { + "minimum": float(np.min(values)), + "median": float(np.median(values)), + "maximum": float(np.max(values)), + } + + +def direction_summary(values: list[float]) -> dict[str, float | int]: + """Return direction counts and a two-sided sign test for VALUES.""" + negative = sum(value < 0.0 for value in values) + positive = sum(value > 0.0 for value in values) + return { + "negative": negative, + "positive": positive, + "ties": len(values) - negative - positive, + "two_sided_sign_p": exact_sign_test(negative, positive), + } + + +def candidate_quality(reference: np.ndarray, candidate: np.ndarray) -> dict[str, float]: + """Return paired fidelity metrics for one equal-geometry candidate.""" + measurement = measure(reference, candidate, name="candidate", path=Path("")) + return { + "residual_rms": measurement.residual_rms, + "psnr_db": measurement.psnr_db, + "ssim": measurement.ssim, + "changed_pixel_fraction": measurement.changed_pixel_fraction, + } + + +def run_ablation( + sources: list[Path], + *, + tile_model: PeriodicTileModel, + phase_model: PhaseCarrierModel, + tile_threshold: float, + phase_threshold: float, + active_threshold: float, + strengths: tuple[float, ...], + phase_strength: float, + seed: int, +) -> dict[str, object]: + """Evaluate aligned subtraction and controls on exact-geometry SOURCES.""" + if not sources: + raise ValueError("at least one source is required") + if phase_strength not in strengths: + raise ValueError("phase strength must be one of the swept strengths") + if (tile_model.height, tile_model.width) != (phase_model.height, phase_model.width): + raise ValueError("tile and phase model geometries differ") + if not all(np.isfinite(value) for value in (tile_threshold, phase_threshold, active_threshold)): + raise ValueError("thresholds must be finite") + + templates = control_templates(tile_model.template, seed=seed) + rows: list[dict[str, object]] = [] + for source_path in sources: + source = load_rgb(source_path) + if source.shape != (tile_model.height, tile_model.width, 3): + raise ValueError(f"{source_path}: geometry does not match the models") + source_hash = artifact_sha256(source_path) + original_tile = tile_score_pixels(source, tile_model) + original_phase, original_support = phase_score_pixels(source, phase_model) + for strength in strengths: + for control, template in templates.items(): + candidate = subtract_tiled_template( + source, + template * tile_model.expected_norm, + strength=strength, + ) + tile_score = tile_score_pixels(candidate, tile_model) + row: dict[str, object] = { + "path": str(source_path), + "artifact_sha256": source_hash, + "control": control, + "strength": strength, + "original_tile_score": original_tile, + "tile_score": tile_score, + "tile_delta": tile_score - original_tile, + "tile_accepted": tile_score >= tile_threshold, + "original_phase_score": original_phase, + "original_active_support": original_support, + } + if strength == phase_strength: + phase_score, active_support = phase_score_pixels(candidate, phase_model) + row.update( + { + **candidate_quality(source, candidate), + "phase_score": phase_score, + "active_support": active_support, + "phase_delta": phase_score - original_phase, + "phase_accepted": phase_score >= phase_threshold and active_support >= active_threshold, + } + ) + rows.append(row) + + grouped: dict[tuple[str, float], list[dict[str, object]]] = defaultdict(list) + for row in rows: + grouped[(str(row["control"]), float(row["strength"]))].append(row) + tile_summaries: list[dict[str, object]] = [] + for (control, strength), group in sorted(grouped.items()): + deltas = [float(row["tile_delta"]) for row in group] + tile_summaries.append( + { + "control": control, + "strength": strength, + "accepted": sum(bool(row["tile_accepted"]) for row in group), + "delta": summarize(deltas), + "direction": direction_summary(deltas), + } + ) + + selected = [row for row in rows if float(row["strength"]) == phase_strength] + phase_summaries: list[dict[str, object]] = [] + for control in templates: + group = [row for row in selected if row["control"] == control] + phase_summaries.append( + { + "control": control, + "accepted": sum(bool(row["phase_accepted"]) for row in group), + "delta": summarize([float(row["phase_delta"]) for row in group]), + "active_support": summarize([float(row["active_support"]) for row in group]), + "psnr_db": summarize([float(row["psnr_db"]) for row in group]), + "ssim": summarize([float(row["ssim"]) for row in group]), + "changed_pixel_fraction": summarize([float(row["changed_pixel_fraction"]) for row in group]), + } + ) + + paired_comparisons: list[dict[str, object]] = [] + for control in ("shifted", "orthogonal_random"): + for metric in ("tile_delta", "phase_delta"): + aligned = { + str(row["artifact_sha256"]): float(row[metric]) for row in selected if row["control"] == "aligned" + } + comparison = { + str(row["artifact_sha256"]): float(row[metric]) for row in selected if row["control"] == control + } + differences = [aligned[key] - comparison[key] for key in sorted(aligned)] + paired_comparisons.append( + { + "aligned_minus": control, + "metric": metric, + "difference": summarize(differences), + "direction": direction_summary(differences), + } + ) + + return { + "source_count": len(sources), + "tile_threshold": tile_threshold, + "phase_threshold": phase_threshold, + "active_threshold": active_threshold, + "strengths": strengths, + "phase_strength": phase_strength, + "seed": seed, + "original": { + "tile_accepted": sum( + float(row["original_tile_score"]) >= tile_threshold for row in selected if row["control"] == "aligned" + ), + "phase_accepted": sum( + float(row["original_phase_score"]) >= phase_threshold + and float(row["original_active_support"]) >= active_threshold + for row in selected + if row["control"] == "aligned" + ), + }, + "tile_summaries": tile_summaries, + "phase_summaries": phase_summaries, + "paired_comparisons": paired_comparisons, + "items": rows, + } + + +@click.command() +@click.argument("tile_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("phase_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("sources", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--tile-threshold", type=float, required=True) +@click.option("--phase-threshold", type=float, required=True) +@click.option("--active-threshold", type=float, required=True) +@click.option("--strengths", default="1,1.5,2,3,4", show_default=True) +@click.option("--phase-strength", type=click.FloatRange(min=0.0, min_open=True), default=2.0, show_default=True) +@click.option("--seed", type=int, default=20260810, show_default=True) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def main( + tile_model_path: Path, + phase_model_path: Path, + sources: tuple[Path, ...], + tile_threshold: float, + phase_threshold: float, + active_threshold: float, + strengths: str, + phase_strength: float, + seed: int, + report_out: Path, +) -> None: + """Run a fixed periodic-tile causal ablation on exact-geometry SOURCES.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + strength_values = parse_positive_floats(strengths, option_name="strengths") + report = run_ablation( + list(sources), + tile_model=load_tile_model(tile_model_path), + phase_model=load_phase_model(phase_model_path), + tile_threshold=tile_threshold, + phase_threshold=phase_threshold, + active_threshold=active_threshold, + strengths=strength_values, + phase_strength=phase_strength, + seed=seed, + ) + report["tile_model"] = str(tile_model_path) + report["tile_model_sha256"] = artifact_sha256(tile_model_path) + report["phase_model"] = str(phase_model_path) + report["phase_model_sha256"] = artifact_sha256(phase_model_path) + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + log.info("Wrote periodic-tile causal ablation: %s", report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_periodic_tile_probe.py b/scripts/synthid_periodic_tile_probe.py index 7690e5d..c33d4c9 100644 --- a/scripts/synthid_periodic_tile_probe.py +++ b/scripts/synthid_periodic_tile_probe.py @@ -92,10 +92,18 @@ def discover_model( ) -def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) -> PeriodicTileScore: - """Score PATH against MODEL, optionally searching cyclic tile shifts.""" +def score_pixels( + pixels: np.ndarray, + model: PeriodicTileModel, + *, + register: bool = False, + path: str = "", +) -> PeriodicTileScore: + """Score exact-geometry RGB PIXELS, optionally searching cyclic tile shifts.""" + if pixels.shape != (model.height, model.width, 3): + raise ValueError("pixel geometry does not match periodic-tile model") folded = fold_residual_template( - _load_rgb(path, height=model.height, width=model.width), + pixels, tile_height=model.tile_height, tile_width=model.tile_width, denoise_sigma=model.denoise_sigma, @@ -109,7 +117,7 @@ def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) score = float(np.sum(model.template * unit)) row_shift = column_shift = 0 return PeriodicTileScore( - path=str(path), + path=path, score=score, active_support=min(norm / (model.expected_norm + 1e-12), 1.0), row_shift=row_shift, @@ -118,6 +126,12 @@ def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) ) +def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) -> PeriodicTileScore: + """Score PATH against MODEL, optionally searching cyclic tile shifts.""" + pixels = _load_rgb(path, height=model.height, width=model.width) + return score_pixels(pixels, model, register=register, path=str(path)) + + def calibrate_threshold(paths: list[Path], model: PeriodicTileModel, *, register: bool = False) -> float: """Return the first float above every negative score in PATHS.""" if not paths: diff --git a/scripts/synthid_phase_carrier.py b/scripts/synthid_phase_carrier.py index 92b8cc6..4cf04c6 100644 --- a/scripts/synthid_phase_carrier.py +++ b/scripts/synthid_phase_carrier.py @@ -179,6 +179,27 @@ def _frequency_values(pixels: np.ndarray, model: PhaseCarrierModel) -> np.ndarra return extract_frequency_values(pixels, model.rows, model.columns, model.channels) +def score_pixels(pixels: np.ndarray, model: PhaseCarrierModel, *, path: str = "") -> PhaseCarrierScore: + """Score exact-geometry RGB PIXELS against MODEL.""" + if pixels.shape != (model.height, model.width, 3): + raise ValueError("pixel geometry does not match phase-carrier model") + values = _frequency_values(np.asarray(pixels, dtype=np.float64), model) + magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0) + active_weights = model.weights * magnitude_gate + active_weight = float(np.sum(active_weights)) + score = ( + 0.0 + if active_weight == 0.0 + else float(np.sum(active_weights * np.cos(np.angle(values) - model.phases)) / active_weight) + ) + return PhaseCarrierScore( + path=path, + score=score, + active_weight_fraction=active_weight, + peak_count=len(model.rows), + ) + + def score_image( path: Path, model: PhaseCarrierModel, @@ -192,21 +213,7 @@ def score_image( width=model.width, canonicalize_geometry=canonicalize_geometry, ) - values = _frequency_values(pixels, model) - magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0) - active_weights = model.weights * magnitude_gate - active_weight = float(np.sum(active_weights)) - score = ( - 0.0 - if active_weight == 0.0 - else float(np.sum(active_weights * np.cos(np.angle(values) - model.phases)) / active_weight) - ) - return PhaseCarrierScore( - path=str(path), - score=score, - active_weight_fraction=active_weight, - peak_count=len(model.rows), - ) + return score_pixels(pixels, model, path=str(path)) def score_translations( diff --git a/src/remove_ai_watermarks/__init__.py b/src/remove_ai_watermarks/__init__.py index a34122d..4ee4f8c 100644 --- a/src/remove_ai_watermarks/__init__.py +++ b/src/remove_ai_watermarks/__init__.py @@ -13,6 +13,7 @@ High-level API (lazy, so ``import remove_ai_watermarks`` stays cheap):: raiw.remove_video_metadata("in.mp4", "out.mp4") # verified metadata strip raiw.remove_video_invisible("in.mp4", "out.mp4") # oracle-certified SynthID removal raiw.remove_video_visible("in.mp4", "out.mp4") # stable visible video-mark removal + raiw.detect_synthid("in.png") # -> SynthIDDetection For a provenance verdict use the ``identify`` submodule:: @@ -39,7 +40,9 @@ __all__ = [ "InvisibleOptions", "MetadataStripIncomplete", "RemoveAllResult", + "SynthIDDetection", "__version__", + "detect_synthid", "identify_video", "inspect_video_metadata", "remove_all", @@ -64,6 +67,7 @@ if TYPE_CHECKING: remove_visible, visible_provenance, ) + from remove_ai_watermarks.synthid_detector import SynthIDDetection, detect_synthid from remove_ai_watermarks.video import ( identify_video, inspect_video_metadata, @@ -103,4 +107,8 @@ def __getattr__(name: str) -> object: from remove_ai_watermarks import video return getattr(video, name) + if name in ("SynthIDDetection", "detect_synthid"): + from remove_ai_watermarks import synthid_detector + + return getattr(synthid_detector, name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/remove_ai_watermarks/assets/__init__.py b/src/remove_ai_watermarks/assets/__init__.py index a816eee..f480ba0 100644 --- a/src/remove_ai_watermarks/assets/__init__.py +++ b/src/remove_ai_watermarks/assets/__init__.py @@ -1 +1 @@ -"""Embedded assets for visible watermark removal.""" +"""Embedded runtime assets for watermark detection and removal.""" diff --git a/src/remove_ai_watermarks/assets/synthid_periodic_tile_2048_v1.npz b/src/remove_ai_watermarks/assets/synthid_periodic_tile_2048_v1.npz new file mode 100644 index 0000000000000000000000000000000000000000..33bbf4cb6817e65edc72c919065b278abdaca6b7 GIT binary patch literal 7501 zcmchcWl&t(nudWO!JP#6;2r`&1C55@78+^XT^nf#!Gc3@4epv?jZ4tr5ZtYC4Q>N< zPiF2pXXe)Ym^*9N-fQow_3rnrZ+*Ykqa^nPkqr(G?&-r#4L9sK6Mg^3z=I=zGqrQH zF>nUCnm9UH*x9n%+PlNU;lTMntbH6(|Cr!XcAssN;`H_?bt#pme+H)NVViTWz0mXhs!*i4CXHq~4d1gjOl!I_dk}QB$jYa^^E`^MY45##`B$yH| z48N10JtQ$Vu`n}t{2DP#+SwSr|K;|9=Kx7WZ-gT>|Wf0L~WHCZKvxQQZ!!Lx6x26Lw7-ji`;fxe#MaKj$pqwNVP<3SdsjaF(U8*rx-m^N|F?~a z%z2@Y{g=*^4e}5Eodn~t9ws*S)&|Zdze9X@{ttJvXaAVf;?G41BhI@|^CW@H*)oL^ z^t3!=Cg@CRGQ>W_J_{;5@-U#ptY z_)aI;OI_(SRYh4}-7hWErY5+!_o46gw@v(tu84GBL(JZHOA9;OP@$sB$q8o{+kWWI z)s)DEs@;GyzS?v&8(&l~LRyLZ9Et<4nzx4DDSd_JJ!f}Bx14yDVAI1G0DS?dMK$H$ zHY*h;So_UPTV(C+8RKoD~jq}Tn-7hxbCW&Uqc^s83WY+Q)^ z3C}IW?>bt0-41jHE}k-uW(qSM4EfjCSv3nJceY=wM_)!$CJocYWVDD2qJmu4i7lF6 zh%NXyMZ{^7aZadi$W@KXGq&&u!mPE?eBOSihi$jFRxWVeuX)iq8soXF4J3hw^5ox` zW4BJ$4x!}vNQsF-;WCeW@%vIeRgO=bn7Pz9TY zV^!BYOWmj`tmivd!r9t4JMty>CtvB0lM=3r0$IbXDX?y{fQZ$WI!v@A#5fHy4#slv zZ?YP;E_{ zI~-h*2U;%J{ER>PO-G_{B4_o$qpUZ2)`8Qwqk4EKZ)xDxP?S)!;KI^&G~#O}k!*dp z)CL061vejCHw+HdU&FP?9TB`VU%fGe(rhf=YwaD6UP@p|)_;~8pbpqv>s=22$$B9& zq3KaXMUtl@tnC*`(4PHsTX02HnDem0<5bCuTrch$q4s-d_&u*(9J^p|SBY|Yw2eQ; zMUBUp^uP+E43Uri!fq#Gwtd0}MKfCE>4x*qMk!C;y}EYC48V;R-Wb=%OZd1}LHi`O zS8cPWQL$k=J>b=DPweT(pC8-+AG;zy8)z}8E?IkHTE}Su*&5gUppJrR;gZM0rb>rf z77nVLtQHvSEp2d^U#>N+s#vJbk}SKbSco+m(Df?nO)xdWL!q-R+;1*-m^sLo33BEt z(`IU$a2s^O_AX>TEwf1G7k5}5%6h1M(Xl1^HPDi9w?_E{6Q-AZNysh6&#vnzDO&5P|Gnp+M!21LQ3ENv`HOYtEgVJnJ-t4x?#P1r>Q-s_s+Y90Rc@ z>%?(<`=c&YWk24YYAbdDG-H8TtRajYS)cYNp{ZE5s)(jF4b!w^l+{+-a$HfFa z5mgsEcAJLYqn+q0D}5-QrXmrI;6wE&DgSPe4(sbM<_;m#;Q#zgEO4V$!^0cOSHF%{ zViEHR$uC*OCHuT{Q~gXO)QpV~NysxLqY05CG*T;lVz6zxvy9YpKhSTgHM!^xQ|+cf z>9Vk&o9rbmsMssCzw4%#hn79^^YSMTtKsW-XD>GPg_R`*X$bghDIoUK&X6`2M&eeY z3Szl1w-Gh(slrXl0AdNpg1w<7EulnY5p(dwPv6G-2vr+CEhQ1>kx?Zs)b>|H;ZbfQ zT#8=%n(n`l#>vWs!7?0#{gdqb6|6Gtn@(5tk%(vTFn0H4yN!s77Y(PaJ7;8dX$d+) zv&M*&V5~Vv&#G_t4+hrp!BJmn*{voBfZ#4s*NNkOE2+joklhGuE}dbN=bFPo7J(y)hV|CDi^c)4ZTW& zDIS=6uPZImgac^$;6#8(hy zzE0s(VZYN9nW;;sFXzE^cyrw(*UcFOk-dF;6rb-f7>lC}8p@1)*HR}FXfTPMY9{an zRV39hL>4v-QwC5b5PGC;oJ7Upn%6_?pJUqy)@#%B4(~k=B3xYvF9Dl z%NI6SK5q$~DcO)|91x?l9?KT*_M;`V26&p=Cwwz!lSeL;CWn;|=zf_0NO-A!6FuBE zF7GP56FgHxGS@=h+cS5PB|X;sHmw4*9(RvGX!X6(z)1u%@<;DH>oRzu`$L(XXnNBUbZi? zE?a50*85)SBQ4Ck!CQc-*9pYNi=LicrS<`@CcDGppeUz?MPlk+7BQ@$7|6ocU^&Z} zHrNcPccF}?=hJpfc<-_Fx3+b~K|j5U=G+@*6A1H0n(p;vZg7PE3B++nxWnOXJ^i(t zGuMP({Ro3kS~zblJ7Hwb%VqG-pP<>%AHy0ddv6>#$i4d&k8evtkvvlRS72W-sOnIA zYdHK@DJ?|O$YRVZu&9n>_xsx2lqH{`VdbnC)Vi3=o<%Jc-FDMXc)x7qp-|B?>JXf0-sqkoqh2WsWW}mGk(Q>-{KFW_!C*=6)yS zlVZY;(cG1F3PD3s3Y>VLSE9)nqjG)$n|DR>FDx#Syuw%BmIp2H9f2Zh2--VaRMCUL zcJOl3nSHz*2(;^w3iqeXt;tKT3;SK4S@Yl0`P#+8Cz;GNjrvR<27EialFHXvN}s!p zs23nVRdi|I@pQ7*`3S77uUu9^E zXpW(V1<5cl((LM(5&C`z*bUZxE)avprF8 z^TG5WU0dKHu@rB>AM2Vlep?~cqeR0XjH#ejzmb-9^kJluL#&v6r1hIlV)ko6q)YWW z6t+~n<+9m&V4)uA4V$1u2t}@#MtgIG@tw!hc@cRv686BPV?m-xQQX!neIb3zc;DMF zh9kjHz_8VQ>0&OK7H6N82D=9r9JUS%tNa`>vR+cgu|k1{B+TN`T9QjHdXGGRs_TGHE!Ewbk^m%OZ>dLZW&^Ml)s>i*SKEqmF5uRZ$UVK zcHnt;*>}L2pHGNtmf!!n4nzbXI8ezBPvY25XlB|u5B3l7_9<+`-mA10;%QucjhTIe zoDSK;Xj^pj%r6hiU!6_!5vfXN6pQ|17F+6GrmyIHRJ9I=`J_n3#w~QuSJkd*IDDT5 zW4?fl((4uO6NXtde)?nI#o;QgVJ&i<6G+VKF}ye4=OU%u5}pK$JF1SSlFZg>yziHm z+yJu~8M6USg{{-b<8?_Y!wgezD$Xk;{9=+R=}Gv^TEfGkNBezDN%* zVfnbvg5(92zqF74>8dPKL||IhrhT+PqoeETMPHEcsf=b#o#CxTU3RTrqKek1a<>yA z96B57wc?!38d>Iron?9c&!h-|1D2>eppL`T@K+J# zY3X;90HGuO@~Q}aI8>)f~$IfD^w9I_zZs`qNa&~XKzH| zN=L)lmNjH6PFK&UQ#>Z??s;{orhUmckPXU7`7%KQ0rW*KK*~ZSD1d@Q2iqOSx`yZH zr`Cyb_MLM|$7FqGIn_@a;O{PQF%~UTr{J+B;lJ7mIXK@QsZ1BF@StgWy3omWS7s&u zgspk)S7)B(<_b(?^vrMb~7j*P+qjy3xxiL(b+DZUu3VkB;x}oArgd_@oQV z^7Owz&Xz}3ol{FR3q%`s{iCHB2*}Dse+!&4oN%_k_nFrJJbp`+%Zo&m zBN8zm!-z0)AL4Z?yu#IbwG~`mOqNkK?Xo>3hqrL9$ss>vLKRWXb4zihf8JiWQ48z@ z@Aa*B7k02#eGva93lwf9gBjiwnPgpa&`R<~dc7wdriJ=#VefX7X{_xc^t?TrwWy9k zayxN3gJ2f;TU#(g>w^%G1FH)iEgTNFt8|A0)(w|1XVU8q2T0j^OBS^mia2lj@p-d% zr3f}>Zxm=z)?{(jO!$zpTzMocEYJzhj0tDTH6tJhYlx&@@KH8nl zu76p52bNBnymk~soj)6F9FH=WXlqN0)x-RL6g1yS+N_p+30Z+;x=b%=aqBN}6JWan zx6yD{0yq>tgvi4tPF_n8P8a+cgve5_r&FBg|Z!4hl%BtQysdVTNj@5(L-w zB6bDWTuKacZ;PYG{oad}vy^h>xI2o0=-9LS_zlzawa1DfLVOPswG0?YlHS8qn((wj*rG>|R_EIUGEEGO)R=Odd zly*2)Ba?MM9i_P|M?X6l_-TxIVP&ubF-2<+L-qtb)kkr_-VGrB)AxFy&dW&tSB^lHFZ&6bU*EyqT;=|C;d*s!!@<97e80*g=17bkuhZ=ltsgq#N6heJ6LMtXx6U3cTx(nosrUq zbp{yR$MF~JdkIOzze0-i>?KSGS|9q3d*Ib!)Nl%YiUv!?Y+4#_V){e?1<=u6 z8@dLC`JS3{_d83;-Pt@9|7@@md=k#L#9i5Wor#W~{L{NP`1xi==m#B{;3^i1Z5nXl zyU_}0G7p^B$04jTNz2CD|l%pCIM z(>$OsyY3$@SmuN))51dm2M*V*)OVz%(j3YTvmwKa%Hl-dIob2d2{SU_+d}B4i7|SC zG&8oAlJ?c^%eKBIG-SYo2tm+>SlN-}`g0r4x#0%aS-&$nevKw~eyScylp&o@7yHY; zr0!wRlq$xIYP%Et;kfr|k>2rTU+M8RbsaFFmrw$?ArNG11~0VNz*dhV46W_vYWTa( zI|^3W^f<02C4@a1WH~;Bz9U=kDy)y;wancSc&<1iK$h9-dyv=qdo#)XMTxTxnCgP> zt|3vEo|E1TZ7h5`B?$30Ib+Fe!<)843EQ5X3tTQFgHl#IAJ11F_+c(%=CtWE-E)q6 z#oNl)xIcV|`B^=2D)yH~3Pvuh8*v&(WG-0sa%V9k z3%qh70hF`fuToReOyaNw@1iuxcC!Hzb1xS)e27PDwYSbdAYW_7*7 z@&F`waiJqN?f!;8vKv#1LdH(ZWT=%%Y93w^PthhU|!h5(Ba zcwO`g8zhQpapqkby|u&j7zn=&pUBfOBSD z*;MD_D$+_R77Dul_Y>J7DLCA8y3J=@;{<@kM3NxbvJxvpvu!EI@kg8hyj9e0H8NUG zWMM;Y<&chLJYXDMK)$a0U4am^d$Ep=NGjEK zOOZCFmj6XMH=Uy4G_ub=s1Q7jl8$VQ_7lBnJU?r!!#=bA)LEv%O-kwf9dc^@S}M;^ zHMDz_KjWhjh1r7NhxCuQzyxe>V&rUM46=P7TE9=~f8zqV|0`Tj`J?gr)h{^#xCdka zkHGfi&tUBlmp!60yg%$7rw{Ot&%a}~zv>@x*l#`BqvVhA0Y2p6?Z2SVztW-p7Cp=# z;4vPY|C8eURrbhye%oRHNf!Pf`wNKrS7NL`i3t$?5dV+HkC^9?-aKv%?w_>D58A&$ zpTAl^vYAKj^2lH?{v@Y+@?iZ>q~@>6$A)?AgTEX1lED4FeU#*okRSj5^x;~1D2XQN H NoReturn: :func:`identify` finds no locally-detectable invisible AI signal, running it anyway would damage a clean image for nothing -- the dominant paid score-0 cause on no-watermark uploads. So skip it, but do NOT imply the image is - clean: a pixel SynthID is undetectable here once its metadata proxy is gone. - Write no output and exit :data:`EXIT_NO_INVISIBLE_SIGNAL`; ``--force`` runs - the scrub regardless. + clean: only one SynthID carrier family in a calibrated image-size range has + a local detector, so other sizes or epochs can still be present after + their metadata proxy is gone. Write no output and exit + :data:`EXIT_NO_INVISIBLE_SIGNAL`; ``--force`` runs the scrub regardless. """ console.print( - " No invisible AI watermark detected (no C2PA/SynthID provenance, no open\n" - " watermark). Skipped the diffusion scrub -- regenerating the pixels would\n" - " only degrade the image with nothing to remove, so no output was written.\n" - " This does NOT prove the image is clean: a pixel watermark such as SynthID\n" - " cannot be detected here once its metadata proxy is absent (it may have\n" - " been stripped earlier). If you know the image is AI-generated and want the\n" - " pixels regenerated regardless, re-run with --force:\n" + " No supported invisible AI watermark detected (no provenance, supported\n" + " SynthID carrier, or open watermark). Skipped the diffusion scrub --\n" + " regenerating the pixels would only degrade the image with nothing to\n" + " remove, so no output was written.\n" + " This does NOT prove the image is clean: the local SynthID detector covers\n" + " one carrier family in a calibrated image-size range. If you know the image\n" + " is AI-generated and want the pixels regenerated regardless, re-run with\n" + " --force:\n" f" remove-ai-watermarks invisible {source.name} --force" ) raise SystemExit(EXIT_NO_INVISIBLE_SIGNAL) @@ -1316,6 +1318,41 @@ def cmd_video_batch( raise SystemExit(1) +# ── SynthID pixel detection ── +@main.command("detect-synthid") +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--json", "as_json", is_flag=True, help="Emit the detector result as JSON.") +def cmd_detect_synthid(source: Path, as_json: bool) -> None: + """Detect the SynthID periodic pixel carrier at calibrated image sizes. + + A negative result means this detector did not find its supported carrier; it + is not proof that the image contains no SynthID watermark. + """ + from remove_ai_watermarks.synthid_detector import detect_synthid + + source = _validate_image(source) + try: + result = detect_synthid(source) + except RuntimeError as exc: + raise click.ClickException(str(exc)) from exc + + if as_json: + click.echo(json.dumps(result.to_dict(), indent=2)) + return + + _banner() + console.print(f"\n SynthID pixel carrier: {result.status}") + console.print(f" Geometry: {result.width}x{result.height}") + if result.score is not None: + console.print(f" Score: {result.score:.6f} (threshold: {result.threshold:.6f})") + console.print(f" Detector: {result.detector}") + console.print( + " Scope: one confirmed periodic carrier family in a calibrated image-size range.\n" + " Arbitrary spatial resampling is not registered. A negative or\n" + " unsupported result is not proof that SynthID is absent." + ) + + # ── Provenance identification ── @main.command("identify") @click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @@ -1360,8 +1397,8 @@ def cmd_identify(ctx: click.Context, source: Path, no_visible: bool, as_json: bo if report.is_ai_generated is None: console.print( " No locally-readable AI signal found. This is not the same as 'clean': " - "metadata is often stripped by re-encoding, screenshots, or upload, and SynthID-class " - "pixel watermarks (Gemini / Nano Banana / gpt-image) have no local detector. " + "metadata is often stripped by re-encoding, screenshots, or upload, and the local " + "SynthID pixel detector covers one carrier family in a calibrated image-size range. " "See caveats below." ) @@ -1455,8 +1492,9 @@ def cmd_all( stage_text = { ("invisible", "no-signal"): ( "Skipped (no invisible AI watermark detected; pixels left intact).\n" - " Not a clean-image guarantee: a pixel SynthID is undetectable once its\n" - " metadata proxy is gone. Re-run with --force to scrub regardless." + " Not a clean-image guarantee: the local SynthID detector covers one\n" + " carrier family in a calibrated image-size range. Re-run with --force\n" + " to scrub regardless." ), ("invisible", "unavailable"): ( f"Warning: Skipped - GPU dependencies not installed.\n Install them with: pip install {INVISIBLE_EXTRA}" diff --git a/src/remove_ai_watermarks/identify.py b/src/remove_ai_watermarks/identify.py index d765028..2a4324a 100644 --- a/src/remove_ai_watermarks/identify.py +++ b/src/remove_ai_watermarks/identify.py @@ -6,15 +6,14 @@ Aggregates every locally-readable signal into a single :class:`ProvenanceReport` the signing platform (OpenAI, Google, Adobe, Microsoft). - **IPTC ``digitalSourceType``** "Made with AI" marker (Meta, X, others). - **PNG text / EXIF generation parameters** (Stable Diffusion, ComfyUI, InvokeAI). -- **SynthID provenance evidence** -- Google AI C2PA follows Google's all-media - policy; current OpenAI C2PA explicitly declares a watermark action. +- **SynthID evidence** -- supported C2PA provenance plus a positive-only local + detector for one confirmed periodic carrier family in a calibrated image-size range. - **Registered visible marks** (optional; needs cv2/numpy, no GPU) through the shared watermark registry. -Hard limit: a stripped image (re-encoded, screenshotted, social-media upload) -loses all metadata, and the SynthID *pixel* watermark is not locally decodable -(proprietary decoder). Absence of signals is therefore reported as ``Unknown``, -never as "clean". See CLAUDE.md "SynthID detection is metadata-only". +Hard limit: Google does not publish its payload decoder. The local pixel detector +covers only one measured carrier family in a calibrated image-size range, so +absence of signals is reported as ``Unknown``, never as "clean". """ from __future__ import annotations @@ -111,8 +110,14 @@ _STRIP_CAVEAT = ( "text chunks are stripped by re-encoding, screenshots, or social-media upload." ) _SYNTHID_CAVEAT = ( - "SynthID presence comes from supported provenance here; the pixel watermark is not locally " - "decoded (proprietary decoder). Confirm via the Gemini app or openai.com/verify." + "SynthID presence comes from supported provenance here. The separate local pixel detector " + "covers one measured carrier family in a calibrated image-size range; confirm other cases with " + "the provider oracle." +) +_SYNTHID_PIXEL_CAVEAT = ( + "The local SynthID pixel result is a positive-only match to one measured periodic carrier family " + "in a calibrated image-size range, not a proprietary payload decode. A negative or unsupported " + "result is not proof of absence." ) _IPTC_ONLY_CAVEAT = "The IPTC 'Made with AI' tag flags AI provenance but does not identify the specific platform." _INVISIBLE_WM_CAVEAT = ( @@ -951,6 +956,15 @@ def _trustmark(image_path: Path) -> str | None: return detect_trustmark(image_path) +def _synthid_pixel_watermark(image_path: Path, decode: _SharedDecode) -> bool: + """Whether the supported positive-only SynthID carrier is detected.""" + from remove_ai_watermarks.synthid_detector import detect_synthid, is_available + + if not is_available() or (image := decode.get()) is None: + return False + return detect_synthid(image_path, image=image).detected + + class _SharedDecode: """One decode of the source pixels, shared by every detector in a single report. @@ -1295,6 +1309,17 @@ def _identify_from_evidence( if platform is None: platform = f"{scheme} (open DWT-DCT watermark)" + # ── Positive-only SynthID periodic carrier ────────────────────── + # This is deliberately separate from C2PA provenance. It survives lossless + # metadata stripping, but covers only one carrier family and a calibrated + # image-size range. + if check_invisible and pixel_path is not None and _synthid_pixel_watermark(pixel_path, decode): + signals.append(Signal("synthid_pixel", "calibrated periodic carrier", "high")) + watermarks.append("SynthID periodic pixel carrier (calibrated image size)") + caveats.append(_SYNTHID_PIXEL_CAVEAT) + if platform is None: + platform = "SynthID carrier detected (provider not attributed locally)" + # ── Adobe TrustMark invisible watermark (open decoder, no key) ─── # The watermark behind Adobe Durable Content Credentials. Decoded locally, # but it binds provenance for human-authored content too, so it enriches the @@ -1306,7 +1331,7 @@ def _identify_from_evidence( platform = "Adobe (TrustMark / Content Credentials)" # ── Verdict so far (metadata + embedded watermark) ────────────── - invisible_wm = any(s.name == "invisible_watermark" for s in signals) + invisible_wm = any(s.name in {"invisible_watermark", "synthid_pixel"} for s in signals) exif_gen = any(s.name == "exif_generator" for s in signals) xai_sig = any(s.name == "xai_signature" for s in signals) ai_from_metadata = bool( @@ -1405,8 +1430,9 @@ def identify( image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container). check_visible: Also run the registered visible-mark detectors through cv2. Set False for a metadata-only, dependency-light scan. - check_invisible: Also decode optional open invisible watermarks - (SD/SDXL/FLUX). No-op when the decoder extra is not installed. + check_invisible: Also run optional pixel detectors for the supported + SynthID carrier and open SD/SDXL/FLUX watermarks. No-op when their + numeric extras are not installed. File-backed metadata extraction runs first. The extracted evidence is then evaluated independently, followed by the optional pixel-backed visible and @@ -1436,13 +1462,13 @@ def has_invisible_target(image_path: Path) -> bool: to remove. Runs :func:`identify` with ``check_visible=False`` -- a visible mark is handled by the separate visible pass and is NOT a diffusion target -- and ``check_invisible=True`` so an open watermark counts. Returns - ``report.ai_from_metadata`` (C2PA AI issuer / SynthID provenance, IPTC, AIGC, local - gen params, EXIF/xAI, open DWT-DCT / TrustMark). + ``report.ai_from_metadata`` (C2PA AI issuer / SynthID provenance or periodic + carrier, IPTC, AIGC, local gen params, EXIF/xAI, open DWT-DCT / TrustMark). - IMPORTANT -- this cannot prove a pixel SynthID is absent: SynthID is detectable - only through its C2PA proxy, so a metadata-stripped AI image reads as no signal - here. A False therefore means "no locally-detectable invisible target", not - "clean". Callers must NOT present a skip as a finished clean result. + IMPORTANT -- this cannot prove a pixel SynthID is absent: the local detector + covers one carrier family in a calibrated image-size range. A False therefore + means "no supported locally-detectable invisible target", not "clean". Callers + must NOT present a skip as a finished clean result. Fail-safe: any error resolves to True so the removal still runs -- leaving a watermark on a paid removal is worse than over-regenerating a clean image. diff --git a/src/remove_ai_watermarks/metadata.py b/src/remove_ai_watermarks/metadata.py index 50f146e..0348f5a 100644 --- a/src/remove_ai_watermarks/metadata.py +++ b/src/remove_ai_watermarks/metadata.py @@ -803,8 +803,9 @@ def synthid_source(image_path: Path) -> str | None: None. The evidence is readable only while the C2PA manifest is intact. Absence is - not proof: C2PA can be stripped while the pixel watermark survives, and the - pixel watermark itself is not locally detectable (proprietary decoder). + not proof: C2PA can be stripped while the pixel watermark survives. This + metadata helper does not call the separate, geometry-limited local carrier + detector. Args: image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container). diff --git a/src/remove_ai_watermarks/synthid_detector.py b/src/remove_ai_watermarks/synthid_detector.py new file mode 100644 index 0000000..c6980a5 --- /dev/null +++ b/src/remove_ai_watermarks/synthid_detector.py @@ -0,0 +1,258 @@ +"""Detect the confirmed periodic SynthID image carrier at calibrated image sizes. + +This is a positive-only detector for one measured carrier epoch, not Google's +private payload decoder. A positive result is strong local evidence for the +carrier. A negative result means only that this exact detector did not find it; +image sizes outside the calibrated pixel-count range are reported separately. + +The numeric runtime requires the ``pixels`` extra. Imports remain lazy so the +package's metadata-only paths stay dependency-light. +""" + +# The optional numeric libraries do not provide complete types for this path. +# pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from numpy.typing import NDArray + +SynthIDDetectionStatus = Literal["detected", "not_detected", "unsupported"] + +DETECTOR_ID = "synthid-periodic-tile-v2" +MODEL_FILENAME = "synthid_periodic_tile_2048_v1.npz" +# The template remains frozen at this model geometry. Runtime images are never +# resized. The supported pixel-count interval is the separately challenged domain: +# below it too few repetitions make the positive-only statistic unreliable, and +# above it resource use and specificity have not been calibrated. +MODEL_WIDTH = 2048 +MODEL_HEIGHT = 2048 +MIN_SUPPORTED_PIXELS = 1_000_000 +MAX_SUPPORTED_PIXELS = 18_000_000 +TILE_THRESHOLD = 0.17357069773071196 +INSTALL_HINT = "install the pixel extra: uv add 'remove-ai-watermarks[pixels]'" + + +@dataclass(frozen=True) +class SynthIDDetection: + """One local periodic-carrier verdict.""" + + status: SynthIDDetectionStatus + width: int + height: int + score: float | None + threshold: float + detector: str = DETECTOR_ID + + @property + def detected(self) -> bool: + """Whether the supported carrier crossed its frozen threshold.""" + return self.status == "detected" + + def to_dict(self) -> dict[str, str | int | float | None]: + """Return a JSON-safe result without a local file path.""" + return { + "status": self.status, + "width": self.width, + "height": self.height, + "score": self.score, + "threshold": self.threshold, + "detector": self.detector, + } + + +def is_available() -> bool: + """True when the optional numeric runtime is installed.""" + from remove_ai_watermarks.optional_deps import module_available + + return module_available("cv2", "numpy") + + +@lru_cache(maxsize=1) +def _load_template() -> tuple[NDArray[Any], float, int, int, int, int]: + """Load and validate the bundled pickle-free detector model.""" + import numpy as np + + model_path = Path(__file__).parent / "assets" / MODEL_FILENAME + with np.load(model_path, allow_pickle=False) as artifact: + if int(artifact["format_version"]) != 1: + raise RuntimeError("unsupported SynthID detector model format") + height = int(artifact["height"]) + width = int(artifact["width"]) + tile_height = int(artifact["tile_height"]) + tile_width = int(artifact["tile_width"]) + denoise_sigma = float(artifact["denoise_sigma"]) + template = np.asarray(artifact["template"], dtype=np.float64) + if not _geometry_supported(width, height): + raise RuntimeError("bundled SynthID detector has unexpected geometry") + if template.shape != (tile_height, tile_width, 3): + raise RuntimeError("bundled SynthID detector has an invalid template shape") + if not np.all(np.isfinite(template)) or not np.isclose(np.linalg.norm(template), 1.0): + raise RuntimeError("bundled SynthID detector has an invalid template") + if not np.isfinite(denoise_sigma) or denoise_sigma <= 0.0: + raise RuntimeError("bundled SynthID detector has an invalid denoise sigma") + return template, denoise_sigma, height, width, tile_height, tile_width + + +def fold_residual_template( + pixels: NDArray[Any], + *, + tile_height: int, + tile_width: int, + denoise_sigma: float, +) -> NDArray[Any]: + """Estimate a zero-mean periodic residual template by modulo folding.""" + import cv2 + import numpy as np + + if pixels.ndim != 3 or pixels.shape[2] != 3: + raise ValueError("pixels must have shape (height, width, 3)") + if tile_height < 1 or tile_width < 1 or denoise_sigma <= 0.0: + raise ValueError("tile dimensions and denoise sigma must be positive") + height, width = pixels.shape[:2] + if height < tile_height or width < tile_width: + raise ValueError("image geometry must be at least as large as the tile geometry") + divisible = height % tile_height == 0 and width % tile_width == 0 + full_height = height - height % tile_height + full_width = width - width % tile_width + repeats_y = full_height // tile_height + repeats_x = full_width // tile_width + remaining_height = height - full_height + remaining_width = width - full_width + counts = np.full((tile_height, tile_width), repeats_y * repeats_x, dtype=np.int64) + counts[:remaining_height] += repeats_x + counts[:, :remaining_width] += repeats_y + counts[:remaining_height, :remaining_width] += 1 + + # OpenCV filters channels independently. Processing one channel at a time + # keeps the 18 MP upper bound from requiring two full three-channel float32 + # buffers in addition to the decoded image. + folded = np.empty((tile_height, tile_width, 3), dtype=np.float64) + for channel in range(3): + residual = pixels[:, :, channel].astype(np.float32) + residual -= cv2.GaussianBlur( + residual, + (0, 0), + sigmaX=denoise_sigma, + sigmaY=denoise_sigma, + borderType=cv2.BORDER_REFLECT_101, + ) + if divisible: + folded[:, :, channel] = residual.reshape( + repeats_y, + tile_height, + repeats_x, + tile_width, + ).mean(axis=(0, 2), dtype=np.float64) + continue + folded_sum = ( + residual[:full_height, :full_width] + .reshape( + repeats_y, + tile_height, + repeats_x, + tile_width, + ) + .sum(axis=(0, 2), dtype=np.float64) + ) + if remaining_height: + bottom = residual[full_height:, :full_width].reshape( + remaining_height, + repeats_x, + tile_width, + ) + folded_sum[:remaining_height] += bottom.sum(axis=1, dtype=np.float64) + if remaining_width: + right = residual[:full_height, full_width:].reshape( + repeats_y, + tile_height, + remaining_width, + ) + folded_sum[:, :remaining_width] += right.sum(axis=0, dtype=np.float64) + if remaining_height and remaining_width: + folded_sum[:remaining_height, :remaining_width] += residual[ + full_height:, + full_width:, + ] + folded[:, :, channel] = folded_sum / counts + return folded - np.mean(folded, axis=(0, 1), keepdims=True) + + +def unit_tile(tile: NDArray[Any]) -> tuple[NDArray[Any], float]: + """Return TILE normalized by its L2 norm and the original norm.""" + import numpy as np + + norm = float(np.linalg.norm(tile)) + if norm == 0.0: + return np.zeros_like(tile, dtype=np.float64), 0.0 + return np.asarray(tile, dtype=np.float64) / norm, norm + + +def _image_size(image_path: Path) -> tuple[int, int]: + from PIL import Image + + with Image.open(image_path) as image: + return image.size + + +def _geometry_supported(width: int, height: int) -> bool: + """Whether the image has a calibrated number of periodic-tile samples.""" + pixels = width * height + return MIN_SUPPORTED_PIXELS <= pixels <= MAX_SUPPORTED_PIXELS + + +def detect_synthid(image_path: str | Path, *, image: NDArray[Any] | None = None) -> SynthIDDetection: + """Detect the supported periodic carrier in IMAGE_PATH. + + ``not_detected`` is not a clean-image guarantee. It means only that the + frozen periodic carrier did not cross its calibrated threshold. + """ + path = Path(image_path) + if image is None: + width, height = _image_size(path) + else: + if image.ndim != 3 or image.shape[2] != 3: + raise ValueError("image must be a three-channel BGR array") + height, width = image.shape[:2] + if not _geometry_supported(width, height): + return SynthIDDetection( + status="unsupported", + width=width, + height=height, + score=None, + threshold=TILE_THRESHOLD, + ) + if not is_available(): + raise RuntimeError(f"SynthID pixel detection needs numpy and OpenCV; {INSTALL_HINT}") + + import numpy as np + from PIL import Image + + template, sigma, _model_height, _model_width, tile_height, tile_width = _load_template() + if image is None: + with Image.open(path) as source: + pixels = np.asarray(source.convert("RGB"), dtype=np.uint8) + else: + pixels = np.asarray(image[:, :, ::-1], dtype=np.uint8) + if pixels.shape != (height, width, 3): + raise RuntimeError("decoded image geometry does not match its header") + folded = fold_residual_template( + pixels, + tile_height=tile_height, + tile_width=tile_width, + denoise_sigma=sigma, + ) + normalized, _norm = unit_tile(folded) + score = float(np.sum(template * normalized)) + return SynthIDDetection( + status="detected" if score >= TILE_THRESHOLD else "not_detected", + width=width, + height=height, + score=score, + threshold=TILE_THRESHOLD, + ) diff --git a/tests/test_api.py b/tests/test_api.py index 2b0f35f..6b59986 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -18,8 +18,12 @@ CHATGPT = SAMPLES / "chatgpt-1.png" class TestTopLevelExports: def test_lazy_reexports_resolve(self): + from remove_ai_watermarks import synthid_detector + assert raiw.remove_visible is api.remove_visible assert raiw.visible_provenance is api.visible_provenance + assert raiw.detect_synthid is synthid_detector.detect_synthid + assert raiw.SynthIDDetection is synthid_detector.SynthIDDetection def test_unknown_attribute_raises(self): with pytest.raises(AttributeError): diff --git a/tests/test_cli.py b/tests/test_cli.py index cf482b2..a1722a0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -736,6 +736,28 @@ class TestIdentifyCommand: assert result.exit_code != 0 +class TestDetectSynthIDCommand: + def test_help(self, runner): + result = runner.invoke(main, ["detect-synthid", "--help"]) + assert result.exit_code == 0 + assert "calibrated image sizes" in result.output + + def test_unsupported_geometry_is_machine_readable(self, runner, tmp_clean_png): + result = runner.invoke(main, ["detect-synthid", str(tmp_clean_png), "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["status"] == "unsupported" + assert payload["score"] is None + + def test_non_json_output_preserves_negative_scope(self, runner, tmp_clean_png): + result = runner.invoke(main, ["detect-synthid", str(tmp_clean_png)]) + + assert result.exit_code == 0, result.output + assert "unsupported" in result.output + assert "not proof that SynthID is absent" in result.output + + class TestBatchCommand: """Tests for the 'batch' subcommand.""" diff --git a/tests/test_identify.py b/tests/test_identify.py index 02b894f..0feffe1 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -886,6 +886,32 @@ class TestIdentifyVisibleTextMarks: # ── Caveats and serialization ─────────────────────────────────────── +class TestSynthIDPixelCarrier: + def test_positive_pixel_carrier_is_high_confidence_ai_evidence(self, tmp_clean_png: Path): + with ( + patch("remove_ai_watermarks.identify._invisible_watermark", return_value=None), + patch("remove_ai_watermarks.identify._synthid_pixel_watermark", return_value=True), + patch("remove_ai_watermarks.identify._trustmark", return_value=None), + ): + report = identify(tmp_clean_png, check_visible=False, check_invisible=True) + + assert report.is_ai_generated is True + assert report.confidence == "high" + assert any(signal.name == "synthid_pixel" for signal in report.signals) + assert any("positive-only" in caveat for caveat in report.caveats) + + def test_negative_pixel_carrier_does_not_claim_clean(self, tmp_clean_png: Path): + with ( + patch("remove_ai_watermarks.identify._invisible_watermark", return_value=None), + patch("remove_ai_watermarks.identify._synthid_pixel_watermark", return_value=False), + patch("remove_ai_watermarks.identify._trustmark", return_value=None), + ): + report = identify(tmp_clean_png, check_visible=False, check_invisible=True) + + assert report.is_ai_generated is None + assert not any(signal.name == "synthid_pixel" for signal in report.signals) + + @pytest.mark.skipif(not SAMPLES_DIR.exists(), reason="data/fixtures/provenance not present") class TestIdentifyCaveats: def test_legacy_openai_has_no_synthid_claim(self): diff --git a/tests/test_synthid_detector.py b/tests/test_synthid_detector.py new file mode 100644 index 0000000..2f23429 --- /dev/null +++ b/tests/test_synthid_detector.py @@ -0,0 +1,217 @@ +"""Runtime tests for the positive-only SynthID periodic carrier detector.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +import remove_ai_watermarks.synthid_detector as detector + + +@pytest.fixture(scope="module") +def supported_images(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + """Create supported-geometry positive and negative synthetic fixtures.""" + directory = tmp_path_factory.mktemp("synthid-detector") + template, *_model = detector._load_template() + scaled_tile = np.rint(template / np.max(np.abs(template))) + marked = np.full((detector.MODEL_HEIGHT, detector.MODEL_WIDTH, 3), 128, dtype=np.float64) + marked += np.tile(scaled_tile, (128, 128, 1)) + + positive = directory / "positive.png" + negative = directory / "negative.png" + Image.fromarray(np.clip(np.rint(marked), 0, 255).astype(np.uint8), "RGB").save(positive) + Image.new("RGB", (detector.MODEL_WIDTH, detector.MODEL_HEIGHT), (128, 128, 128)).save(negative) + return positive, negative + + +def test_bundled_model_is_the_frozen_calibrated_artifact() -> None: + model = Path(detector.__file__).parent / "assets" / detector.MODEL_FILENAME + + assert hashlib.sha256(model.read_bytes()).hexdigest() == ( + "ee7838da8542c206c3403284b68e98f0ac99429e82f262c1a438f50a638b488b" + ) + + +@pytest.mark.parametrize( + ("width", "height"), + [(1000, 1000), (1001, 1000), (3000, 6000), (768, 1364)], +) +def test_supported_geometry_uses_the_challenged_pixel_count_range(width: int, height: int) -> None: + assert detector._geometry_supported(width, height) + + +@pytest.mark.parametrize( + ("width", "height"), + [(999, 1000), (3001, 6000), (64, 32)], +) +def test_geometry_outside_the_challenged_pixel_count_range_is_unsupported( + width: int, + height: int, +) -> None: + assert not detector._geometry_supported(width, height) + + +def test_detects_supported_periodic_carrier(supported_images: tuple[Path, Path]) -> None: + positive, _negative = supported_images + + result = detector.detect_synthid(positive) + + assert result.status == "detected" + assert result.detected is True + assert result.score is not None + assert result.score > result.threshold + assert result.to_dict()["detector"] == detector.DETECTOR_ID + + +def test_detects_unregistered_non_divisible_geometry_in_size_range(tmp_path: Path) -> None: + width, height = 1001, 1000 + template, *_model = detector._load_template() + scaled_tile = np.rint(template / np.max(np.abs(template))) + repeats_y = (height + scaled_tile.shape[0] - 1) // scaled_tile.shape[0] + repeats_x = (width + scaled_tile.shape[1] - 1) // scaled_tile.shape[1] + carrier = np.tile(scaled_tile, (repeats_y, repeats_x, 1))[:height, :width] + pixels = np.clip(np.rint(carrier + 128.0), 0, 255).astype(np.uint8) + path = tmp_path / "non-divisible-positive.png" + Image.fromarray(pixels, "RGB").save(path) + + result = detector.detect_synthid(path) + + assert result.status == "detected" + assert (result.width, result.height) == (width, height) + assert result.score is not None + assert result.score > result.threshold + + +def test_supported_negative_does_not_claim_clean(supported_images: tuple[Path, Path]) -> None: + _positive, negative = supported_images + + result = detector.detect_synthid(negative) + + assert result.status == "not_detected" + assert result.detected is False + assert result.score == pytest.approx(0.0) + + +def test_threshold_mutation_changes_the_real_verdict( + monkeypatch: pytest.MonkeyPatch, + supported_images: tuple[Path, Path], +) -> None: + positive, _negative = supported_images + baseline = detector.detect_synthid(positive) + assert baseline.score is not None + assert baseline.status == "detected" + mutated_threshold = float(np.nextafter(baseline.score, np.inf)) + assert mutated_threshold > baseline.score + + monkeypatch.setattr(detector, "TILE_THRESHOLD", mutated_threshold) + mutated = detector.detect_synthid(positive) + + assert mutated.status == "not_detected" + assert mutated.threshold == mutated_threshold + + +def test_unsupported_geometry_is_distinct_from_negative(tmp_path: Path) -> None: + path = tmp_path / "small.png" + Image.new("RGB", (64, 32), "white").save(path) + + result = detector.detect_synthid(path) + + assert result.status == "unsupported" + assert result.score is None + assert (result.width, result.height) == (64, 32) + + +def test_shared_bgr_decode_matches_file_decode(supported_images: tuple[Path, Path]) -> None: + import cv2 + + positive, _negative = supported_images + bgr = cv2.imread(str(positive)) + assert bgr is not None + + from_file = detector.detect_synthid(positive) + from_array = detector.detect_synthid(positive, image=bgr) + + assert from_array == from_file + + +def test_supported_geometry_requires_pixel_dependencies( + monkeypatch: pytest.MonkeyPatch, + supported_images: tuple[Path, Path], +) -> None: + _positive, negative = supported_images + monkeypatch.setattr(detector, "is_available", lambda: False) + + with pytest.raises(RuntimeError, match="pixel extra"): + detector.detect_synthid(negative) + + +def test_fold_accepts_non_divisible_geometry_without_resampling() -> None: + rng = np.random.default_rng(20260810) + tile = rng.normal(0.0, 8.0, size=(16, 16, 3)) + repeated = np.tile(tile, (19, 20, 1)) + 128.0 + + divisible = detector.fold_residual_template( + repeated, + tile_height=16, + tile_width=16, + denoise_sigma=1.0, + ) + non_divisible = detector.fold_residual_template( + repeated[:299, :317], + tile_height=16, + tile_width=16, + denoise_sigma=1.0, + ) + divisible_unit, _ = detector.unit_tile(divisible) + non_divisible_unit, _ = detector.unit_tile(non_divisible) + + assert non_divisible.shape == (16, 16, 3) + assert float(np.sum(divisible_unit * non_divisible_unit)) > 0.999 + + +def test_non_divisible_fold_matches_modulo_cell_means() -> None: + import cv2 + + rng = np.random.default_rng(44041) + pixels = rng.integers(0, 256, size=(53, 71, 3), dtype=np.uint8) + source = pixels.astype(np.float32) + residual = source - cv2.GaussianBlur( + source, + (0, 0), + sigmaX=1.25, + sigmaY=1.25, + borderType=cv2.BORDER_REFLECT_101, + ) + expected = np.empty((16, 16, 3), dtype=np.float64) + for tile_y in range(16): + for tile_x in range(16): + expected[tile_y, tile_x] = residual[tile_y::16, tile_x::16].mean( + axis=(0, 1), + dtype=np.float64, + ) + expected -= np.mean(expected, axis=(0, 1), keepdims=True) + + actual = detector.fold_residual_template( + pixels, + tile_height=16, + tile_width=16, + denoise_sigma=1.25, + ) + + np.testing.assert_allclose(actual, expected, rtol=0.0, atol=0.0) + + +def test_fold_rejects_tile_larger_than_image() -> None: + pixels = np.zeros((15, 16, 3), dtype=np.uint8) + + with pytest.raises(ValueError, match="at least as large"): + detector.fold_residual_template( + pixels, + tile_height=16, + tile_width=16, + denoise_sigma=1.0, + ) diff --git a/tests/test_synthid_oracle_batch.py b/tests/test_synthid_oracle_batch.py new file mode 100644 index 0000000..b777ba7 --- /dev/null +++ b/tests/test_synthid_oracle_batch.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_oracle_batch as batch +from synthid_periodic_tile import fold_residual_template, unit_tile +from synthid_periodic_tile_probe import PeriodicTileModel +from synthid_periodic_tile_probe import save_model as save_tile_model +from synthid_phase_carrier import PhaseCarrierModel +from synthid_phase_carrier import save_model as save_phase_model + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + rng = np.random.default_rng(23) + raw_tile = rng.normal(size=(8, 8, 3)) + raw_tile -= np.mean(raw_tile, axis=(0, 1), keepdims=True) + raw_tile, _ = unit_tile(raw_tile) + source = np.clip(np.rint(128.0 + 24.0 * np.tile(raw_tile, (8, 8, 1))), 0, 255).astype(np.uint8) + source_path = tmp_path / "source.png" + Image.fromarray(source, mode="RGB").save(source_path) + + folded = fold_residual_template( + source, + tile_height=8, + tile_width=8, + denoise_sigma=1.0, + ) + template, expected_norm = unit_tile(folded) + tile_model = PeriodicTileModel(64, 64, 8, 8, 1.0, template, expected_norm) + tile_model_path = tmp_path / "tile-model.npz" + save_tile_model(tile_model_path, tile_model) + + spectra = np.stack([np.fft.rfft2(source[:, :, channel]) for channel in range(3)], axis=2) + magnitude = np.abs(spectra) + magnitude[0, 0, :] = 0.0 + row, column, channel = np.unravel_index(int(np.argmax(magnitude)), magnitude.shape) + phase_model = PhaseCarrierModel( + 64, + 64, + np.asarray([row], dtype=np.int32), + np.asarray([column], dtype=np.int32), + np.asarray([channel], dtype=np.int8), + np.asarray([np.angle(spectra[row, column, channel])]), + np.asarray([1.0]), + np.asarray([magnitude[row, column, channel]]), + ) + phase_model_path = tmp_path / "phase-model.npz" + save_phase_model(phase_model_path, phase_model) + return source_path, tile_model_path, phase_model_path + + +def _build(tmp_path: Path) -> tuple[Path, Path]: + source, tile_model, phase_model = _fixture(tmp_path) + output_dir = tmp_path / "oracle-batch" + manifest_path = batch.build_batch( + [source], + output_dir=output_dir, + tile_model_path=tile_model, + phase_model_path=phase_model, + tile_threshold=0.5, + phase_threshold=0.5, + active_threshold=0.0, + strength=2.0, + seed=20260810, + provider="google", + repository_root=Path(__file__).resolve().parent.parent, + ) + return output_dir, manifest_path + + +def test_build_preregisters_fixed_request_order_without_copying_source(tmp_path: Path) -> None: + output_dir, manifest_path = _build(tmp_path) + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + assert manifest["status"] == "preregistered_unsubmitted" + assert manifest["request_count"] == 5 + assert [row["role"] for row in manifest["rows"]] == list(batch.ROLE_ORDER) + assert manifest["rows"][0]["in_batch"] is False + assert not (output_dir / "source.png").exists() + assert manifest["provider"] == "google" + assert "shifted, and orthogonal_random are detected" in manifest["decision_rule"] + assert all(row["synthid_result"] is None for row in manifest["rows"]) + assert all(row["c2pa_result"] is None for row in manifest["rows"]) + template = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8")) + assert template["manifest_sha256"] == batch.artifact_sha256(manifest_path) + assert [row["artifact_sha256"] for row in template["rows"]] == [row["artifact_sha256"] for row in manifest["rows"]] + assert ( + batch.verify_batch( + manifest_path, + repository_root=Path(__file__).resolve().parent.parent, + )["source_count"] + == 1 + ) + + +def test_verify_rejects_derivative_mutation(tmp_path: Path) -> None: + output_dir, manifest_path = _build(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + aligned_row = next(row for row in manifest["rows"] if row["role"] == "aligned") + aligned_path = output_dir / aligned_row["path"] + with Image.open(aligned_path) as image: + pixels = np.asarray(image.convert("RGB"), dtype=np.uint8).copy() + pixels[0, 0, 0] ^= 1 + Image.fromarray(pixels, mode="RGB").save(aligned_path) + assert batch.artifact_sha256(aligned_path) != aligned_row["artifact_sha256"] + + with pytest.raises(ValueError, match="artifact hash mismatch"): + batch.verify_batch( + manifest_path, + repository_root=Path(__file__).resolve().parent.parent, + ) + + +def test_build_rejects_output_inside_repository(tmp_path: Path) -> None: + source, tile_model, phase_model = _fixture(tmp_path) + repository_root = Path(__file__).resolve().parent.parent + + with pytest.raises(ValueError, match="outside the repository"): + batch.build_batch( + [source], + output_dir=repository_root / ".local-eval/oracle-batch-test", + tile_model_path=tile_model, + phase_model_path=phase_model, + tile_threshold=0.5, + phase_threshold=0.5, + active_threshold=0.0, + strength=2.0, + seed=20260810, + provider="google", + repository_root=repository_root, + ) + + +def test_build_rejects_nonempty_output_directory(tmp_path: Path) -> None: + source, tile_model, phase_model = _fixture(tmp_path) + output_dir = tmp_path / "existing" + output_dir.mkdir() + (output_dir / "marker.txt").write_text("occupied", encoding="utf-8") + + with pytest.raises(ValueError, match="must not already contain"): + batch.build_batch( + [source], + output_dir=output_dir, + tile_model_path=tile_model, + phase_model_path=phase_model, + tile_threshold=0.5, + phase_threshold=0.5, + active_threshold=0.0, + strength=2.0, + seed=20260810, + provider="google", + repository_root=Path(__file__).resolve().parent.parent, + ) + + +def test_evaluate_requires_controls_and_aligned_outcome(tmp_path: Path) -> None: + output_dir, manifest_path = _build(tmp_path) + results = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8")) + for row in results["rows"]: + row["synthid_result"] = "not_detected" if row["role"] == "aligned" else "detected" + row["c2pa_result"] = "unavailable" + row["raw_response"] = f"verbatim {row['role']} result" + row["submitted_at"] = "2026-08-10T20:00:00Z" + results_path = output_dir / "results.json" + results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + + report = batch.evaluate_results( + manifest_path, + results_path, + repository_root=Path(__file__).resolve().parent.parent, + ) + + assert report["counts"] == { + "causal_success": 1, + "aligned_still_detected": 0, + "control_failed": 0, + "indeterminate": 0, + } + + shifted = next(row for row in results["rows"] if row["role"] == "shifted") + shifted["synthid_result"] = "not_detected" + results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + control_report = batch.evaluate_results( + manifest_path, + results_path, + repository_root=Path(__file__).resolve().parent.parent, + ) + assert control_report["counts"]["control_failed"] == 1 + + +def test_evaluate_rejects_incomplete_result(tmp_path: Path) -> None: + output_dir, manifest_path = _build(tmp_path) + results = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8")) + results["rows"].pop() + results_path = output_dir / "incomplete-results.json" + results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="cover every"): + batch.evaluate_results( + manifest_path, + results_path, + repository_root=Path(__file__).resolve().parent.parent, + ) diff --git a/tests/test_synthid_periodic_tile_ablation.py b/tests/test_synthid_periodic_tile_ablation.py new file mode 100644 index 0000000..38fe18e --- /dev/null +++ b/tests/test_synthid_periodic_tile_ablation.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_periodic_tile_ablation as ablation +from synthid_periodic_tile import fold_residual_template, unit_tile +from synthid_periodic_tile_probe import PeriodicTileModel +from synthid_phase_carrier import PhaseCarrierModel + + +def test_exact_sign_test_detects_one_sided_direction() -> None: + assert ablation.exact_sign_test(30, 0) == pytest.approx(1.862645149230957e-9) + assert ablation.exact_sign_test(0, 0) == 1.0 + + +def test_control_templates_are_norm_matched_and_random_control_is_orthogonal() -> None: + rng = np.random.default_rng(7) + template, _ = unit_tile(rng.normal(size=(8, 8, 3))) + + controls = ablation.control_templates(template, seed=11) + + assert set(controls) == {"aligned", "shifted", "orthogonal_random"} + assert all(np.linalg.norm(control) == pytest.approx(1.0) for control in controls.values()) + assert np.sum(controls["orthogonal_random"] * template) == pytest.approx(0.0, abs=1e-12) + assert not np.array_equal(controls["shifted"], template) + + +def test_aligned_subtraction_controls_both_synthetic_representations(tmp_path: Path) -> None: + rng = np.random.default_rng(17) + raw_tile = rng.normal(size=(8, 8, 3)) + raw_tile -= np.mean(raw_tile, axis=(0, 1), keepdims=True) + raw_tile, _ = unit_tile(raw_tile) + source = np.clip(np.rint(128.0 + 24.0 * np.tile(raw_tile, (8, 8, 1))), 0, 255).astype(np.uint8) + source_path = tmp_path / "source.png" + Image.fromarray(source, mode="RGB").save(source_path) + + folded = fold_residual_template( + source, + tile_height=8, + tile_width=8, + denoise_sigma=1.0, + ) + template, expected_norm = unit_tile(folded) + tile_model = PeriodicTileModel( + height=64, + width=64, + tile_height=8, + tile_width=8, + denoise_sigma=1.0, + template=template, + expected_norm=expected_norm, + ) + + spectra = np.stack([np.fft.rfft2(source[:, :, channel]) for channel in range(3)], axis=2) + magnitude = np.abs(spectra) + magnitude[0, 0, :] = 0.0 + row, column, channel = np.unravel_index(int(np.argmax(magnitude)), magnitude.shape) + phase_model = PhaseCarrierModel( + height=64, + width=64, + rows=np.asarray([row], dtype=np.int32), + columns=np.asarray([column], dtype=np.int32), + channels=np.asarray([channel], dtype=np.int8), + phases=np.asarray([np.angle(spectra[row, column, channel])]), + weights=np.asarray([1.0]), + expected_magnitudes=np.asarray([magnitude[row, column, channel]]), + ) + + report = ablation.run_ablation( + [source_path], + tile_model=tile_model, + phase_model=phase_model, + tile_threshold=0.5, + phase_threshold=0.5, + active_threshold=0.0, + strengths=(1.0, 2.0), + phase_strength=2.0, + seed=20260810, + ) + + assert report["original"] == {"tile_accepted": 1, "phase_accepted": 1} + aligned = next(row for row in report["phase_summaries"] if row["control"] == "aligned") + assert aligned["accepted"] == 0 + comparisons = {(row["aligned_minus"], row["metric"]): row for row in report["paired_comparisons"]} + assert comparisons[("shifted", "tile_delta")]["difference"]["median"] < 0.0 + assert comparisons[("orthogonal_random", "tile_delta")]["difference"]["median"] < 0.0 + + +def test_phase_strength_must_be_part_of_sweep() -> None: + template = np.zeros((8, 8, 3), dtype=np.float64) + template[0, 0, 0] = 1.0 + tile_model = PeriodicTileModel(64, 64, 8, 8, 1.0, template, 1.0) + phase_model = PhaseCarrierModel( + 64, + 64, + np.asarray([1]), + np.asarray([1]), + np.asarray([0]), + np.asarray([0.0]), + np.asarray([1.0]), + np.asarray([1.0]), + ) + + with pytest.raises(ValueError, match="phase strength"): + ablation.run_ablation( + [Path("unused.png")], + tile_model=tile_model, + phase_model=phase_model, + tile_threshold=0.0, + phase_threshold=0.0, + active_threshold=0.0, + strengths=(1.0,), + phase_strength=2.0, + seed=1, + ) diff --git a/tests/test_synthid_periodic_tile_probe.py b/tests/test_synthid_periodic_tile_probe.py index 159b5d6..9fee720 100644 --- a/tests/test_synthid_periodic_tile_probe.py +++ b/tests/test_synthid_periodic_tile_probe.py @@ -80,6 +80,25 @@ def test_registration_recovers_cyclic_tile_shift(tmp_path: Path) -> None: assert (registered.row_shift, registered.column_shift) == (7, 6) +def test_array_scoring_matches_file_scoring(tmp_path: Path) -> None: + carrier = _carrier(12) + positives = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, carrier, seed=index) + positives.append(path) + model = probe.discover_model(positives, tile_height=8, tile_width=8) + with Image.open(positives[0]) as image: + pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) + + file_score = probe.score_image(positives[0], model) + array_score = probe.score_pixels(pixels, model) + + assert array_score.score == pytest.approx(file_score.score) + assert array_score.active_support == pytest.approx(file_score.active_support) + assert array_score.path == "" + + def test_fft_correlations_match_explicit_cyclic_shifts() -> None: rng = np.random.default_rng(30) template = rng.normal(size=(4, 5, 3)) diff --git a/tests/test_synthid_phase_carrier.py b/tests/test_synthid_phase_carrier.py index 464e470..b394c57 100644 --- a/tests/test_synthid_phase_carrier.py +++ b/tests/test_synthid_phase_carrier.py @@ -104,6 +104,24 @@ def test_scoring_rejects_geometry_mismatch(tmp_path: Path) -> None: carrier.score_image(mismatch, model) +def test_array_scoring_matches_file_scoring(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + model = carrier.discover_model(positives, peak_count=4, min_radius=1.0) + with Image.open(positives[0]) as image: + pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) + + file_score = carrier.score_image(positives[0], model) + array_score = carrier.score_pixels(pixels, model) + + assert array_score.score == pytest.approx(file_score.score) + assert array_score.active_weight_fraction == pytest.approx(file_score.active_weight_fraction) + assert array_score.path == "" + + def test_scoring_can_canonicalize_geometry(tmp_path: Path) -> None: positives: list[Path] = [] for index in range(3): diff --git a/tests/test_synthid_tile_attack.py b/tests/test_synthid_tile_attack.py index d2560eb..cab4897 100644 --- a/tests/test_synthid_tile_attack.py +++ b/tests/test_synthid_tile_attack.py @@ -4,7 +4,6 @@ import sys from pathlib import Path import numpy as np -import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) @@ -44,13 +43,15 @@ def test_subtraction_reduces_repeated_tile_energy() -> None: assert after < before -def test_folding_rejects_nondivisible_geometry() -> None: +def test_folding_accepts_nondivisible_geometry() -> None: pixels = np.zeros((63, 64, 3), dtype=np.uint8) - with pytest.raises(ValueError, match="divisible"): - fold_residual_template( - pixels, - tile_height=8, - tile_width=16, - denoise_sigma=1.0, - ) + folded = fold_residual_template( + pixels, + tile_height=8, + tile_width=16, + denoise_sigma=1.0, + ) + + assert folded.shape == (8, 16, 3) + assert np.count_nonzero(folded) == 0