mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-08-31 09:40:38 +02:00
Add calibrated SynthID pixel detector
This commit is contained in:
@@ -30,6 +30,7 @@ removal.
|
||||
| Goal | Command | GPU |
|
||||
| --- | --- | --- |
|
||||
| Find provenance signals and watermarks | `identify` | No |
|
||||
| Detect the SynthID pixel carrier in the calibrated image-size range | `detect-synthid` | No |
|
||||
| Remove known visible AI marks | `visible` | No |
|
||||
| Erase a region you select | `erase` | No |
|
||||
| Strip AI metadata | `metadata` | No |
|
||||
@@ -48,6 +49,7 @@ removal.
|
||||
| Need | Install |
|
||||
| --- | --- |
|
||||
| Metadata inspection and stripping | `remove-ai-watermarks` |
|
||||
| Local SynthID carrier detection in the calibrated size range | `remove-ai-watermarks[pixels]` |
|
||||
| Visible detection and removal | `remove-ai-watermarks[visible]` |
|
||||
| Visible video processing | `remove-ai-watermarks[video]` |
|
||||
| Video SynthID removal | `remove-ai-watermarks[video,diffusion]` |
|
||||
@@ -74,6 +76,19 @@ Inspect an image:
|
||||
remove-ai-watermarks identify image.png
|
||||
```
|
||||
|
||||
Detect the supported SynthID pixel carrier after installing the pixel runtime:
|
||||
|
||||
```bash
|
||||
uv tool install --force "remove-ai-watermarks[pixels]"
|
||||
remove-ai-watermarks detect-synthid image.png
|
||||
```
|
||||
|
||||
This detector is positive-only and limited to one measured carrier family in
|
||||
the [calibrated image-size range](docs/synthid.md#32-how-our-tool-detects-the-supported-carrier).
|
||||
It expects the recovered carrier at its measured 16-pixel sampling scale;
|
||||
arbitrary spatial resampling is not registered. `not_detected` or `unsupported`
|
||||
is not a clean-image guarantee.
|
||||
|
||||
For visible watermark removal, install the pixel dependencies:
|
||||
|
||||
```bash
|
||||
@@ -333,6 +348,9 @@ import remove_ai_watermarks as raiw
|
||||
result, removed = raiw.remove_visible("watermarked.png", "clean.png")
|
||||
print(removed)
|
||||
|
||||
synthid = raiw.detect_synthid("image.png")
|
||||
print(synthid.status, synthid.score)
|
||||
|
||||
provenance = raiw.identify_video("input.mp4")
|
||||
report = raiw.inspect_video_metadata("input.mp4")
|
||||
complete = raiw.remove_video_all("input.mp4", "clean.mp4")
|
||||
|
||||
+20
-2
@@ -14,6 +14,7 @@ defaults. This page focuses on choosing the right command.
|
||||
| Command or signal | Required installation |
|
||||
| --- | --- |
|
||||
| `metadata` and metadata-only `identify` | Default package |
|
||||
| `detect-synthid` and the calibrated-size SynthID pixel signal in `identify` | `remove-ai-watermarks[pixels]` |
|
||||
| Visible signals in `identify` | `remove-ai-watermarks[visible]` (`pixels` is the minimal runtime) |
|
||||
| Open DWT-DCT signals in `identify` | `remove-ai-watermarks[detect]` |
|
||||
| Adobe TrustMark signals in `identify` | `remove-ai-watermarks[trustmark]` |
|
||||
@@ -54,8 +55,25 @@ Metadata only inspection:
|
||||
remove-ai-watermarks identify image.png --no-visible
|
||||
```
|
||||
|
||||
Despite the historical option name, `--no-visible` skips both visible and open
|
||||
invisible pixel detectors. Metadata inspection still runs.
|
||||
Despite the historical option name, `--no-visible` skips all pixel detectors,
|
||||
including the supported SynthID carrier, visible marks, open DWT-DCT, and
|
||||
TrustMark. Metadata inspection still runs.
|
||||
|
||||
## Detect the supported SynthID pixel carrier
|
||||
|
||||
```bash
|
||||
remove-ai-watermarks detect-synthid image.png
|
||||
remove-ai-watermarks detect-synthid image.png --json
|
||||
```
|
||||
|
||||
The command returns one of `detected`, `not_detected`, or `unsupported`. The
|
||||
runtime detector covers one frozen periodic carrier family in the
|
||||
[calibrated image-size range](synthid.md#32-how-our-tool-detects-the-supported-carrier)
|
||||
and needs the `pixels` extra. It never resizes the input and does not register
|
||||
a carrier whose sampling period changed through arbitrary spatial resampling.
|
||||
It is positive-only: `not_detected` means the score stayed below this detector's
|
||||
threshold, while `unsupported` means the image geometry is outside its scope.
|
||||
Neither result proves that another SynthID epoch or payload is absent.
|
||||
|
||||
## Remove known visible marks
|
||||
|
||||
|
||||
@@ -79,8 +79,9 @@ enough yet barely touches flat fills. So the survivors FLIP by content type —
|
||||
choice alone does not guarantee removal.
|
||||
|
||||
**2. Seed non-determinism near threshold.** img2img uses a random seed unless `--seed`
|
||||
is passed, and there is no local SynthID detector to self-verify. The bracelet survived
|
||||
controlnet @0.15 in one run and CLEARED @0.15 in another (same pipeline+strength+res).
|
||||
is passed, and these geometries are outside the current local detector's scope.
|
||||
The bracelet survived controlnet @0.15 in one run and CLEARED @0.15 in another
|
||||
(same pipeline+strength+res).
|
||||
So a single clean run does NOT establish a strength as safe — characterizing a reliable
|
||||
floor needs a seed-repeatability sweep (N runs, varied seed), not one pass.
|
||||
|
||||
@@ -144,8 +145,8 @@ Gemini app; the two payloads are vendor-specific and never cross-checked):
|
||||
but never recovered original identity precisely — every setting traded one problem
|
||||
for another. See `docs/synthid-robust-identity-research-2026-06-08.md`
|
||||
"Empirical follow-up" for the full sweep.
|
||||
- **No local SynthID detector exists** → the service can't self-verify; bake in strength
|
||||
margin and periodic oracle spot-checks.
|
||||
- **No applicable local detector exists for these geometries** → the service
|
||||
can't self-verify; bake in strength margin and periodic oracle spot-checks.
|
||||
- **Lesson:** visual-quality / face-identity recovery does NOT prove removal — only the
|
||||
oracle does, across MULTIPLE content types; never conclude from a partial result (the
|
||||
photoreal-only data first read as "controlnet shields, default removes"; the flat-graphic
|
||||
@@ -298,8 +299,9 @@ attention-slicing; ~1-2 min/image, so a coarse sweep is a sub-hour background ru
|
||||
is needed ONLY for the separate
|
||||
native-large-Gemini (2816 px) case, which OOMs even without a ControlNet (that requires a
|
||||
GPU task). The genuine external dependency is NOT compute but the **manual SynthID oracle**:
|
||||
there is no local SynthID detector, so removal is verified by hand in the Gemini app
|
||||
("Verify with SynthID") per image, regardless of where the diffusion runs.
|
||||
these geometries are outside the current local detector's scope, so removal is
|
||||
verified by hand in the Gemini app ("Verify with SynthID") per image, regardless
|
||||
of where the diffusion runs.
|
||||
|
||||
Runner: **`scripts/controlnet_sweep.py`** (built 2026-06-02) implements exactly this sweep —
|
||||
SDXL base 1.0 + an SDXL-native ControlNet img2img, one output per (control x strength x scale)
|
||||
@@ -441,8 +443,9 @@ shielding risk; defer to a v2 after the single-canny path is dialed in.
|
||||
|
||||
**Hard caveat:** every change that increases preservation (higher scale, denser canny, fuller window,
|
||||
softer edges) marginally REDUCES effective regeneration and so raises the chance the watermark
|
||||
survives -- exactly the shielding failure mode. There is no local SynthID detector, so each tuning
|
||||
change must be re-confirmed on the oracle. These are img2img-context recommendations derived from
|
||||
survives -- exactly the shielding failure mode. These geometries are outside the
|
||||
current local detector's scope, so each tuning change must be re-confirmed on
|
||||
the oracle. These are img2img-context recommendations derived from
|
||||
generation-context sources plus our own measurements; treat the playbook as hypotheses to verify, not
|
||||
settled defaults.
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ application actually uses:
|
||||
|
||||
| Extra | Capability | Automatically includes | Torch or model download |
|
||||
| --- | --- | --- | --- |
|
||||
| `pixels` | Shared BGR array and image-processing runtime | NumPy, headless OpenCV | No |
|
||||
| `pixels` | Shared BGR runtime and calibrated-size SynthID carrier detection | NumPy, headless OpenCV | No |
|
||||
| `heif` | HEIC, HEIF, and AVIF pixel decoding | pillow-heif | No |
|
||||
| `visible` | Visible mark detection, OpenCV inpainting, and manual erasing | `pixels` | No |
|
||||
| `video` | Visible video identification/removal and timestamp preservation | `visible`, PyAV | No |
|
||||
|
||||
@@ -64,12 +64,19 @@ identity or exact texture, and each then runs the same face stage.
|
||||
`qwen-zimage` is the higher fidelity of the two. Both are large, slow, and may
|
||||
still alter small text or difficult faces.
|
||||
|
||||
### Removal cannot be verified locally for proprietary SynthID
|
||||
### Local SynthID detection covers one image-carrier family
|
||||
|
||||
The project has no public local SynthID pixel decoder. It recognizes presence
|
||||
from supported provenance metadata: Google AI C2PA under Google's all-media
|
||||
policy, or current OpenAI C2PA with an explicit watermark action. A missing
|
||||
provenance signal is not a negative pixel verdict.
|
||||
Google does not publish the proprietary SynthID payload decoder. The project
|
||||
ships a positive-only detector for one measured periodic image carrier in a
|
||||
calibrated image-size range, plus provenance-based recognition
|
||||
from Google AI C2PA or current OpenAI C2PA with an explicit watermark action.
|
||||
It does not cover images outside that size range, crop, strong JPEG compression,
|
||||
video, or future carrier epochs. Arbitrary dimensions inside the range are
|
||||
supported only while the recovered carrier retains its measured 16-pixel
|
||||
sampling lattice. A spatial resize changes that period; the runtime does not
|
||||
yet search fractional periods or infer the pre-resize geometry. A `not_detected`
|
||||
or `unsupported` result is not a negative universal verdict, and removal still
|
||||
requires the matching provider oracle for confirmation.
|
||||
|
||||
For important outputs:
|
||||
|
||||
|
||||
@@ -451,7 +451,56 @@ metadata extraction from verdict logic:
|
||||
both extractors reach the same answer. It did not, and the record path silently
|
||||
reported no SynthID for images the file path flagged.
|
||||
- `identify` preserves the path-based API and adds the optional registered
|
||||
visible-mark and open invisible-watermark decoders after extraction.
|
||||
visible-mark, open invisible-watermark, and supported SynthID carrier
|
||||
detectors after extraction.
|
||||
|
||||
### SynthID periodic carrier detector
|
||||
|
||||
[`synthid_detector.py`](../src/remove_ai_watermarks/synthid_detector.py) is the
|
||||
runtime form of the frozen 2048x2048 periodic-tile experiment. It folds a
|
||||
Gaussian high-pass residual modulo 16x16 within a calibrated pixel-count range
|
||||
and compares the normalized RGB tile with the bundled float64 template
|
||||
`assets/synthid_periodic_tile_2048_v1.npz`. Exact multiples use the original
|
||||
reshape-and-mean path; other sizes use count-correct modulo folding,
|
||||
without resize. Channels are filtered and folded sequentially, and partial edge
|
||||
blocks are accumulated without a full-frame padding buffer so the 18-megapixel
|
||||
ceiling does not require multiple three-channel float workspaces. The model hash
|
||||
is pinned by a test, and the unchanged operating threshold is
|
||||
`0.17357069773071196`.
|
||||
|
||||
The direct API returns `detected`, `not_detected`, or `unsupported`; the last is
|
||||
distinct because no resize is performed. Support is based on a calibrated range
|
||||
of 1,000,000 through 18,000,000 decoded pixels. The frozen threshold accepted
|
||||
none of 5,000 public COCO views balanced across every observed target geometry,
|
||||
and none of a separate 5,000-view challenge over 256 generated geometries
|
||||
covering every pair of modulo-16 edge remainders. The original 2048x2048
|
||||
verdicts and exact scores remain unchanged. Runtime matches do not attribute a
|
||||
provider. `identify` adds only positive matches as high-confidence
|
||||
evidence and never turns a local negative into a clean verdict.
|
||||
|
||||
Arbitrary geometry is not the same as arbitrary spatial resampling. On a
|
||||
stratified 80-image fixed-positive sample, one-step resizes at seven nonidentity
|
||||
scales from 0.5 through 1.5 reduced the unchanged 16x16 detector from 80 accepted
|
||||
sources to zero at every scale. Restoring the original geometry recovered 58-80
|
||||
sources, showing that the carrier period scaled with the pixels. A discovery
|
||||
bank that scaled the template to integer periods 8, 10, 12, 14, 18, 20, and 24
|
||||
was promising: a threshold frozen above 3,000 resized COCO controls accepted no
|
||||
view in a 2,000-control final partition and accepted 672 of 800 source-disjoint
|
||||
provider positives. It also stayed below threshold on the tracked OpenAI and
|
||||
Adobe controls. The branch remains research-only because noninteger periods at
|
||||
scales 0.8, 0.9, 1.1, 1.2, and 1.333 collapsed, while separate per-period
|
||||
thresholds accepted five final controls. The runtime therefore keeps only the
|
||||
fixed 16-pixel lattice.
|
||||
|
||||
A follow-up fractional-period probe sampled the 30 strongest template harmonics
|
||||
over a continuous 7.5-24.5 period range. The correct period appeared within
|
||||
0.05 pixels among the top three candidates for 58 of 60 transformed positives.
|
||||
Testing nine neighboring reconstructed geometries then recovered 44 of 60 at
|
||||
the native threshold, compared with an upper bound of 48 when the true source
|
||||
geometry was supplied. The complete search still failed its small frozen
|
||||
control split: a threshold above 250 development controls accepted two of 150
|
||||
final controls. Multiplying the canonical score by spectral-period confidence
|
||||
also accepted two. This branch is not a calibrated runtime fallback.
|
||||
|
||||
### Portable metadata record
|
||||
|
||||
|
||||
@@ -12,6 +12,27 @@ path that still runs on CPU and combines `video` and `diffusion`. Add `heif`
|
||||
independently when path-based pixel APIs must decode HEIC, HEIF, or AVIF. See
|
||||
the complete [feature-extra matrix](installation.md#feature-extras).
|
||||
|
||||
## Detect the supported SynthID carrier
|
||||
|
||||
Install `remove-ai-watermarks[pixels]`, then call the lazy top-level API:
|
||||
|
||||
```python
|
||||
import remove_ai_watermarks as raiw
|
||||
|
||||
result = raiw.detect_synthid("input.png")
|
||||
print(result.status) # "detected" | "not_detected" | "unsupported"
|
||||
print(result.score) # float for a supported image size, otherwise None
|
||||
print(result.threshold) # frozen operating point
|
||||
```
|
||||
|
||||
The detector is positive-only and covers one measured periodic carrier family
|
||||
in the [calibrated image-size range](synthid.md#32-how-our-tool-detects-the-supported-carrier).
|
||||
Arbitrary dimensions are accepted inside that range, but arbitrary spatial
|
||||
resampling can change the carrier period and is not registered. `not_detected`
|
||||
means only that this model did not find its carrier; `unsupported` is kept
|
||||
separate from a negative result. Neither is proof that the image contains no
|
||||
SynthID watermark.
|
||||
|
||||
## Remove visible marks
|
||||
|
||||
Install `remove-ai-watermarks[visible]` before using the visible-removal API.
|
||||
|
||||
@@ -79,6 +79,7 @@ The inspection and stripping code handles signals in these groups:
|
||||
- xAI and Grok EXIF signature fields;
|
||||
- Samsung AI editing markers;
|
||||
- Hugging Face job metadata;
|
||||
- one positive-only SynthID periodic pixel carrier in a calibrated image-size range;
|
||||
- open Stable Diffusion style DWT-DCT watermarks with the `detect` extra;
|
||||
- Adobe TrustMark with the `trustmark` extra.
|
||||
|
||||
@@ -125,12 +126,21 @@ Current pipeline values, both CUDA-only:
|
||||
The `controlnet`, `sdxl`, `qwen` and `default` values were removed. A retired name
|
||||
is rejected at parse time rather than remapped onto a surviving profile.
|
||||
|
||||
SynthID does not have a public local pixel decoder in this project. The tool
|
||||
recognizes presence from supported provenance: Google AI C2PA under Google's
|
||||
all-media watermark policy, and current OpenAI C2PA carrying an explicit
|
||||
`c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action does not
|
||||
assert SynthID. After provenance metadata is removed, a local negative result
|
||||
is still inconclusive.
|
||||
Google does not publish the SynthID payload decoder. This project ships a
|
||||
positive-only detector for one measured periodic image-carrier family in a
|
||||
calibrated image-size range, available through `detect-synthid`
|
||||
and the default pixel pass in `identify` when the `pixels` extra is installed.
|
||||
The unchanged fixed threshold accepted none of the public COCO views in both
|
||||
an observed-geometry challenge and a generated-geometry challenge covering all
|
||||
modulo-16 edge cases. Arbitrary dimensions in the calibrated range are accepted,
|
||||
but the input must retain the measured 16-pixel carrier scale: arbitrary spatial
|
||||
resampling is not registered. The detector does not attribute a provider locally.
|
||||
|
||||
The tool also recognizes presence from supported provenance: Google AI C2PA
|
||||
under Google's all-media watermark policy, and current OpenAI C2PA carrying an
|
||||
explicit `c2pa.watermarked.*` action. Legacy OpenAI C2PA without that action
|
||||
does not assert SynthID. A pixel result of `not_detected` or `unsupported`
|
||||
remains inconclusive for other sizes, epochs, codecs, and payloads.
|
||||
|
||||
For MP4, MOV, and M4V, `video invisible` or the explicit
|
||||
`video all --invisible` option can regenerate the video through a VAE and strip
|
||||
@@ -149,7 +159,7 @@ not a universal clean verdict.
|
||||
|
||||
| Provider or family | Visible | Invisible path | Metadata or provenance |
|
||||
| --- | --- | --- | --- |
|
||||
| Google Gemini | Sparkle | Diffusion regeneration for SynthID | C2PA and related source signals |
|
||||
| Google Gemini | Sparkle | Local positive-only calibrated-size detector; diffusion regeneration | C2PA and related source signals |
|
||||
| Google Veo video | Veo diamond and legacy text | Oracle-certified VAE removal for SynthID | C2PA and related source signals |
|
||||
| OpenAI image generators | None registered | Diffusion regeneration for supported invisible signals | C2PA and generator provenance |
|
||||
| Stable Diffusion and SDXL | None registered | Diffusion regeneration; optional open decoder | Embedded parameters and text metadata |
|
||||
|
||||
@@ -1074,10 +1074,10 @@ provider-specific expert for the supported 1536x2816 carrier epoch. Identity
|
||||
and bounded translation views use the frozen phase and support thresholds;
|
||||
unsupported geometry, insufficient carrier magnitude, and ambiguous phase
|
||||
return `abstain`. Vendor attribution may select the expert that supplied
|
||||
accepted evidence, but it must not turn an abstention into a provider label. The next
|
||||
calibration gate still requires at least 3,000 native-support negatives,
|
||||
same-provider oracle negatives, matched non-target solid outputs, and a new
|
||||
temporal positive that influenced neither profile nor threshold.
|
||||
accepted evidence, but it must not turn an abstention into a provider label.
|
||||
The next calibration gate still requires at least 3,000 native-support
|
||||
negatives, same-provider oracle negatives, matched non-target solid outputs,
|
||||
and a new temporal positive that influenced neither profile nor threshold.
|
||||
|
||||
### 2026-08-10: 2048 periodic-tile detector
|
||||
|
||||
@@ -1151,6 +1151,323 @@ family, but the two cross-source carrier matches prohibit a stronger vendor
|
||||
claim until an oracle distinguishes direct provider output from shared-backend
|
||||
output.
|
||||
|
||||
A pixel-space ablation then tested whether the frozen tile merely predicted the
|
||||
local scores or controlled them. At twice the train-median tile norm, aligned
|
||||
subtraction changed the fixed-tile decision from 29 of 30 accepted originals to
|
||||
zero and the independently fitted sparse-phase decision from 27 of 30 to zero.
|
||||
The median fidelity was 53.74 dB PSNR and 0.99681 SSIM. Cyclically shifting the
|
||||
same tile by one row and column left one phase acceptance, while a seeded
|
||||
zero-mean random tile orthogonal to the learned template left 13; their median
|
||||
PSNRs were within 0.13 dB of the aligned edit. For both local representations,
|
||||
the aligned edit reduced the score more than either control on all 30 paired
|
||||
images, with a two-sided sign-test p-value of 1.86e-9 for each comparison.
|
||||
The same aligned edit reversed both local decisions on each of the two disputed
|
||||
cross-source matches, while the orthogonal control left both phase decisions
|
||||
accepted. That two-item result is descriptive but makes an accidental threshold
|
||||
crossing less likely; it still cannot distinguish a shared carrier from direct
|
||||
provider output.
|
||||
|
||||
This is the strongest local causal evidence for the 16x16 mechanism, but the
|
||||
strength was selected after inspecting this locked test and is therefore
|
||||
discovery-only. The shifted control also suppressed the phase representation
|
||||
substantially, so local score reversal cannot certify signal removal. A matching
|
||||
provider oracle must still compare untouched sources, aligned candidates, and
|
||||
norm-matched controls before any removal claim. The reproducible local harness
|
||||
is `scripts/synthid_periodic_tile_ablation.py`; its report records input and
|
||||
model hashes without writing derivative images.
|
||||
|
||||
### 2026-08-10: confirmatory oracle-batch readiness
|
||||
|
||||
The corpus audit found no new 2048x2048 positive collected after the tile rule
|
||||
and subtraction strength were frozen. Existing test images cannot be renamed as
|
||||
a temporal holdout, so the confirmatory oracle batch remains blocked on new
|
||||
provider output at the supported native geometry.
|
||||
|
||||
The batch mechanics are ready in `scripts/synthid_oracle_batch.py`. For each new
|
||||
source, it preregisters five requests in a fixed order: untouched source,
|
||||
lossless re-encode, aligned subtraction, cyclic-shifted subtraction, and an
|
||||
orthogonal norm-matched tile. It refuses to write media inside the repository,
|
||||
records artifact and decoded-pixel hashes, pins both model artifacts and the
|
||||
matching provider, and detects any post-registration byte change. Oracle results
|
||||
live in a separate manifest-bound file so the preregistration remains immutable;
|
||||
the evaluator requires complete ordered results, verbatim responses,
|
||||
timezone-aware timestamps, and separate SynthID and C2PA verdicts.
|
||||
|
||||
A causal success requires the source, re-encode, shifted, and orthogonal views
|
||||
all to remain `detected` while only the aligned view becomes `not_detected`.
|
||||
`Indeterminate` and refusal remain indeterminate. A one-source dry run on an old
|
||||
test positive verified all five artifacts and the local scoring path, but is
|
||||
pipeline validation only and contributes no confirmatory evidence.
|
||||
|
||||
### 2026-08-10: first confirmatory oracle attempt
|
||||
|
||||
Four new 2048x2048 Gemini outputs were generated after the local rule and
|
||||
strength were frozen, then registered as a 20-request batch. The manifest and
|
||||
all 20 artifacts passed hash verification before submission. The first source
|
||||
group produced these ordered results:
|
||||
|
||||
| Role | Local tile/phase decision | Provider result |
|
||||
| --- | --- | --- |
|
||||
| source | detected / detected | SynthID indeterminate; Google C2PA present |
|
||||
| re-encode control | detected / detected | Google AI signal detected |
|
||||
| aligned subtraction | detected / not detected | Google AI signal detected |
|
||||
| shifted control | detected / detected | Google AI signal detected |
|
||||
| orthogonal control | detected / detected | refused: verification quota exceeded |
|
||||
|
||||
The quota response instructed the account to wait 24 hours. No adaptive retry
|
||||
was made, and the remaining 15 preregistered requests were not submitted. The
|
||||
batch cannot receive its preregistered aggregate verdict until complete ordered
|
||||
results exist.
|
||||
|
||||
Even incomplete, this attempt rejects the frozen removal recipe for the first
|
||||
temporal source: aligned subtraction crossed the phase threshold but remained
|
||||
above the tile threshold and did not clear the provider oracle. The remaining
|
||||
sources can still measure transfer and disagreement between the two local
|
||||
experts, but they cannot turn this first aligned result into a universal
|
||||
pixel-only removal success.
|
||||
|
||||
### 2026-08-10: positive-only runtime detector
|
||||
|
||||
Removal is deferred while the transferred tile signal is exposed as a bounded
|
||||
detector. `src/remove_ai_watermarks/synthid_detector.py` loads the frozen model
|
||||
as a bundled pickle-free runtime asset and returns `detected`, `not_detected`,
|
||||
or `unsupported` without resizing input. The direct API and
|
||||
`detect-synthid` CLI initially covered only native 2048x2048 images; `identify`
|
||||
consumes a positive match as high-confidence evidence but never treats a
|
||||
negative or unsupported result as proof of absence.
|
||||
|
||||
The operating point and model are unchanged from the locked experiment. The
|
||||
detector accepted all four post-freeze Gemini outputs, including the source
|
||||
whose frozen phase score missed, while retaining the prior 29-of-30 locked-test
|
||||
sensitivity and zero accepted validation/test negatives. The unresolved two
|
||||
external-source matches remain the reason a runtime positive names the carrier
|
||||
but does not attribute a provider.
|
||||
|
||||
### 2026-08-10: calibrated native-geometry extension
|
||||
|
||||
The frozen 16x16 template and threshold transfer across the common native
|
||||
geometries in the provider-positive evidence. Non-divisible image dimensions
|
||||
use direct modulo folding with per-cell sample counts; divisible dimensions,
|
||||
including 2048x2048, retain the original numerical path. All prior 2048x2048
|
||||
records were replayed and matched exactly, including floating-point scores.
|
||||
|
||||
The runtime registry was expanded only after the unchanged threshold accepted
|
||||
none of 60,000 public COCO challenge views, 3,000 at each of 20 target
|
||||
geometries. This is a geometry extension of the same positive-only carrier
|
||||
expert, not an OpenAI pixel detector or a proprietary payload decoder.
|
||||
|
||||
### 2026-08-11: calibrated image-size range
|
||||
|
||||
The fixed template was then evaluated on every provider-positive image in the
|
||||
local evidence set rather than only its common geometries. It accepted 3,928 of
|
||||
4,698 images across 757 exact geometries. Sensitivity separated by pixel count:
|
||||
1,987 of 2,021 images at or above three megapixels crossed the threshold, while
|
||||
1,940 of 2,672 images from one through two megapixels did. This establishes a
|
||||
carrier-family boundary, not universal SynthID recall: explicit C2PA watermark
|
||||
actions also occur below threshold, and three strong carriers use a different
|
||||
cyclic phase.
|
||||
|
||||
Two public-image geometry challenges tested whether geometry itself creates
|
||||
false matches. The first balanced 5,000 COCO images across all 757 observed
|
||||
geometries, with every geometry present in both development and final partitions;
|
||||
the maximum fixed score was 0.12549 and none crossed the unchanged 0.17357
|
||||
threshold. The second transformed the same 5,000 source images at 256 generated
|
||||
geometries from one through 18 megapixels, covering every width/height remainder
|
||||
pair modulo 16. Its maximum was 0.16387 and again none crossed the threshold.
|
||||
Runtime support therefore uses that challenged pixel-count interval instead of
|
||||
an enumerated geometry registry, still without resizing.
|
||||
|
||||
A cyclic-registration branch was rejected after the public-image challenge.
|
||||
At a low threshold it recovered 130 fixed-phase misses but false-fired on
|
||||
tracked OpenAI and Adobe controls. Raising the threshold above all development
|
||||
controls recovered only 23 in-range provider positives while adding 84
|
||||
cross-source matches beyond the fixed detector. Three strong shifted Google
|
||||
carriers remain a documented miss rather than weakening the runtime rule.
|
||||
|
||||
Top-level source labels were not treated as watermark-negative truth. Of 237
|
||||
non-Google-labeled corpus images crossing the fixed threshold, 81 retained
|
||||
SynthID provenance. High-scoring Adobe manifests explicitly named Gemini Flash
|
||||
as the remote provider, and perceptual matching found direct Google derivatives
|
||||
under editing or relabeling categories. The remaining source labels cannot prove
|
||||
absence of the carrier, so they constrain provider attribution but do not justify
|
||||
raising the pixel threshold.
|
||||
|
||||
### 2026-08-11: spatial-resampling boundary
|
||||
|
||||
Arbitrary decoded geometry did not imply arbitrary resize robustness. A
|
||||
stratified sample of 80 fixed-positive provider images was rescaled in memory at
|
||||
0.5, 0.625, 0.75, 0.875, 1.125, 1.25, and 1.5. The unchanged 16x16 detector
|
||||
accepted none of the 80 images at every nonidentity scale. Restoring each view
|
||||
to its source dimensions recovered 58-80, which isolates the failure to the
|
||||
scaled sampling lattice rather than simple information destruction.
|
||||
|
||||
Scaling the template and modulo fold to the corresponding integer periods 8,
|
||||
10, 12, 14, 18, 20, and 24 recovered 73-80 of the discovery images at the
|
||||
native threshold. Searching all eight periods raised control scores, so a
|
||||
single threshold was frozen above the maximum of 3,000 deterministically
|
||||
resized COCO development views. It accepted none of 2,000 final controls and
|
||||
672 of 800 source-disjoint provider positives, with 77-88 accepted at each
|
||||
scale. Six tracked OpenAI and Adobe controls remained below the frozen threshold.
|
||||
|
||||
This did not generalize to arbitrary scaling. Factors 0.8, 0.9, 1.1, 1.2, and
|
||||
1.333 create noninteger carrier periods; rounding the period accepted zero of
|
||||
80 images in the fixed-phase branch at every factor, and cyclic registration
|
||||
accepted only 1-16. Period-specific thresholds improved sensitivity but accepted
|
||||
five of the 2,000 final controls. The integer-period bank is therefore retained
|
||||
as a research result, not added to the runtime. A universal resize branch needs
|
||||
fractional-period registration or reliable reconstruction of the pre-resize
|
||||
geometry, followed by a new multiple-search calibration.
|
||||
|
||||
A discovery follow-up tested that fractional-period direction with the 30
|
||||
strongest template harmonics. A continuous 7.5-24.5 spectral search placed the
|
||||
true period within 0.05 pixels among its top three candidates for 58 of 60
|
||||
transformed positives. Searching nine neighboring integer reconstruction
|
||||
geometries recovered 44 of 60 at the native threshold; even the true source
|
||||
geometry recovered only 48, so resampling damage sets a separate upper bound.
|
||||
The full search then failed a frozen 250-development/150-final COCO challenge:
|
||||
the calibration threshold accepted two final controls. A preselected product
|
||||
of canonical correlation and spectral confidence increased positive
|
||||
acceptance from 49 to 52 of 60 at its development threshold but accepted the
|
||||
same two final controls. This fractional FFT baseline is rejected for runtime
|
||||
use; better period estimation alone does not solve multiple-search specificity.
|
||||
|
||||
### 2026-08-10: OpenAI periodic-carrier challenge
|
||||
|
||||
The OpenAI track repeated the Google carrier method without runtime provenance
|
||||
features. C2PA selected the research labels, but every candidate score consumed
|
||||
only decoded RGB pixels. The temporally split corpus contained 4,615 OpenAI
|
||||
training positives, 1,280 validation positives, 1,364 development-test
|
||||
positives, 8,668 difficult external-source controls, and 3,000 COCO controls.
|
||||
The development test is not called a locked holdout because earlier
|
||||
representations had already exposed it.
|
||||
|
||||
High-pass residuals were folded modulo 8x8, 16x16, and 32x32 and normalized as
|
||||
in the Google fixed-tile expert. A single 16x16 OpenAI template accepted only
|
||||
one of 1,280 validation positives at the threshold above all 2,287 validation
|
||||
controls. A 16-template geometry bank improved that to 12 positives but
|
||||
accepted one of 3,372 development-test controls. The corresponding
|
||||
development-test sensitivity was 9 of 1,364. The frozen Google template did
|
||||
not transfer: at its unchanged threshold it accepted 14 validation positives
|
||||
and 572 validation controls.
|
||||
|
||||
The apparent native-size signal was a confound. At 1024x1536, 91.6% of the
|
||||
learned template energy projected onto a 2x2 periodic lattice, compared with
|
||||
5.6% for the frozen Google template. Firefly, China AIGC-labeled, local
|
||||
diffusion, and unidentified hosted-generator controls reached correlations of
|
||||
0.92-0.98 against the uncorrected OpenAI template. This is operational evidence
|
||||
for a shared decoder or upscaler lattice, not an OpenAI watermark carrier.
|
||||
|
||||
Projecting the 2x2 nuisance lattice out of every 16x16 fold isolated a weaker
|
||||
epoch-specific residual. The 1024x1536 expert accepted 5 of 55 validation and
|
||||
10 of 51 development-test positives while accepting none of the full
|
||||
validation or development-test controls. Nine of the ten development-test
|
||||
matches occurred from July 12 through July 14; later dates contributed only
|
||||
one of 27. The 1536x1024 expert accepted 6 of 37 validation and 2 of 51
|
||||
development-test positives with no development-test controls. Amplitude,
|
||||
active-support, channel-consensus, payload-invariant power, and spherical
|
||||
multi-template branches did not improve this frontier without false positives.
|
||||
|
||||
The OpenAI branch therefore remains research-only and is not added to the
|
||||
runtime detector. The next valid training epoch requires same-content
|
||||
OpenAI-oracle `detected` and `not_detected` pairs, or provider-authorized decoder
|
||||
labels, followed by a fresh temporal holdout. Source provenance alone cannot
|
||||
distinguish a true watermark carrier from the measured generator lattice.
|
||||
|
||||
### 2026-08-10: OpenAI content-dependent decoder challenge
|
||||
|
||||
The periodic result did not exhaust the neural-watermark hypothesis. OpenAI's
|
||||
ChatGPT Images 2.0 system card describes its watermark as content-specific, and
|
||||
the SynthID-Image paper says the encoder is content-dependent by design. The
|
||||
paper also separates watermark detection from payload recovery and evaluates
|
||||
the external SynthID-O model at its preferred 512x512 input size. These facts
|
||||
support a learned full-field decoder, but they do not establish that OpenAI and
|
||||
Google deploy identical model weights, payloads, or pixel spectra.
|
||||
|
||||
A second corpus audit found 261 same-provider candidate controls. Their signed
|
||||
provenance contained `created` and `converted` actions without the
|
||||
`c2pa.watermarked.unbound` action, but a subsequent software-agent audit showed
|
||||
that they do not all come from the positive generation epoch. Only 118 name
|
||||
`gpt-image 2.0`: 97 training, 16 validation, and five development-test images.
|
||||
Another 108 name an earlier version and 35 do not expose a usable version. The
|
||||
full 261-image split remains 217/26/18. These are useful confound controls, but
|
||||
even the exact-version subset is action-derived rather than a provider-oracle
|
||||
negative label.
|
||||
|
||||
Two pixel-only native-resolution experts were trained with the mixed-version
|
||||
control set. A
|
||||
local residual CNN accepted 132 of 1,280 validation positives and 149 of 1,364
|
||||
development-test positives, with zero same-provider or COCO controls in both
|
||||
splits and two of 1,373 difficult development-test controls. A separate
|
||||
spectral, autocorrelation, bit-plane, and residual-error expert accepted 175
|
||||
validation and 167 development-test positives with no accepted controls. A
|
||||
validation-selected linear combination accepted 243 positives in each split,
|
||||
with no accepted validation or development-test control. The zero-error
|
||||
development-test bound across 3,391 controls is 0.0883% at one-sided 95%
|
||||
confidence, but the same-provider subset contains only 18 images and therefore
|
||||
does not independently establish provider specificity.
|
||||
|
||||
The exact-version audit did not invalidate the native score counts, but it
|
||||
weakened their attribution. None of the 16 validation or five development-test
|
||||
`gpt-image 2.0` controls crossed the residual, forensic, ensemble, canonical,
|
||||
or JPEG-equalized thresholds. That is directionally useful, but five temporal
|
||||
test controls cannot establish deployment specificity. Perceptual-hash and
|
||||
local CLIP searches compared all 118 exact-version controls with 7,259
|
||||
positives. They recovered a few near-content matches but no clean before/after
|
||||
watermark pair. Sub-pixel affine alignment left content, edge, and photometric
|
||||
changes in the residual, while 8x8, 16x16, and 32x32 folded residual
|
||||
correlations varied in sign and magnitude. Those candidates cannot be used as
|
||||
causal watermark differences.
|
||||
|
||||
Transformation challenges rejected the combined native expert as a SynthID
|
||||
decoder. Of its 243 accepted development-test positives, 121 remained accepted
|
||||
after a 5% crop round trip, six after a 75% resize round trip, and none after a
|
||||
JPEG-95 round trip. No transformed same-provider control was accepted. The
|
||||
codec fragility is inconsistent with treating the score as the documented
|
||||
robust watermark without further evidence.
|
||||
|
||||
The 512x512 branch also failed the held-out specificity gate. Its selected
|
||||
model accepted 238 validation and 194 development-test positives, but also
|
||||
accepted two of 18 same-provider, four of 1,373 difficult, and nine of 2,000
|
||||
COCO development-test controls. Finally, a CNN trained and evaluated only
|
||||
after applying the same JPEG-95 round trip to every positive and control
|
||||
accepted eight validation positives and six development-test positives while
|
||||
accepting five of 2,000 development-test COCO controls. Equalizing the codec
|
||||
therefore removed nearly all of the apparent signal instead of revealing a
|
||||
robust hidden decoder feature.
|
||||
|
||||
An independent three-CNN OpenAI surrogate was also evaluated from its
|
||||
published weights, without adopting its code or claims. At the published 0.5
|
||||
threshold its ensemble marked 82.0% of COCO validation controls and 83.4% of
|
||||
COCO development-test controls as watermarked. It also marked 56.3% of
|
||||
difficult development-test controls and 15 of 16 exact-version OpenAI
|
||||
validation controls, compared with 87.3% of OpenAI validation positives. The
|
||||
repository does not publish its training corpus or logs, and its implementation
|
||||
applies supposedly shared pair augmentation independently. This result rejects
|
||||
the model as a SynthID decoder and demonstrates why same-content-looking
|
||||
regenerations are not sufficient negative labels.
|
||||
|
||||
A final transform-augmented full-field branch tested the closest practical
|
||||
approximation to the documented neural decoder. A 512x512 CNN consumed both
|
||||
RGB and a fixed high-pass residual and trained with random crop/resize,
|
||||
JPEG 70-100, horizontal flip, and mild color transforms. Training sampling
|
||||
balanced watermarked positives, difficult controls, and the exact-version
|
||||
OpenAI candidate controls. Selection required a score above every control for
|
||||
both native and JPEG-95 validation views. All four fixed training epochs
|
||||
accepted zero of 500 selection positives. On the full validation set the
|
||||
frozen model accepted zero of 1,280 OpenAI positives in native, JPEG-95,
|
||||
75%-resize, and 5%-crop views, while accepting up to two unrelated controls.
|
||||
On development test it accepted zero of 1,364 OpenAI positives in native,
|
||||
JPEG-95, and resize views. The crop view accepted one OpenAI positive and one
|
||||
of 1,373 difficult controls. Transform augmentation therefore did not recover
|
||||
a usable weaker OpenAI signal.
|
||||
|
||||
The measured OpenAI-native separation is best classified as an export or
|
||||
rendering noiseprint, not a validated SynthID detector. None of these models is
|
||||
added to runtime. Transform-augmented training has now also failed. A
|
||||
defensible next epoch needs provider-authorized labels or clean same-content
|
||||
before/after watermark pairs, plus a fresh temporal holdout that has not
|
||||
influenced representation or threshold selection.
|
||||
|
||||
## Decision record
|
||||
|
||||
The program has four possible honest outcomes per provider:
|
||||
@@ -1184,5 +1501,6 @@ Only after this milestone should implementation of D3 and D4 begin.
|
||||
## Primary sources
|
||||
|
||||
- OpenAI, [Content provenance](https://developers.openai.com/api/docs/guides/content-provenance).
|
||||
- OpenAI, [ChatGPT Images 2.0 system card](https://deploymentsafety.openai.com/chatgpt-images-2-0/automated-evaluations-and-adversarial-testing).
|
||||
- Google, [Verify AI-generated images, videos, and audio](https://support.google.com/gemini/answer/16722517?hl=en).
|
||||
- Gowal et al., [SynthID-Image: Image watermarking at internet scale](https://arxiv.org/abs/2510.09263).
|
||||
|
||||
+101
-12
@@ -308,6 +308,30 @@ limited JPEG and crop robustness. The pickle-free research implementation is
|
||||
`scripts/synthid_periodic_tile_probe.py`; exact evidence and caveats are in the
|
||||
[`2048 periodic-tile experiment`](synthid-detector-removal-plan.md#2026-08-10-2048-periodic-tile-detector).
|
||||
|
||||
An aligned-subtraction ablation strengthened the mechanism finding without
|
||||
clearing the oracle gate. At a discovery-selected amplitude, it reversed both
|
||||
the fixed-tile and independently fitted phase decisions on all 30 test images
|
||||
at a median 53.74 dB PSNR and 0.99681 SSIM. The aligned edit reduced both scores
|
||||
more than cyclic-shifted and orthogonal random tile controls on every paired
|
||||
image. A shifted tile nevertheless suppressed the phase score enough to leave
|
||||
only one accepted image, so these local reversals remain surrogate evidence,
|
||||
not verified SynthID removal. The exact controls and caveats are recorded in
|
||||
the linked experiment section.
|
||||
|
||||
The immutable oracle-batch and result evaluator are implemented. On 2026-08-10,
|
||||
four new 2048x2048 Gemini images were generated after the rule was frozen and
|
||||
registered as a 20-request confirmatory batch. The first source group exhausted
|
||||
the account's verification quota after five requests. The untouched source
|
||||
returned Google C2PA Content Credentials without a separate SynthID verdict;
|
||||
the lossless re-encode, aligned subtraction, and cyclic-shifted control all
|
||||
still returned a Google AI signal. The orthogonal control was refused because
|
||||
the quota had been exceeded, and the remaining 15 requests were not submitted.
|
||||
This incomplete run is already negative evidence for the frozen pixel-only
|
||||
recipe: the aligned candidate cleared the phase detector but remained positive
|
||||
under the tile detector and the provider oracle. The local carrier expert
|
||||
therefore ships only as a positive-only, exact-geometry detector; it is not a
|
||||
universal SynthID detector or a remover.
|
||||
|
||||
A controlled study (June 2026, clean v0.8.6 with text/face protection OFF,
|
||||
native resolution on this repo's default SDXL pipeline) measured the minimum
|
||||
img2img strength that removes the SynthID pixel watermark, verified per image on
|
||||
@@ -402,7 +426,7 @@ diffusion prior."
|
||||
|
||||
## 3. Detectability and verifier access
|
||||
|
||||
### 3.1 No public local decoder
|
||||
### 3.1 No public payload decoder
|
||||
|
||||
The SynthID decoder is proprietary and not released:
|
||||
|
||||
@@ -411,8 +435,8 @@ The SynthID decoder is proprietary and not released:
|
||||
> available to trusted testers."
|
||||
> -- Gowal et al., arXiv:2510.09263
|
||||
|
||||
There are no released decoder weights and no reproducible algorithm for local
|
||||
detection. Google provides verification in Gemini and a limited SynthID Detector
|
||||
There are no released payload-decoder weights or public algorithm. Google
|
||||
provides verification in Gemini and a limited SynthID Detector
|
||||
portal. OpenAI now documents a synchronous Content Provenance API whose image
|
||||
response contains separate C2PA and SynthID outcomes. That API is a remote,
|
||||
OpenAI-scoped verifier, not a local decoder. Its documentation also says not to
|
||||
@@ -425,12 +449,73 @@ Google's SynthID Detector service is:
|
||||
> professionals" on a waitlist
|
||||
> -- deepmind.google/models/synthid/
|
||||
|
||||
The external variant SynthID-O is available "through partnerships" only. Our
|
||||
tool does not currently detect SynthID pixels locally. The gated research path
|
||||
for determining whether that can change is documented in
|
||||
The external variant SynthID-O is available "through partnerships" only. This
|
||||
project instead detects one empirically recovered periodic carrier family in a
|
||||
calibrated image-size range. It does not decode the proprietary payload or
|
||||
generalize that local signal to unsupported sizes,
|
||||
codecs, video, or future epochs. The evidence and gates are documented in
|
||||
[`synthid-detector-removal-plan.md`](synthid-detector-removal-plan.md).
|
||||
|
||||
### 3.2 How our tool recognizes SynthID from provenance
|
||||
### 3.2 How our tool detects the supported carrier
|
||||
|
||||
`remove-ai-watermarks detect-synthid image.png` folds the image residual modulo
|
||||
16x16 and compares it with a frozen float64 template. It evaluates only native
|
||||
input, without resize. Exact-multiple dimensions retain the original folding
|
||||
path; non-divisible dimensions use count-correct modulo folding. The model and
|
||||
threshold remain frozen from the 2048x2048 experiment. The fixed threshold
|
||||
accepted none of 5,000 public COCO views balanced across every observed target
|
||||
geometry. A separate 5,000-view challenge used 256 generated geometries from one
|
||||
through 18 megapixels and covered every pair of width/height remainders modulo
|
||||
16; it also produced no accepted view. Runtime support is therefore the
|
||||
challenged interval of 1,000,000 through 18,000,000 decoded pixels rather than
|
||||
an enumerated width-by-height registry. The original 2048x2048 scores remain
|
||||
exactly unchanged.
|
||||
|
||||
This geometry support does not imply arbitrary resize robustness. The fixed
|
||||
carrier has a 16-pixel sampling lattice. In a stratified 80-image positive
|
||||
sample, direct detection fell from 80 accepted originals to zero after each of
|
||||
seven nonidentity resizes from 0.5 through 1.5. Scaling the template and folding
|
||||
period to matching integers recovered the signal, and a conservative threshold
|
||||
above 3,000 resized COCO development controls accepted 672 of 800 source-disjoint
|
||||
provider positives with no acceptance in 2,000 final controls. That branch is
|
||||
not shipped: noninteger periods from ordinary scale factors collapsed, and
|
||||
less conservative per-period thresholds accepted five final controls. The
|
||||
runtime therefore detects arbitrary decoded dimensions only when the carrier
|
||||
retains its measured 16-pixel scale.
|
||||
|
||||
A positive result identifies the carrier but does not attribute a provider.
|
||||
Provider identity still comes from provenance.
|
||||
|
||||
The command reports `not_detected` separately from `unsupported`. Both are
|
||||
inconclusive outside the measured carrier family and calibrated image-size range.
|
||||
|
||||
The same modulo-folding method has been tested separately on a large,
|
||||
temporally split OpenAI-labeled corpus. Its strongest native-size template was
|
||||
dominated by a generic 2x2 generator lattice that also appeared in multiple
|
||||
non-OpenAI controls. Removing that nuisance component left a sparse,
|
||||
time-limited signal with inadequate sensitivity. OpenAI pixel detection is
|
||||
therefore not part of the runtime expert.
|
||||
|
||||
Learned residual, forensic, ensemble, and canonical 512x512 representations
|
||||
were also tested against 261 same-provider candidate controls. A later
|
||||
software-agent audit found that only 118 of those controls explicitly name
|
||||
`gpt-image 2.0`; the rest come from earlier or unknown versions. The strongest
|
||||
native ensemble accepted 243 of 1,364 development-test positives with no
|
||||
accepted controls, but accepted none of those positives after a JPEG-95 round
|
||||
trip. A model trained after equalizing every image through JPEG-95 accepted
|
||||
only six development-test positives and five COCO controls. The apparent
|
||||
native signal is therefore treated as an export noiseprint rather than a
|
||||
validated robust watermark. A local CLIP search over the exact-version subset
|
||||
found no clean same-content before/after pair, and a published third-party CNN
|
||||
surrogate mislabeled 83.4% of held-out COCO controls at its stated threshold.
|
||||
A transform-augmented 512x512 RGB-plus-residual CNN then accepted zero of 1,364
|
||||
development-test positives in native, JPEG-95, and resize views; its crop view
|
||||
accepted one positive and one difficult control. Detailed counts and rejected
|
||||
alternatives are in
|
||||
the [`OpenAI periodic-carrier challenge`](synthid-detector-removal-plan.md#2026-08-10-openai-periodic-carrier-challenge)
|
||||
and [`OpenAI content-dependent decoder challenge`](synthid-detector-removal-plan.md#2026-08-10-openai-content-dependent-decoder-challenge).
|
||||
|
||||
### 3.3 How our tool recognizes SynthID from provenance
|
||||
|
||||
We recognize SynthID indirectly from supported C2PA evidence; this is not a
|
||||
pixel watermark decode. Google states that all media generated by its tools is
|
||||
@@ -451,7 +536,7 @@ This is why:
|
||||
- A quiet `identify` output is not proof that SynthID was removed -- it only
|
||||
means the metadata signal is gone.
|
||||
|
||||
### 3.3 Oracle scope: each vendor detects only their own
|
||||
### 3.4 Oracle scope: each vendor detects only their own
|
||||
|
||||
OpenAI's current Content Provenance API documentation says it checks supported
|
||||
OpenAI signals and is not a general-purpose AI detector. Google's current Gemini
|
||||
@@ -470,7 +555,7 @@ A Google-SynthID image reads clean on openai.com/verify. An OpenAI image reads
|
||||
clean in the Gemini oracle. They are different payloads within the same
|
||||
framework.
|
||||
|
||||
### 3.4 Video verification and attack harness
|
||||
### 3.5 Video verification and attack harness
|
||||
|
||||
Gemini's built-in verification flow reports whether and where it detects Google
|
||||
SynthID in a video. This remains a proprietary oracle: invoke `@synthid`, use
|
||||
@@ -699,7 +784,8 @@ photoreal, `sdxl` on flat graphics, the §5.1 content-x-pipeline table), BUT on
|
||||
hard case (flat fills) `sdxl` is the WEAKER remover (plain img2img barely perturbs a
|
||||
flat region at low strength), so it needs AT LEAST controlnet's strength -- the
|
||||
certified floor is therefore the right floor for `sdxl` too. This is a MARGIN argument
|
||||
for `sdxl`, not a separate certification (no local SynthID detector to self-verify).
|
||||
for `sdxl`, not a separate certification (the tested geometries are outside the
|
||||
current local detector's scope).
|
||||
The higher strength costs little quality where it matters, because `controlnet` is now
|
||||
the default pipeline, so `sdxl` is reached only via an explicit `--pipeline sdxl` (a
|
||||
deliberate opt-down), where over-regeneration has no faces/text to damage.
|
||||
@@ -810,7 +896,7 @@ random (unset) seed differed between runs. So **0.15 is the borderline floor for
|
||||
controlnet photoreal, not a robust guarantee**: at the threshold the same
|
||||
image+settings can pass or fail run-to-run. img2img runs with `seed=None` (random)
|
||||
unless `--seed` is passed, so a removal SERVICE gets a coin-flip near threshold and
|
||||
has no local SynthID detector to self-verify.
|
||||
has no applicable local SynthID detector at these geometries to self-verify.
|
||||
|
||||
**Controlnet strength ladder on the two photoreal images (oracle, `--auto`,
|
||||
`--max-resolution 1536`):**
|
||||
@@ -825,7 +911,7 @@ has no local SynthID detector to self-verify.
|
||||
non-deterministic borderline); both photoreal survivors cleared at 0.20. Honest
|
||||
caveat: 0.20 is one confirming run WITH margin, not an N-run repeatability proof --
|
||||
for a removal service, add a little more margin or validate repeatability, since
|
||||
there is no local SynthID detector to self-check. **Implications:** (1) the
|
||||
these geometries are outside the current local detector's scope. **Implications:** (1) the
|
||||
content×pipeline table above conflates a borderline/non-deterministic 0.15 result
|
||||
with deterministic content behavior -- the photoreal-survives-controlnet effect is
|
||||
solid at 0.10 but at 0.15 it is near-threshold noise; (2) for reliable removal pick
|
||||
@@ -910,3 +996,6 @@ reproducible verification requires a fixed seed.
|
||||
8. Jiang et al. (2025). **VideoMarkBench: Benchmarking Robustness of Video
|
||||
Watermarking.** arXiv:2505.21620.
|
||||
https://arxiv.org/abs/2505.21620
|
||||
|
||||
9. OpenAI. **ChatGPT Images 2.0 system card.**
|
||||
https://deploymentsafety.openai.com/chatgpt-images-2-0/automated-evaluations-and-adversarial-testing
|
||||
|
||||
@@ -64,7 +64,12 @@ payloads. Removal remuxes either container through ffmpeg with stream copy.
|
||||
may carry IPTC metadata but no registered C2PA or pixel watermark. The open
|
||||
DWT-DCT decoder only applies when the producing pipeline actually ran its
|
||||
encoder and the carrier remains decodable.
|
||||
- **Invisible but NOT locally detectable (proprietary, API/oracle only — same wall as SynthID):** Amazon Titan Image Generator + Nova Canvas (Bedrock `DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026), NVIDIA Cosmos (SynthID video). No local detector possible; treat like SynthID.
|
||||
- **Invisible but NOT locally detectable (proprietary, API/oracle only):**
|
||||
Amazon Titan Image Generator + Nova Canvas (Bedrock
|
||||
`DetectGeneratedContent` API), Kakao (new SynthID image adopter, May 2026),
|
||||
and NVIDIA Cosmos (SynthID video). No public payload decoder is available;
|
||||
unlike the project's calibrated-size SynthID carrier expert, these signals
|
||||
have no measured local detector here.
|
||||
- **C2PA 2.4 "Durable Content Credentials" (April 2026; verified against the spec) raise the bar for metadata stripping.** 2.4 defines soft bindings (an invisible watermark or a content fingerprint) plus a server-side manifest repository and a new `c2pa.repository-receipt` assertion. Per the spec: "if a C2PA manifest is removed from an asset, but a copy of that manifest remains in a provenance store elsewhere, the manifest and asset may be matched using available soft bindings." So our local `metadata --remove` deletes the *embedded* manifest, but a fingerprint/watermark soft binding can still re-link the image to its manifest in a repository server-side. Stripping the file is becoming necessary-but-not-sufficient against durable provenance. (Our parsers target the stable embedded-manifest format documented in C2PA 2.1 §11; that format is unchanged in 2.4 -- the new pieces are repository/soft-binding infra, not the on-file box layout, so no parser change is implied.) Spec: https://spec.c2pa.org/specifications/specifications/2.4/specs/C2PA_Specification.html We now READ the soft-binding `alg` (`C2PA_SOFT_BINDINGS` / `soft_binding_vendors_in`) to name the forensic-watermark vendor, and locally DECODE the one open scheme, Adobe TrustMark (`trustmark_detector`); the rest (Digimarc/Imatag/Steg.AI/...) stay name-only (proprietary decoders).
|
||||
- **Built in the dated batch:** soft-binding vendor detection, IPTC Photo
|
||||
Metadata AI-disclosure fields, C2PA detection and stripping for supported
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Build and verify an immutable confirmatory batch for a SynthID oracle.
|
||||
|
||||
The batch contains a referenced untouched source plus four lossless PNG views:
|
||||
an exact-pixel re-encode, aligned periodic-tile subtraction, a cyclically
|
||||
shifted tile, and an orthogonal random tile. Building the batch performs no
|
||||
network requests. Oracle results remain empty until a separately authorized
|
||||
submission records them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
from PIL import Image
|
||||
from synthid_periodic_tile_ablation import (
|
||||
candidate_quality,
|
||||
control_templates,
|
||||
phase_score_pixels,
|
||||
tile_score_pixels,
|
||||
)
|
||||
from synthid_periodic_tile_probe import PeriodicTileModel
|
||||
from synthid_periodic_tile_probe import load_model as load_tile_model
|
||||
from synthid_phase_carrier import PhaseCarrierModel
|
||||
from synthid_phase_carrier import load_model as load_phase_model
|
||||
from synthid_pixel_attack import load_rgb
|
||||
from synthid_research_manifest import artifact_sha256, pixel_fingerprint
|
||||
from synthid_tile_attack import subtract_tiled_template
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ROLE_ORDER = ("source", "reencode_control", "aligned", "shifted", "orthogonal_random")
|
||||
DERIVATIVE_ROLES = ROLE_ORDER[1:]
|
||||
FORMAT_VERSION = 1
|
||||
SYNTHID_RESULTS = {"detected", "not_detected", "indeterminate", "refused"}
|
||||
C2PA_RESULTS = {"present", "absent", "indeterminate", "unavailable"}
|
||||
|
||||
|
||||
def _inside(path: Path, parent: Path) -> bool:
|
||||
"""Return whether resolved PATH is inside resolved PARENT."""
|
||||
try:
|
||||
path.resolve().relative_to(parent.resolve())
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _json_float(value: float) -> float | None:
|
||||
"""Return finite VALUE or None for JSON interoperability."""
|
||||
return value if math.isfinite(value) else None
|
||||
|
||||
|
||||
def _write_png(path: Path, pixels: np.ndarray) -> None:
|
||||
"""Write exact RGB PIXELS as a deterministic lossless PNG."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists():
|
||||
raise ValueError(f"refusing to overwrite oracle artifact: {path}")
|
||||
Image.fromarray(pixels, mode="RGB").save(path, format="PNG", compress_level=9)
|
||||
|
||||
|
||||
def _score_row(
|
||||
pixels: np.ndarray,
|
||||
*,
|
||||
source: np.ndarray,
|
||||
tile_model: PeriodicTileModel,
|
||||
phase_model: PhaseCarrierModel,
|
||||
tile_threshold: float,
|
||||
phase_threshold: float,
|
||||
active_threshold: float,
|
||||
) -> dict[str, object]:
|
||||
"""Return local frozen scores and paired fidelity for PIXELS."""
|
||||
tile_score = tile_score_pixels(pixels, tile_model)
|
||||
phase_score, active_support = phase_score_pixels(pixels, phase_model)
|
||||
quality = candidate_quality(source, pixels)
|
||||
return {
|
||||
"tile_score": tile_score,
|
||||
"tile_accepted": tile_score >= tile_threshold,
|
||||
"phase_score": phase_score,
|
||||
"active_support": active_support,
|
||||
"phase_accepted": phase_score >= phase_threshold and active_support >= active_threshold,
|
||||
"residual_rms": quality["residual_rms"],
|
||||
"psnr_db": _json_float(quality["psnr_db"]),
|
||||
"ssim": quality["ssim"],
|
||||
"changed_pixel_fraction": quality["changed_pixel_fraction"],
|
||||
}
|
||||
|
||||
|
||||
def _artifact_row(
|
||||
path: Path,
|
||||
*,
|
||||
role: str,
|
||||
source_id: str,
|
||||
in_batch: bool,
|
||||
batch_root: Path,
|
||||
scores: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Return one hash-frozen manifest row for PATH."""
|
||||
pixel_sha256, width, height = pixel_fingerprint(path)
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"role": role,
|
||||
"path": str(path.relative_to(batch_root) if in_batch else path.resolve()),
|
||||
"in_batch": in_batch,
|
||||
"artifact_sha256": artifact_sha256(path),
|
||||
"pixel_sha256": pixel_sha256,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"synthid_result": None,
|
||||
"c2pa_result": None,
|
||||
"submitted_at": None,
|
||||
**scores,
|
||||
}
|
||||
|
||||
|
||||
def build_batch(
|
||||
sources: list[Path],
|
||||
*,
|
||||
output_dir: Path,
|
||||
tile_model_path: Path,
|
||||
phase_model_path: Path,
|
||||
tile_threshold: float,
|
||||
phase_threshold: float,
|
||||
active_threshold: float,
|
||||
strength: float,
|
||||
seed: int,
|
||||
provider: str,
|
||||
repository_root: Path,
|
||||
) -> Path:
|
||||
"""Build a preregistered, unsubmitted oracle batch and return its manifest."""
|
||||
if not sources:
|
||||
raise ValueError("at least one source is required")
|
||||
if strength <= 0.0 or not math.isfinite(strength):
|
||||
raise ValueError("strength must be finite and positive")
|
||||
if provider not in {"google", "openai"}:
|
||||
raise ValueError("provider must be google or openai")
|
||||
if _inside(output_dir, repository_root):
|
||||
raise ValueError("oracle batches must be written outside the repository")
|
||||
if output_dir.exists() and any(output_dir.iterdir()):
|
||||
raise ValueError("output directory must not already contain files")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
tile_model = load_tile_model(tile_model_path)
|
||||
phase_model = load_phase_model(phase_model_path)
|
||||
if (tile_model.height, tile_model.width) != (phase_model.height, phase_model.width):
|
||||
raise ValueError("tile and phase model geometries differ")
|
||||
templates = control_templates(tile_model.template, seed=seed)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
seen_source_hashes: set[str] = set()
|
||||
for source_path in sources:
|
||||
source_hash = artifact_sha256(source_path)
|
||||
if source_hash in seen_source_hashes:
|
||||
raise ValueError("duplicate source artifact")
|
||||
seen_source_hashes.add(source_hash)
|
||||
source = load_rgb(source_path)
|
||||
if source.shape != (tile_model.height, tile_model.width, 3):
|
||||
raise ValueError(f"{source_path}: geometry does not match the frozen models")
|
||||
source_id = source_hash[:16]
|
||||
score_args = {
|
||||
"source": source,
|
||||
"tile_model": tile_model,
|
||||
"phase_model": phase_model,
|
||||
"tile_threshold": tile_threshold,
|
||||
"phase_threshold": phase_threshold,
|
||||
"active_threshold": active_threshold,
|
||||
}
|
||||
source_scores = _score_row(source, **score_args)
|
||||
rows.append(
|
||||
_artifact_row(
|
||||
source_path,
|
||||
role="source",
|
||||
source_id=source_id,
|
||||
in_batch=False,
|
||||
batch_root=output_dir,
|
||||
scores=source_scores,
|
||||
)
|
||||
)
|
||||
for role in DERIVATIVE_ROLES:
|
||||
variant = (
|
||||
source
|
||||
if role == "reencode_control"
|
||||
else subtract_tiled_template(
|
||||
source,
|
||||
templates[role] * tile_model.expected_norm,
|
||||
strength=strength,
|
||||
)
|
||||
)
|
||||
output_path = output_dir / source_id / f"{role}.png"
|
||||
_write_png(output_path, variant)
|
||||
rows.append(
|
||||
_artifact_row(
|
||||
output_path,
|
||||
role=role,
|
||||
source_id=source_id,
|
||||
in_batch=True,
|
||||
batch_root=output_dir,
|
||||
scores=(source_scores if role == "reencode_control" else _score_row(variant, **score_args)),
|
||||
)
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"format_version": FORMAT_VERSION,
|
||||
"status": "preregistered_unsubmitted",
|
||||
"source_count": len(sources),
|
||||
"request_count": len(rows),
|
||||
"request_order": ROLE_ORDER,
|
||||
"provider": provider,
|
||||
"decision_rule": (
|
||||
"Count causal success only when source, reencode_control, shifted, and "
|
||||
"orthogonal_random are detected in the matching provider SynthID oracle, "
|
||||
"aligned is not_detected, and indeterminate is never treated as negative."
|
||||
),
|
||||
"strength": strength,
|
||||
"seed": seed,
|
||||
"tile_threshold": tile_threshold,
|
||||
"phase_threshold": phase_threshold,
|
||||
"active_threshold": active_threshold,
|
||||
"tile_model": str(tile_model_path.resolve()),
|
||||
"tile_model_sha256": artifact_sha256(tile_model_path),
|
||||
"phase_model": str(phase_model_path.resolve()),
|
||||
"phase_model_sha256": artifact_sha256(phase_model_path),
|
||||
"rows": rows,
|
||||
}
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2, allow_nan=False) + "\n", encoding="utf-8")
|
||||
manifest_hash = artifact_sha256(manifest_path)
|
||||
(output_dir / "manifest.sha256").write_text(manifest_hash + " manifest.json\n", encoding="utf-8")
|
||||
results_template = {
|
||||
"format_version": FORMAT_VERSION,
|
||||
"manifest_sha256": manifest_hash,
|
||||
"provider": provider,
|
||||
"rows": [
|
||||
{
|
||||
"source_id": row["source_id"],
|
||||
"role": row["role"],
|
||||
"artifact_sha256": row["artifact_sha256"],
|
||||
"synthid_result": None,
|
||||
"c2pa_result": None,
|
||||
"raw_response": None,
|
||||
"submitted_at": None,
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
}
|
||||
(output_dir / "results-template.json").write_text(
|
||||
json.dumps(results_template, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
verify_batch(manifest_path, repository_root=repository_root)
|
||||
return manifest_path
|
||||
|
||||
|
||||
def verify_batch(manifest_path: Path, *, repository_root: Path) -> dict[str, object]:
|
||||
"""Verify manifest, model, source, and derivative hashes without mutation."""
|
||||
batch_root = manifest_path.parent
|
||||
if _inside(batch_root, repository_root):
|
||||
raise ValueError("oracle batches must remain outside the repository")
|
||||
digest_path = batch_root / "manifest.sha256"
|
||||
expected_manifest_hash = digest_path.read_text(encoding="utf-8").split()[0]
|
||||
if artifact_sha256(manifest_path) != expected_manifest_hash:
|
||||
raise ValueError("manifest hash mismatch")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.get("format_version") != FORMAT_VERSION:
|
||||
raise ValueError("unsupported oracle-batch manifest version")
|
||||
rows = manifest.get("rows")
|
||||
if not isinstance(rows, list) or len(rows) != manifest.get("request_count"):
|
||||
raise ValueError("manifest request count mismatch")
|
||||
if artifact_sha256(Path(manifest["tile_model"])) != manifest["tile_model_sha256"]:
|
||||
raise ValueError("tile model hash mismatch")
|
||||
if artifact_sha256(Path(manifest["phase_model"])) != manifest["phase_model_sha256"]:
|
||||
raise ValueError("phase model hash mismatch")
|
||||
|
||||
groups: dict[str, list[str]] = {}
|
||||
artifact_hashes: set[str] = set()
|
||||
for row in rows:
|
||||
source_id = str(row["source_id"])
|
||||
groups.setdefault(source_id, []).append(str(row["role"]))
|
||||
path = batch_root / str(row["path"]) if row["in_batch"] else Path(str(row["path"]))
|
||||
if artifact_sha256(path) != row["artifact_sha256"]:
|
||||
raise ValueError(f"artifact hash mismatch for {source_id}/{row['role']}")
|
||||
pixel_sha256, width, height = pixel_fingerprint(path)
|
||||
if (pixel_sha256, width, height) != (row["pixel_sha256"], row["width"], row["height"]):
|
||||
raise ValueError(f"pixel fingerprint mismatch for {source_id}/{row['role']}")
|
||||
artifact_hashes.add(str(row["artifact_sha256"]))
|
||||
if any(tuple(roles) != ROLE_ORDER for roles in groups.values()):
|
||||
raise ValueError("each source must have the fixed request-role order")
|
||||
if len(groups) != manifest.get("source_count"):
|
||||
raise ValueError("manifest source count mismatch")
|
||||
if len(artifact_hashes) < len(rows) - len(groups):
|
||||
raise ValueError("unexpected duplicate derivative artifacts")
|
||||
template = json.loads((batch_root / "results-template.json").read_text(encoding="utf-8"))
|
||||
expected_identities = [(row["source_id"], row["role"], row["artifact_sha256"]) for row in rows]
|
||||
template_identities = [(row["source_id"], row["role"], row["artifact_sha256"]) for row in template.get("rows", [])]
|
||||
if template.get("manifest_sha256") != expected_manifest_hash or template.get("provider") != manifest["provider"]:
|
||||
raise ValueError("results template does not identify the preregistered batch")
|
||||
if template_identities != expected_identities:
|
||||
raise ValueError("results template row identities differ from the manifest")
|
||||
return manifest
|
||||
|
||||
|
||||
def _parse_submitted_at(value: object) -> None:
|
||||
"""Reject timestamps that are absent or not timezone-aware ISO-8601."""
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as error:
|
||||
raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp") from error
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("submitted_at must be a timezone-aware ISO-8601 timestamp")
|
||||
|
||||
|
||||
def evaluate_results(
|
||||
manifest_path: Path,
|
||||
results_path: Path,
|
||||
*,
|
||||
repository_root: Path,
|
||||
) -> dict[str, object]:
|
||||
"""Validate a complete result file and apply the preregistered decision rule."""
|
||||
manifest = verify_batch(manifest_path, repository_root=repository_root)
|
||||
results = json.loads(results_path.read_text(encoding="utf-8"))
|
||||
manifest_hash = artifact_sha256(manifest_path)
|
||||
if results.get("format_version") != FORMAT_VERSION:
|
||||
raise ValueError("unsupported oracle-results version")
|
||||
if results.get("manifest_sha256") != manifest_hash or results.get("provider") != manifest["provider"]:
|
||||
raise ValueError("results do not identify the preregistered batch")
|
||||
manifest_rows = manifest["rows"]
|
||||
result_rows = results.get("rows")
|
||||
if not isinstance(result_rows, list) or len(result_rows) != len(manifest_rows):
|
||||
raise ValueError("oracle results must cover every preregistered request")
|
||||
|
||||
grouped: dict[str, dict[str, str]] = {}
|
||||
for expected, result in zip(manifest_rows, result_rows, strict=True):
|
||||
identity = (result.get("source_id"), result.get("role"), result.get("artifact_sha256"))
|
||||
expected_identity = (expected["source_id"], expected["role"], expected["artifact_sha256"])
|
||||
if identity != expected_identity:
|
||||
raise ValueError("oracle result order or artifact identity differs from the manifest")
|
||||
synthid_result = result.get("synthid_result")
|
||||
c2pa_result = result.get("c2pa_result")
|
||||
if synthid_result not in SYNTHID_RESULTS:
|
||||
raise ValueError("invalid or missing SynthID result")
|
||||
if c2pa_result not in C2PA_RESULTS:
|
||||
raise ValueError("invalid or missing C2PA result")
|
||||
if not isinstance(result.get("raw_response"), str) or not result["raw_response"].strip():
|
||||
raise ValueError("raw_response must preserve the nonempty verbatim oracle result")
|
||||
_parse_submitted_at(result.get("submitted_at"))
|
||||
grouped.setdefault(str(expected["source_id"]), {})[str(expected["role"])] = str(synthid_result)
|
||||
|
||||
source_results: list[dict[str, str]] = []
|
||||
for source_id, roles in grouped.items():
|
||||
if tuple(roles) != ROLE_ORDER:
|
||||
raise ValueError("oracle results do not preserve the fixed role order")
|
||||
values = set(roles.values())
|
||||
if values & {"indeterminate", "refused"}:
|
||||
verdict = "indeterminate"
|
||||
elif (
|
||||
roles["source"] == "detected"
|
||||
and roles["reencode_control"] == "detected"
|
||||
and roles["shifted"] == "detected"
|
||||
and roles["orthogonal_random"] == "detected"
|
||||
and roles["aligned"] == "not_detected"
|
||||
):
|
||||
verdict = "causal_success"
|
||||
elif any(roles[role] != "detected" for role in ("source", "reencode_control", "shifted", "orthogonal_random")):
|
||||
verdict = "control_failed"
|
||||
else:
|
||||
verdict = "aligned_still_detected"
|
||||
source_results.append({"source_id": source_id, "verdict": verdict})
|
||||
counts = {
|
||||
verdict: sum(row["verdict"] == verdict for row in source_results)
|
||||
for verdict in ("causal_success", "aligned_still_detected", "control_failed", "indeterminate")
|
||||
}
|
||||
return {
|
||||
"format_version": FORMAT_VERSION,
|
||||
"manifest_sha256": manifest_hash,
|
||||
"provider": manifest["provider"],
|
||||
"source_count": manifest["source_count"],
|
||||
"counts": counts,
|
||||
"sources": source_results,
|
||||
}
|
||||
|
||||
|
||||
@click.group()
|
||||
def main() -> None:
|
||||
"""Build and verify an immutable confirmatory oracle batch."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
|
||||
@main.command("build")
|
||||
@click.argument("tile_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("phase_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("sources", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.option("--output-dir", type=click.Path(file_okay=False, path_type=Path), required=True)
|
||||
@click.option("--tile-threshold", type=float, required=True)
|
||||
@click.option("--phase-threshold", type=float, required=True)
|
||||
@click.option("--active-threshold", type=float, required=True)
|
||||
@click.option("--strength", type=click.FloatRange(min=0.0, min_open=True), default=2.0, show_default=True)
|
||||
@click.option("--seed", type=int, default=20260810, show_default=True)
|
||||
@click.option("--provider", type=click.Choice(["google", "openai"]), required=True)
|
||||
def build_command(
|
||||
tile_model_path: Path,
|
||||
phase_model_path: Path,
|
||||
sources: tuple[Path, ...],
|
||||
output_dir: Path,
|
||||
tile_threshold: float,
|
||||
phase_threshold: float,
|
||||
active_threshold: float,
|
||||
strength: float,
|
||||
seed: int,
|
||||
provider: str,
|
||||
) -> None:
|
||||
"""Build a frozen batch from exact-geometry SOURCES."""
|
||||
manifest = build_batch(
|
||||
list(sources),
|
||||
output_dir=output_dir,
|
||||
tile_model_path=tile_model_path,
|
||||
phase_model_path=phase_model_path,
|
||||
tile_threshold=tile_threshold,
|
||||
phase_threshold=phase_threshold,
|
||||
active_threshold=active_threshold,
|
||||
strength=strength,
|
||||
seed=seed,
|
||||
provider=provider,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
log.info("Wrote preregistered oracle batch: %s", manifest)
|
||||
|
||||
|
||||
@main.command("verify")
|
||||
@click.argument("manifest_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
def verify_command(manifest_path: Path) -> None:
|
||||
"""Verify an existing batch without changing it."""
|
||||
manifest = verify_batch(manifest_path, repository_root=Path(__file__).resolve().parent.parent)
|
||||
log.info("Verified %d immutable oracle requests", manifest["request_count"])
|
||||
|
||||
|
||||
@main.command("evaluate")
|
||||
@click.argument("manifest_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("results_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True)
|
||||
def evaluate_command(manifest_path: Path, results_path: Path, report_out: Path) -> None:
|
||||
"""Validate RESULTS_PATH and write the preregistered batch verdict."""
|
||||
repository_root = Path(__file__).resolve().parent.parent
|
||||
if _inside(report_out, repository_root):
|
||||
raise click.BadParameter("oracle result reports must be written outside the repository")
|
||||
report = evaluate_results(
|
||||
manifest_path,
|
||||
results_path,
|
||||
repository_root=repository_root,
|
||||
)
|
||||
report_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
if report_out.exists():
|
||||
raise click.BadParameter("refusing to overwrite an oracle result report")
|
||||
report_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
log.info("Wrote oracle batch verdict: %s", report_out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,50 +1,12 @@
|
||||
"""Shared periodic-residual helpers for SynthID research probes."""
|
||||
"""Compatibility imports for shared periodic-residual helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from remove_ai_watermarks.synthid_detector import fold_residual_template, unit_tile
|
||||
|
||||
def fold_residual_template(
|
||||
pixels: np.ndarray,
|
||||
*,
|
||||
tile_height: int,
|
||||
tile_width: int,
|
||||
denoise_sigma: float,
|
||||
) -> np.ndarray:
|
||||
"""Estimate a zero-mean periodic residual template by modulo folding."""
|
||||
if pixels.ndim != 3 or pixels.shape[2] != 3:
|
||||
raise ValueError("pixels must have shape (height, width, 3)")
|
||||
if tile_height < 1 or tile_width < 1 or denoise_sigma <= 0.0:
|
||||
raise ValueError("tile dimensions and denoise sigma must be positive")
|
||||
height, width = pixels.shape[:2]
|
||||
if height % tile_height != 0 or width % tile_width != 0:
|
||||
raise ValueError("image geometry must be divisible by the tile geometry")
|
||||
source = pixels.astype(np.float32)
|
||||
denoised = cv2.GaussianBlur(
|
||||
source,
|
||||
(0, 0),
|
||||
sigmaX=denoise_sigma,
|
||||
sigmaY=denoise_sigma,
|
||||
borderType=cv2.BORDER_REFLECT_101,
|
||||
)
|
||||
residual = source - denoised
|
||||
repeats_y = height // tile_height
|
||||
repeats_x = width // tile_width
|
||||
folded = residual.reshape(repeats_y, tile_height, repeats_x, tile_width, 3).mean(
|
||||
axis=(0, 2),
|
||||
dtype=np.float64,
|
||||
)
|
||||
return folded - np.mean(folded, axis=(0, 1), keepdims=True)
|
||||
|
||||
|
||||
def unit_tile(tile: np.ndarray) -> tuple[np.ndarray, float]:
|
||||
"""Return TILE normalized by its L2 norm and the original norm."""
|
||||
norm = float(np.linalg.norm(tile))
|
||||
if norm == 0.0:
|
||||
return np.zeros_like(tile, dtype=np.float64), 0.0
|
||||
return np.asarray(tile, dtype=np.float64) / norm, norm
|
||||
__all__ = ["cyclic_tile_correlations", "fold_residual_template", "unit_tile"]
|
||||
|
||||
|
||||
def cyclic_tile_correlations(template: np.ndarray, tile: np.ndarray) -> np.ndarray:
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Measure whether a frozen periodic tile causally controls local carrier scores.
|
||||
|
||||
This harness compares subtraction of the learned tile with cyclically shifted
|
||||
and orthogonal random tiles of the same norm. It measures local research
|
||||
detectors only. A score reversal is not evidence that a provider oracle would
|
||||
stop detecting SynthID.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
import numpy as np
|
||||
from synthid_periodic_tile import unit_tile
|
||||
from synthid_periodic_tile_probe import PeriodicTileModel
|
||||
from synthid_periodic_tile_probe import load_model as load_tile_model
|
||||
from synthid_periodic_tile_probe import score_pixels as score_tile_pixels
|
||||
from synthid_phase_carrier import PhaseCarrierModel, score_pixels
|
||||
from synthid_phase_carrier import load_model as load_phase_model
|
||||
from synthid_pixel_attack import load_rgb, measure
|
||||
from synthid_research_manifest import artifact_sha256
|
||||
from synthid_tile_attack import parse_positive_floats, subtract_tiled_template
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def exact_sign_test(negative: int, positive: int) -> float:
|
||||
"""Return an exact two-sided sign-test p-value after excluding ties."""
|
||||
count = negative + positive
|
||||
if count == 0:
|
||||
return 1.0
|
||||
tail = sum(math.comb(count, index) for index in range(min(negative, positive) + 1)) / 2**count
|
||||
return min(1.0, 2.0 * tail)
|
||||
|
||||
|
||||
def control_templates(template: np.ndarray, *, seed: int) -> dict[str, np.ndarray]:
|
||||
"""Return aligned, shifted, and norm-matched orthogonal control tiles."""
|
||||
rng = np.random.default_rng(seed)
|
||||
random_tile = rng.normal(size=template.shape)
|
||||
random_tile -= np.mean(random_tile, axis=(0, 1), keepdims=True)
|
||||
random_tile -= np.sum(random_tile * template) * template
|
||||
random_tile, norm = unit_tile(random_tile)
|
||||
if norm == 0.0 or abs(float(np.sum(random_tile * template))) > 1e-12:
|
||||
raise ValueError("could not construct an orthogonal random control")
|
||||
return {
|
||||
"aligned": template,
|
||||
"shifted": np.roll(template, shift=(1, 1), axis=(0, 1)),
|
||||
"orthogonal_random": random_tile,
|
||||
}
|
||||
|
||||
|
||||
def phase_score_pixels(pixels: np.ndarray, model: PhaseCarrierModel) -> tuple[float, float]:
|
||||
"""Return the unregistered phase score and active support for PIXELS."""
|
||||
result = score_pixels(pixels, model)
|
||||
return result.score, result.active_weight_fraction
|
||||
|
||||
|
||||
def tile_score_pixels(pixels: np.ndarray, model: PeriodicTileModel) -> float:
|
||||
"""Return the fixed-phase periodic-tile score for PIXELS."""
|
||||
return score_tile_pixels(pixels, model).score
|
||||
|
||||
|
||||
def summarize(values: list[float]) -> dict[str, float]:
|
||||
"""Return bounded descriptive statistics for VALUES."""
|
||||
return {
|
||||
"minimum": float(np.min(values)),
|
||||
"median": float(np.median(values)),
|
||||
"maximum": float(np.max(values)),
|
||||
}
|
||||
|
||||
|
||||
def direction_summary(values: list[float]) -> dict[str, float | int]:
|
||||
"""Return direction counts and a two-sided sign test for VALUES."""
|
||||
negative = sum(value < 0.0 for value in values)
|
||||
positive = sum(value > 0.0 for value in values)
|
||||
return {
|
||||
"negative": negative,
|
||||
"positive": positive,
|
||||
"ties": len(values) - negative - positive,
|
||||
"two_sided_sign_p": exact_sign_test(negative, positive),
|
||||
}
|
||||
|
||||
|
||||
def candidate_quality(reference: np.ndarray, candidate: np.ndarray) -> dict[str, float]:
|
||||
"""Return paired fidelity metrics for one equal-geometry candidate."""
|
||||
measurement = measure(reference, candidate, name="candidate", path=Path("<memory>"))
|
||||
return {
|
||||
"residual_rms": measurement.residual_rms,
|
||||
"psnr_db": measurement.psnr_db,
|
||||
"ssim": measurement.ssim,
|
||||
"changed_pixel_fraction": measurement.changed_pixel_fraction,
|
||||
}
|
||||
|
||||
|
||||
def run_ablation(
|
||||
sources: list[Path],
|
||||
*,
|
||||
tile_model: PeriodicTileModel,
|
||||
phase_model: PhaseCarrierModel,
|
||||
tile_threshold: float,
|
||||
phase_threshold: float,
|
||||
active_threshold: float,
|
||||
strengths: tuple[float, ...],
|
||||
phase_strength: float,
|
||||
seed: int,
|
||||
) -> dict[str, object]:
|
||||
"""Evaluate aligned subtraction and controls on exact-geometry SOURCES."""
|
||||
if not sources:
|
||||
raise ValueError("at least one source is required")
|
||||
if phase_strength not in strengths:
|
||||
raise ValueError("phase strength must be one of the swept strengths")
|
||||
if (tile_model.height, tile_model.width) != (phase_model.height, phase_model.width):
|
||||
raise ValueError("tile and phase model geometries differ")
|
||||
if not all(np.isfinite(value) for value in (tile_threshold, phase_threshold, active_threshold)):
|
||||
raise ValueError("thresholds must be finite")
|
||||
|
||||
templates = control_templates(tile_model.template, seed=seed)
|
||||
rows: list[dict[str, object]] = []
|
||||
for source_path in sources:
|
||||
source = load_rgb(source_path)
|
||||
if source.shape != (tile_model.height, tile_model.width, 3):
|
||||
raise ValueError(f"{source_path}: geometry does not match the models")
|
||||
source_hash = artifact_sha256(source_path)
|
||||
original_tile = tile_score_pixels(source, tile_model)
|
||||
original_phase, original_support = phase_score_pixels(source, phase_model)
|
||||
for strength in strengths:
|
||||
for control, template in templates.items():
|
||||
candidate = subtract_tiled_template(
|
||||
source,
|
||||
template * tile_model.expected_norm,
|
||||
strength=strength,
|
||||
)
|
||||
tile_score = tile_score_pixels(candidate, tile_model)
|
||||
row: dict[str, object] = {
|
||||
"path": str(source_path),
|
||||
"artifact_sha256": source_hash,
|
||||
"control": control,
|
||||
"strength": strength,
|
||||
"original_tile_score": original_tile,
|
||||
"tile_score": tile_score,
|
||||
"tile_delta": tile_score - original_tile,
|
||||
"tile_accepted": tile_score >= tile_threshold,
|
||||
"original_phase_score": original_phase,
|
||||
"original_active_support": original_support,
|
||||
}
|
||||
if strength == phase_strength:
|
||||
phase_score, active_support = phase_score_pixels(candidate, phase_model)
|
||||
row.update(
|
||||
{
|
||||
**candidate_quality(source, candidate),
|
||||
"phase_score": phase_score,
|
||||
"active_support": active_support,
|
||||
"phase_delta": phase_score - original_phase,
|
||||
"phase_accepted": phase_score >= phase_threshold and active_support >= active_threshold,
|
||||
}
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
grouped: dict[tuple[str, float], list[dict[str, object]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
grouped[(str(row["control"]), float(row["strength"]))].append(row)
|
||||
tile_summaries: list[dict[str, object]] = []
|
||||
for (control, strength), group in sorted(grouped.items()):
|
||||
deltas = [float(row["tile_delta"]) for row in group]
|
||||
tile_summaries.append(
|
||||
{
|
||||
"control": control,
|
||||
"strength": strength,
|
||||
"accepted": sum(bool(row["tile_accepted"]) for row in group),
|
||||
"delta": summarize(deltas),
|
||||
"direction": direction_summary(deltas),
|
||||
}
|
||||
)
|
||||
|
||||
selected = [row for row in rows if float(row["strength"]) == phase_strength]
|
||||
phase_summaries: list[dict[str, object]] = []
|
||||
for control in templates:
|
||||
group = [row for row in selected if row["control"] == control]
|
||||
phase_summaries.append(
|
||||
{
|
||||
"control": control,
|
||||
"accepted": sum(bool(row["phase_accepted"]) for row in group),
|
||||
"delta": summarize([float(row["phase_delta"]) for row in group]),
|
||||
"active_support": summarize([float(row["active_support"]) for row in group]),
|
||||
"psnr_db": summarize([float(row["psnr_db"]) for row in group]),
|
||||
"ssim": summarize([float(row["ssim"]) for row in group]),
|
||||
"changed_pixel_fraction": summarize([float(row["changed_pixel_fraction"]) for row in group]),
|
||||
}
|
||||
)
|
||||
|
||||
paired_comparisons: list[dict[str, object]] = []
|
||||
for control in ("shifted", "orthogonal_random"):
|
||||
for metric in ("tile_delta", "phase_delta"):
|
||||
aligned = {
|
||||
str(row["artifact_sha256"]): float(row[metric]) for row in selected if row["control"] == "aligned"
|
||||
}
|
||||
comparison = {
|
||||
str(row["artifact_sha256"]): float(row[metric]) for row in selected if row["control"] == control
|
||||
}
|
||||
differences = [aligned[key] - comparison[key] for key in sorted(aligned)]
|
||||
paired_comparisons.append(
|
||||
{
|
||||
"aligned_minus": control,
|
||||
"metric": metric,
|
||||
"difference": summarize(differences),
|
||||
"direction": direction_summary(differences),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"source_count": len(sources),
|
||||
"tile_threshold": tile_threshold,
|
||||
"phase_threshold": phase_threshold,
|
||||
"active_threshold": active_threshold,
|
||||
"strengths": strengths,
|
||||
"phase_strength": phase_strength,
|
||||
"seed": seed,
|
||||
"original": {
|
||||
"tile_accepted": sum(
|
||||
float(row["original_tile_score"]) >= tile_threshold for row in selected if row["control"] == "aligned"
|
||||
),
|
||||
"phase_accepted": sum(
|
||||
float(row["original_phase_score"]) >= phase_threshold
|
||||
and float(row["original_active_support"]) >= active_threshold
|
||||
for row in selected
|
||||
if row["control"] == "aligned"
|
||||
),
|
||||
},
|
||||
"tile_summaries": tile_summaries,
|
||||
"phase_summaries": phase_summaries,
|
||||
"paired_comparisons": paired_comparisons,
|
||||
"items": rows,
|
||||
}
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("tile_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("phase_model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.argument("sources", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.option("--tile-threshold", type=float, required=True)
|
||||
@click.option("--phase-threshold", type=float, required=True)
|
||||
@click.option("--active-threshold", type=float, required=True)
|
||||
@click.option("--strengths", default="1,1.5,2,3,4", show_default=True)
|
||||
@click.option("--phase-strength", type=click.FloatRange(min=0.0, min_open=True), default=2.0, show_default=True)
|
||||
@click.option("--seed", type=int, default=20260810, show_default=True)
|
||||
@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True)
|
||||
def main(
|
||||
tile_model_path: Path,
|
||||
phase_model_path: Path,
|
||||
sources: tuple[Path, ...],
|
||||
tile_threshold: float,
|
||||
phase_threshold: float,
|
||||
active_threshold: float,
|
||||
strengths: str,
|
||||
phase_strength: float,
|
||||
seed: int,
|
||||
report_out: Path,
|
||||
) -> None:
|
||||
"""Run a fixed periodic-tile causal ablation on exact-geometry SOURCES."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
strength_values = parse_positive_floats(strengths, option_name="strengths")
|
||||
report = run_ablation(
|
||||
list(sources),
|
||||
tile_model=load_tile_model(tile_model_path),
|
||||
phase_model=load_phase_model(phase_model_path),
|
||||
tile_threshold=tile_threshold,
|
||||
phase_threshold=phase_threshold,
|
||||
active_threshold=active_threshold,
|
||||
strengths=strength_values,
|
||||
phase_strength=phase_strength,
|
||||
seed=seed,
|
||||
)
|
||||
report["tile_model"] = str(tile_model_path)
|
||||
report["tile_model_sha256"] = artifact_sha256(tile_model_path)
|
||||
report["phase_model"] = str(phase_model_path)
|
||||
report["phase_model_sha256"] = artifact_sha256(phase_model_path)
|
||||
report_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
log.info("Wrote periodic-tile causal ablation: %s", report_out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -92,10 +92,18 @@ def discover_model(
|
||||
)
|
||||
|
||||
|
||||
def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) -> PeriodicTileScore:
|
||||
"""Score PATH against MODEL, optionally searching cyclic tile shifts."""
|
||||
def score_pixels(
|
||||
pixels: np.ndarray,
|
||||
model: PeriodicTileModel,
|
||||
*,
|
||||
register: bool = False,
|
||||
path: str = "<array>",
|
||||
) -> PeriodicTileScore:
|
||||
"""Score exact-geometry RGB PIXELS, optionally searching cyclic tile shifts."""
|
||||
if pixels.shape != (model.height, model.width, 3):
|
||||
raise ValueError("pixel geometry does not match periodic-tile model")
|
||||
folded = fold_residual_template(
|
||||
_load_rgb(path, height=model.height, width=model.width),
|
||||
pixels,
|
||||
tile_height=model.tile_height,
|
||||
tile_width=model.tile_width,
|
||||
denoise_sigma=model.denoise_sigma,
|
||||
@@ -109,7 +117,7 @@ def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False)
|
||||
score = float(np.sum(model.template * unit))
|
||||
row_shift = column_shift = 0
|
||||
return PeriodicTileScore(
|
||||
path=str(path),
|
||||
path=path,
|
||||
score=score,
|
||||
active_support=min(norm / (model.expected_norm + 1e-12), 1.0),
|
||||
row_shift=row_shift,
|
||||
@@ -118,6 +126,12 @@ def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False)
|
||||
)
|
||||
|
||||
|
||||
def score_image(path: Path, model: PeriodicTileModel, *, register: bool = False) -> PeriodicTileScore:
|
||||
"""Score PATH against MODEL, optionally searching cyclic tile shifts."""
|
||||
pixels = _load_rgb(path, height=model.height, width=model.width)
|
||||
return score_pixels(pixels, model, register=register, path=str(path))
|
||||
|
||||
|
||||
def calibrate_threshold(paths: list[Path], model: PeriodicTileModel, *, register: bool = False) -> float:
|
||||
"""Return the first float above every negative score in PATHS."""
|
||||
if not paths:
|
||||
|
||||
@@ -179,6 +179,27 @@ def _frequency_values(pixels: np.ndarray, model: PhaseCarrierModel) -> np.ndarra
|
||||
return extract_frequency_values(pixels, model.rows, model.columns, model.channels)
|
||||
|
||||
|
||||
def score_pixels(pixels: np.ndarray, model: PhaseCarrierModel, *, path: str = "<array>") -> PhaseCarrierScore:
|
||||
"""Score exact-geometry RGB PIXELS against MODEL."""
|
||||
if pixels.shape != (model.height, model.width, 3):
|
||||
raise ValueError("pixel geometry does not match phase-carrier model")
|
||||
values = _frequency_values(np.asarray(pixels, dtype=np.float64), model)
|
||||
magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0)
|
||||
active_weights = model.weights * magnitude_gate
|
||||
active_weight = float(np.sum(active_weights))
|
||||
score = (
|
||||
0.0
|
||||
if active_weight == 0.0
|
||||
else float(np.sum(active_weights * np.cos(np.angle(values) - model.phases)) / active_weight)
|
||||
)
|
||||
return PhaseCarrierScore(
|
||||
path=path,
|
||||
score=score,
|
||||
active_weight_fraction=active_weight,
|
||||
peak_count=len(model.rows),
|
||||
)
|
||||
|
||||
|
||||
def score_image(
|
||||
path: Path,
|
||||
model: PhaseCarrierModel,
|
||||
@@ -192,21 +213,7 @@ def score_image(
|
||||
width=model.width,
|
||||
canonicalize_geometry=canonicalize_geometry,
|
||||
)
|
||||
values = _frequency_values(pixels, model)
|
||||
magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0)
|
||||
active_weights = model.weights * magnitude_gate
|
||||
active_weight = float(np.sum(active_weights))
|
||||
score = (
|
||||
0.0
|
||||
if active_weight == 0.0
|
||||
else float(np.sum(active_weights * np.cos(np.angle(values) - model.phases)) / active_weight)
|
||||
)
|
||||
return PhaseCarrierScore(
|
||||
path=str(path),
|
||||
score=score,
|
||||
active_weight_fraction=active_weight,
|
||||
peak_count=len(model.rows),
|
||||
)
|
||||
return score_pixels(pixels, model, path=str(path))
|
||||
|
||||
|
||||
def score_translations(
|
||||
|
||||
@@ -13,6 +13,7 @@ High-level API (lazy, so ``import remove_ai_watermarks`` stays cheap)::
|
||||
raiw.remove_video_metadata("in.mp4", "out.mp4") # verified metadata strip
|
||||
raiw.remove_video_invisible("in.mp4", "out.mp4") # oracle-certified SynthID removal
|
||||
raiw.remove_video_visible("in.mp4", "out.mp4") # stable visible video-mark removal
|
||||
raiw.detect_synthid("in.png") # -> SynthIDDetection
|
||||
|
||||
For a provenance verdict use the ``identify`` submodule::
|
||||
|
||||
@@ -39,7 +40,9 @@ __all__ = [
|
||||
"InvisibleOptions",
|
||||
"MetadataStripIncomplete",
|
||||
"RemoveAllResult",
|
||||
"SynthIDDetection",
|
||||
"__version__",
|
||||
"detect_synthid",
|
||||
"identify_video",
|
||||
"inspect_video_metadata",
|
||||
"remove_all",
|
||||
@@ -64,6 +67,7 @@ if TYPE_CHECKING:
|
||||
remove_visible,
|
||||
visible_provenance,
|
||||
)
|
||||
from remove_ai_watermarks.synthid_detector import SynthIDDetection, detect_synthid
|
||||
from remove_ai_watermarks.video import (
|
||||
identify_video,
|
||||
inspect_video_metadata,
|
||||
@@ -103,4 +107,8 @@ def __getattr__(name: str) -> object:
|
||||
from remove_ai_watermarks import video
|
||||
|
||||
return getattr(video, name)
|
||||
if name in ("SynthIDDetection", "detect_synthid"):
|
||||
from remove_ai_watermarks import synthid_detector
|
||||
|
||||
return getattr(synthid_detector, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""Embedded assets for visible watermark removal."""
|
||||
"""Embedded runtime assets for watermark detection and removal."""
|
||||
|
||||
Binary file not shown.
@@ -297,8 +297,8 @@ _force_option = click.option(
|
||||
help=(
|
||||
"Run the diffusion scrub even when no invisible AI watermark is locally "
|
||||
"detectable. Default: skip it (regeneration only degrades a clean image; a "
|
||||
"skip never claims the image is watermark-free -- a pixel SynthID is "
|
||||
"undetectable once its metadata proxy is gone)."
|
||||
"skip never claims the image is watermark-free -- the local SynthID detector "
|
||||
"covers one carrier family in a calibrated image-size range)."
|
||||
),
|
||||
)
|
||||
_cpu_offload_option = click.option(
|
||||
@@ -455,18 +455,20 @@ def _no_invisible_signal_exit(source: Path) -> NoReturn:
|
||||
:func:`identify` finds no locally-detectable invisible AI signal, running it
|
||||
anyway would damage a clean image for nothing -- the dominant paid score-0
|
||||
cause on no-watermark uploads. So skip it, but do NOT imply the image is
|
||||
clean: a pixel SynthID is undetectable here once its metadata proxy is gone.
|
||||
Write no output and exit :data:`EXIT_NO_INVISIBLE_SIGNAL`; ``--force`` runs
|
||||
the scrub regardless.
|
||||
clean: only one SynthID carrier family in a calibrated image-size range has
|
||||
a local detector, so other sizes or epochs can still be present after
|
||||
their metadata proxy is gone. Write no output and exit
|
||||
:data:`EXIT_NO_INVISIBLE_SIGNAL`; ``--force`` runs the scrub regardless.
|
||||
"""
|
||||
console.print(
|
||||
" No invisible AI watermark detected (no C2PA/SynthID provenance, no open\n"
|
||||
" watermark). Skipped the diffusion scrub -- regenerating the pixels would\n"
|
||||
" only degrade the image with nothing to remove, so no output was written.\n"
|
||||
" This does NOT prove the image is clean: a pixel watermark such as SynthID\n"
|
||||
" cannot be detected here once its metadata proxy is absent (it may have\n"
|
||||
" been stripped earlier). If you know the image is AI-generated and want the\n"
|
||||
" pixels regenerated regardless, re-run with --force:\n"
|
||||
" No supported invisible AI watermark detected (no provenance, supported\n"
|
||||
" SynthID carrier, or open watermark). Skipped the diffusion scrub --\n"
|
||||
" regenerating the pixels would only degrade the image with nothing to\n"
|
||||
" remove, so no output was written.\n"
|
||||
" This does NOT prove the image is clean: the local SynthID detector covers\n"
|
||||
" one carrier family in a calibrated image-size range. If you know the image\n"
|
||||
" is AI-generated and want the pixels regenerated regardless, re-run with\n"
|
||||
" --force:\n"
|
||||
f" remove-ai-watermarks invisible {source.name} --force"
|
||||
)
|
||||
raise SystemExit(EXIT_NO_INVISIBLE_SIGNAL)
|
||||
@@ -1316,6 +1318,41 @@ def cmd_video_batch(
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
# ── SynthID pixel detection ──
|
||||
@main.command("detect-synthid")
|
||||
@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@click.option("--json", "as_json", is_flag=True, help="Emit the detector result as JSON.")
|
||||
def cmd_detect_synthid(source: Path, as_json: bool) -> None:
|
||||
"""Detect the SynthID periodic pixel carrier at calibrated image sizes.
|
||||
|
||||
A negative result means this detector did not find its supported carrier; it
|
||||
is not proof that the image contains no SynthID watermark.
|
||||
"""
|
||||
from remove_ai_watermarks.synthid_detector import detect_synthid
|
||||
|
||||
source = _validate_image(source)
|
||||
try:
|
||||
result = detect_synthid(source)
|
||||
except RuntimeError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
if as_json:
|
||||
click.echo(json.dumps(result.to_dict(), indent=2))
|
||||
return
|
||||
|
||||
_banner()
|
||||
console.print(f"\n SynthID pixel carrier: {result.status}")
|
||||
console.print(f" Geometry: {result.width}x{result.height}")
|
||||
if result.score is not None:
|
||||
console.print(f" Score: {result.score:.6f} (threshold: {result.threshold:.6f})")
|
||||
console.print(f" Detector: {result.detector}")
|
||||
console.print(
|
||||
" Scope: one confirmed periodic carrier family in a calibrated image-size range.\n"
|
||||
" Arbitrary spatial resampling is not registered. A negative or\n"
|
||||
" unsupported result is not proof that SynthID is absent."
|
||||
)
|
||||
|
||||
|
||||
# ── Provenance identification ──
|
||||
@main.command("identify")
|
||||
@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
||||
@@ -1360,8 +1397,8 @@ def cmd_identify(ctx: click.Context, source: Path, no_visible: bool, as_json: bo
|
||||
if report.is_ai_generated is None:
|
||||
console.print(
|
||||
" No locally-readable AI signal found. This is not the same as 'clean': "
|
||||
"metadata is often stripped by re-encoding, screenshots, or upload, and SynthID-class "
|
||||
"pixel watermarks (Gemini / Nano Banana / gpt-image) have no local detector. "
|
||||
"metadata is often stripped by re-encoding, screenshots, or upload, and the local "
|
||||
"SynthID pixel detector covers one carrier family in a calibrated image-size range. "
|
||||
"See caveats below."
|
||||
)
|
||||
|
||||
@@ -1455,8 +1492,9 @@ def cmd_all(
|
||||
stage_text = {
|
||||
("invisible", "no-signal"): (
|
||||
"Skipped (no invisible AI watermark detected; pixels left intact).\n"
|
||||
" Not a clean-image guarantee: a pixel SynthID is undetectable once its\n"
|
||||
" metadata proxy is gone. Re-run with --force to scrub regardless."
|
||||
" Not a clean-image guarantee: the local SynthID detector covers one\n"
|
||||
" carrier family in a calibrated image-size range. Re-run with --force\n"
|
||||
" to scrub regardless."
|
||||
),
|
||||
("invisible", "unavailable"): (
|
||||
f"Warning: Skipped - GPU dependencies not installed.\n Install them with: pip install {INVISIBLE_EXTRA}"
|
||||
|
||||
@@ -6,15 +6,14 @@ Aggregates every locally-readable signal into a single :class:`ProvenanceReport`
|
||||
the signing platform (OpenAI, Google, Adobe, Microsoft).
|
||||
- **IPTC ``digitalSourceType``** "Made with AI" marker (Meta, X, others).
|
||||
- **PNG text / EXIF generation parameters** (Stable Diffusion, ComfyUI, InvokeAI).
|
||||
- **SynthID provenance evidence** -- Google AI C2PA follows Google's all-media
|
||||
policy; current OpenAI C2PA explicitly declares a watermark action.
|
||||
- **SynthID evidence** -- supported C2PA provenance plus a positive-only local
|
||||
detector for one confirmed periodic carrier family in a calibrated image-size range.
|
||||
- **Registered visible marks** (optional; needs cv2/numpy, no GPU) through the
|
||||
shared watermark registry.
|
||||
|
||||
Hard limit: a stripped image (re-encoded, screenshotted, social-media upload)
|
||||
loses all metadata, and the SynthID *pixel* watermark is not locally decodable
|
||||
(proprietary decoder). Absence of signals is therefore reported as ``Unknown``,
|
||||
never as "clean". See CLAUDE.md "SynthID detection is metadata-only".
|
||||
Hard limit: Google does not publish its payload decoder. The local pixel detector
|
||||
covers only one measured carrier family in a calibrated image-size range, so
|
||||
absence of signals is reported as ``Unknown``, never as "clean".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -111,8 +110,14 @@ _STRIP_CAVEAT = (
|
||||
"text chunks are stripped by re-encoding, screenshots, or social-media upload."
|
||||
)
|
||||
_SYNTHID_CAVEAT = (
|
||||
"SynthID presence comes from supported provenance here; the pixel watermark is not locally "
|
||||
"decoded (proprietary decoder). Confirm via the Gemini app or openai.com/verify."
|
||||
"SynthID presence comes from supported provenance here. The separate local pixel detector "
|
||||
"covers one measured carrier family in a calibrated image-size range; confirm other cases with "
|
||||
"the provider oracle."
|
||||
)
|
||||
_SYNTHID_PIXEL_CAVEAT = (
|
||||
"The local SynthID pixel result is a positive-only match to one measured periodic carrier family "
|
||||
"in a calibrated image-size range, not a proprietary payload decode. A negative or unsupported "
|
||||
"result is not proof of absence."
|
||||
)
|
||||
_IPTC_ONLY_CAVEAT = "The IPTC 'Made with AI' tag flags AI provenance but does not identify the specific platform."
|
||||
_INVISIBLE_WM_CAVEAT = (
|
||||
@@ -951,6 +956,15 @@ def _trustmark(image_path: Path) -> str | None:
|
||||
return detect_trustmark(image_path)
|
||||
|
||||
|
||||
def _synthid_pixel_watermark(image_path: Path, decode: _SharedDecode) -> bool:
|
||||
"""Whether the supported positive-only SynthID carrier is detected."""
|
||||
from remove_ai_watermarks.synthid_detector import detect_synthid, is_available
|
||||
|
||||
if not is_available() or (image := decode.get()) is None:
|
||||
return False
|
||||
return detect_synthid(image_path, image=image).detected
|
||||
|
||||
|
||||
class _SharedDecode:
|
||||
"""One decode of the source pixels, shared by every detector in a single report.
|
||||
|
||||
@@ -1295,6 +1309,17 @@ def _identify_from_evidence(
|
||||
if platform is None:
|
||||
platform = f"{scheme} (open DWT-DCT watermark)"
|
||||
|
||||
# ── Positive-only SynthID periodic carrier ──────────────────────
|
||||
# This is deliberately separate from C2PA provenance. It survives lossless
|
||||
# metadata stripping, but covers only one carrier family and a calibrated
|
||||
# image-size range.
|
||||
if check_invisible and pixel_path is not None and _synthid_pixel_watermark(pixel_path, decode):
|
||||
signals.append(Signal("synthid_pixel", "calibrated periodic carrier", "high"))
|
||||
watermarks.append("SynthID periodic pixel carrier (calibrated image size)")
|
||||
caveats.append(_SYNTHID_PIXEL_CAVEAT)
|
||||
if platform is None:
|
||||
platform = "SynthID carrier detected (provider not attributed locally)"
|
||||
|
||||
# ── Adobe TrustMark invisible watermark (open decoder, no key) ───
|
||||
# The watermark behind Adobe Durable Content Credentials. Decoded locally,
|
||||
# but it binds provenance for human-authored content too, so it enriches the
|
||||
@@ -1306,7 +1331,7 @@ def _identify_from_evidence(
|
||||
platform = "Adobe (TrustMark / Content Credentials)"
|
||||
|
||||
# ── Verdict so far (metadata + embedded watermark) ──────────────
|
||||
invisible_wm = any(s.name == "invisible_watermark" for s in signals)
|
||||
invisible_wm = any(s.name in {"invisible_watermark", "synthid_pixel"} for s in signals)
|
||||
exif_gen = any(s.name == "exif_generator" for s in signals)
|
||||
xai_sig = any(s.name == "xai_signature" for s in signals)
|
||||
ai_from_metadata = bool(
|
||||
@@ -1405,8 +1430,9 @@ def identify(
|
||||
image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container).
|
||||
check_visible: Also run the registered visible-mark detectors through cv2.
|
||||
Set False for a metadata-only, dependency-light scan.
|
||||
check_invisible: Also decode optional open invisible watermarks
|
||||
(SD/SDXL/FLUX). No-op when the decoder extra is not installed.
|
||||
check_invisible: Also run optional pixel detectors for the supported
|
||||
SynthID carrier and open SD/SDXL/FLUX watermarks. No-op when their
|
||||
numeric extras are not installed.
|
||||
|
||||
File-backed metadata extraction runs first. The extracted evidence is then
|
||||
evaluated independently, followed by the optional pixel-backed visible and
|
||||
@@ -1436,13 +1462,13 @@ def has_invisible_target(image_path: Path) -> bool:
|
||||
to remove. Runs :func:`identify` with ``check_visible=False`` -- a visible mark
|
||||
is handled by the separate visible pass and is NOT a diffusion target -- and
|
||||
``check_invisible=True`` so an open watermark counts. Returns
|
||||
``report.ai_from_metadata`` (C2PA AI issuer / SynthID provenance, IPTC, AIGC, local
|
||||
gen params, EXIF/xAI, open DWT-DCT / TrustMark).
|
||||
``report.ai_from_metadata`` (C2PA AI issuer / SynthID provenance or periodic
|
||||
carrier, IPTC, AIGC, local gen params, EXIF/xAI, open DWT-DCT / TrustMark).
|
||||
|
||||
IMPORTANT -- this cannot prove a pixel SynthID is absent: SynthID is detectable
|
||||
only through its C2PA proxy, so a metadata-stripped AI image reads as no signal
|
||||
here. A False therefore means "no locally-detectable invisible target", not
|
||||
"clean". Callers must NOT present a skip as a finished clean result.
|
||||
IMPORTANT -- this cannot prove a pixel SynthID is absent: the local detector
|
||||
covers one carrier family in a calibrated image-size range. A False therefore
|
||||
means "no supported locally-detectable invisible target", not "clean". Callers
|
||||
must NOT present a skip as a finished clean result.
|
||||
|
||||
Fail-safe: any error resolves to True so the removal still runs -- leaving a
|
||||
watermark on a paid removal is worse than over-regenerating a clean image.
|
||||
|
||||
@@ -803,8 +803,9 @@ def synthid_source(image_path: Path) -> str | None:
|
||||
None.
|
||||
|
||||
The evidence is readable only while the C2PA manifest is intact. Absence is
|
||||
not proof: C2PA can be stripped while the pixel watermark survives, and the
|
||||
pixel watermark itself is not locally detectable (proprietary decoder).
|
||||
not proof: C2PA can be stripped while the pixel watermark survives. This
|
||||
metadata helper does not call the separate, geometry-limited local carrier
|
||||
detector.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image (PNG, JPEG, WebP, or ISOBMFF container).
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Detect the confirmed periodic SynthID image carrier at calibrated image sizes.
|
||||
|
||||
This is a positive-only detector for one measured carrier epoch, not Google's
|
||||
private payload decoder. A positive result is strong local evidence for the
|
||||
carrier. A negative result means only that this exact detector did not find it;
|
||||
image sizes outside the calibrated pixel-count range are reported separately.
|
||||
|
||||
The numeric runtime requires the ``pixels`` extra. Imports remain lazy so the
|
||||
package's metadata-only paths stay dependency-light.
|
||||
"""
|
||||
|
||||
# The optional numeric libraries do not provide complete types for this path.
|
||||
# pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from numpy.typing import NDArray
|
||||
|
||||
SynthIDDetectionStatus = Literal["detected", "not_detected", "unsupported"]
|
||||
|
||||
DETECTOR_ID = "synthid-periodic-tile-v2"
|
||||
MODEL_FILENAME = "synthid_periodic_tile_2048_v1.npz"
|
||||
# The template remains frozen at this model geometry. Runtime images are never
|
||||
# resized. The supported pixel-count interval is the separately challenged domain:
|
||||
# below it too few repetitions make the positive-only statistic unreliable, and
|
||||
# above it resource use and specificity have not been calibrated.
|
||||
MODEL_WIDTH = 2048
|
||||
MODEL_HEIGHT = 2048
|
||||
MIN_SUPPORTED_PIXELS = 1_000_000
|
||||
MAX_SUPPORTED_PIXELS = 18_000_000
|
||||
TILE_THRESHOLD = 0.17357069773071196
|
||||
INSTALL_HINT = "install the pixel extra: uv add 'remove-ai-watermarks[pixels]'"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SynthIDDetection:
|
||||
"""One local periodic-carrier verdict."""
|
||||
|
||||
status: SynthIDDetectionStatus
|
||||
width: int
|
||||
height: int
|
||||
score: float | None
|
||||
threshold: float
|
||||
detector: str = DETECTOR_ID
|
||||
|
||||
@property
|
||||
def detected(self) -> bool:
|
||||
"""Whether the supported carrier crossed its frozen threshold."""
|
||||
return self.status == "detected"
|
||||
|
||||
def to_dict(self) -> dict[str, str | int | float | None]:
|
||||
"""Return a JSON-safe result without a local file path."""
|
||||
return {
|
||||
"status": self.status,
|
||||
"width": self.width,
|
||||
"height": self.height,
|
||||
"score": self.score,
|
||||
"threshold": self.threshold,
|
||||
"detector": self.detector,
|
||||
}
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""True when the optional numeric runtime is installed."""
|
||||
from remove_ai_watermarks.optional_deps import module_available
|
||||
|
||||
return module_available("cv2", "numpy")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_template() -> tuple[NDArray[Any], float, int, int, int, int]:
|
||||
"""Load and validate the bundled pickle-free detector model."""
|
||||
import numpy as np
|
||||
|
||||
model_path = Path(__file__).parent / "assets" / MODEL_FILENAME
|
||||
with np.load(model_path, allow_pickle=False) as artifact:
|
||||
if int(artifact["format_version"]) != 1:
|
||||
raise RuntimeError("unsupported SynthID detector model format")
|
||||
height = int(artifact["height"])
|
||||
width = int(artifact["width"])
|
||||
tile_height = int(artifact["tile_height"])
|
||||
tile_width = int(artifact["tile_width"])
|
||||
denoise_sigma = float(artifact["denoise_sigma"])
|
||||
template = np.asarray(artifact["template"], dtype=np.float64)
|
||||
if not _geometry_supported(width, height):
|
||||
raise RuntimeError("bundled SynthID detector has unexpected geometry")
|
||||
if template.shape != (tile_height, tile_width, 3):
|
||||
raise RuntimeError("bundled SynthID detector has an invalid template shape")
|
||||
if not np.all(np.isfinite(template)) or not np.isclose(np.linalg.norm(template), 1.0):
|
||||
raise RuntimeError("bundled SynthID detector has an invalid template")
|
||||
if not np.isfinite(denoise_sigma) or denoise_sigma <= 0.0:
|
||||
raise RuntimeError("bundled SynthID detector has an invalid denoise sigma")
|
||||
return template, denoise_sigma, height, width, tile_height, tile_width
|
||||
|
||||
|
||||
def fold_residual_template(
|
||||
pixels: NDArray[Any],
|
||||
*,
|
||||
tile_height: int,
|
||||
tile_width: int,
|
||||
denoise_sigma: float,
|
||||
) -> NDArray[Any]:
|
||||
"""Estimate a zero-mean periodic residual template by modulo folding."""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
if pixels.ndim != 3 or pixels.shape[2] != 3:
|
||||
raise ValueError("pixels must have shape (height, width, 3)")
|
||||
if tile_height < 1 or tile_width < 1 or denoise_sigma <= 0.0:
|
||||
raise ValueError("tile dimensions and denoise sigma must be positive")
|
||||
height, width = pixels.shape[:2]
|
||||
if height < tile_height or width < tile_width:
|
||||
raise ValueError("image geometry must be at least as large as the tile geometry")
|
||||
divisible = height % tile_height == 0 and width % tile_width == 0
|
||||
full_height = height - height % tile_height
|
||||
full_width = width - width % tile_width
|
||||
repeats_y = full_height // tile_height
|
||||
repeats_x = full_width // tile_width
|
||||
remaining_height = height - full_height
|
||||
remaining_width = width - full_width
|
||||
counts = np.full((tile_height, tile_width), repeats_y * repeats_x, dtype=np.int64)
|
||||
counts[:remaining_height] += repeats_x
|
||||
counts[:, :remaining_width] += repeats_y
|
||||
counts[:remaining_height, :remaining_width] += 1
|
||||
|
||||
# OpenCV filters channels independently. Processing one channel at a time
|
||||
# keeps the 18 MP upper bound from requiring two full three-channel float32
|
||||
# buffers in addition to the decoded image.
|
||||
folded = np.empty((tile_height, tile_width, 3), dtype=np.float64)
|
||||
for channel in range(3):
|
||||
residual = pixels[:, :, channel].astype(np.float32)
|
||||
residual -= cv2.GaussianBlur(
|
||||
residual,
|
||||
(0, 0),
|
||||
sigmaX=denoise_sigma,
|
||||
sigmaY=denoise_sigma,
|
||||
borderType=cv2.BORDER_REFLECT_101,
|
||||
)
|
||||
if divisible:
|
||||
folded[:, :, channel] = residual.reshape(
|
||||
repeats_y,
|
||||
tile_height,
|
||||
repeats_x,
|
||||
tile_width,
|
||||
).mean(axis=(0, 2), dtype=np.float64)
|
||||
continue
|
||||
folded_sum = (
|
||||
residual[:full_height, :full_width]
|
||||
.reshape(
|
||||
repeats_y,
|
||||
tile_height,
|
||||
repeats_x,
|
||||
tile_width,
|
||||
)
|
||||
.sum(axis=(0, 2), dtype=np.float64)
|
||||
)
|
||||
if remaining_height:
|
||||
bottom = residual[full_height:, :full_width].reshape(
|
||||
remaining_height,
|
||||
repeats_x,
|
||||
tile_width,
|
||||
)
|
||||
folded_sum[:remaining_height] += bottom.sum(axis=1, dtype=np.float64)
|
||||
if remaining_width:
|
||||
right = residual[:full_height, full_width:].reshape(
|
||||
repeats_y,
|
||||
tile_height,
|
||||
remaining_width,
|
||||
)
|
||||
folded_sum[:, :remaining_width] += right.sum(axis=0, dtype=np.float64)
|
||||
if remaining_height and remaining_width:
|
||||
folded_sum[:remaining_height, :remaining_width] += residual[
|
||||
full_height:,
|
||||
full_width:,
|
||||
]
|
||||
folded[:, :, channel] = folded_sum / counts
|
||||
return folded - np.mean(folded, axis=(0, 1), keepdims=True)
|
||||
|
||||
|
||||
def unit_tile(tile: NDArray[Any]) -> tuple[NDArray[Any], float]:
|
||||
"""Return TILE normalized by its L2 norm and the original norm."""
|
||||
import numpy as np
|
||||
|
||||
norm = float(np.linalg.norm(tile))
|
||||
if norm == 0.0:
|
||||
return np.zeros_like(tile, dtype=np.float64), 0.0
|
||||
return np.asarray(tile, dtype=np.float64) / norm, norm
|
||||
|
||||
|
||||
def _image_size(image_path: Path) -> tuple[int, int]:
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(image_path) as image:
|
||||
return image.size
|
||||
|
||||
|
||||
def _geometry_supported(width: int, height: int) -> bool:
|
||||
"""Whether the image has a calibrated number of periodic-tile samples."""
|
||||
pixels = width * height
|
||||
return MIN_SUPPORTED_PIXELS <= pixels <= MAX_SUPPORTED_PIXELS
|
||||
|
||||
|
||||
def detect_synthid(image_path: str | Path, *, image: NDArray[Any] | None = None) -> SynthIDDetection:
|
||||
"""Detect the supported periodic carrier in IMAGE_PATH.
|
||||
|
||||
``not_detected`` is not a clean-image guarantee. It means only that the
|
||||
frozen periodic carrier did not cross its calibrated threshold.
|
||||
"""
|
||||
path = Path(image_path)
|
||||
if image is None:
|
||||
width, height = _image_size(path)
|
||||
else:
|
||||
if image.ndim != 3 or image.shape[2] != 3:
|
||||
raise ValueError("image must be a three-channel BGR array")
|
||||
height, width = image.shape[:2]
|
||||
if not _geometry_supported(width, height):
|
||||
return SynthIDDetection(
|
||||
status="unsupported",
|
||||
width=width,
|
||||
height=height,
|
||||
score=None,
|
||||
threshold=TILE_THRESHOLD,
|
||||
)
|
||||
if not is_available():
|
||||
raise RuntimeError(f"SynthID pixel detection needs numpy and OpenCV; {INSTALL_HINT}")
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
template, sigma, _model_height, _model_width, tile_height, tile_width = _load_template()
|
||||
if image is None:
|
||||
with Image.open(path) as source:
|
||||
pixels = np.asarray(source.convert("RGB"), dtype=np.uint8)
|
||||
else:
|
||||
pixels = np.asarray(image[:, :, ::-1], dtype=np.uint8)
|
||||
if pixels.shape != (height, width, 3):
|
||||
raise RuntimeError("decoded image geometry does not match its header")
|
||||
folded = fold_residual_template(
|
||||
pixels,
|
||||
tile_height=tile_height,
|
||||
tile_width=tile_width,
|
||||
denoise_sigma=sigma,
|
||||
)
|
||||
normalized, _norm = unit_tile(folded)
|
||||
score = float(np.sum(template * normalized))
|
||||
return SynthIDDetection(
|
||||
status="detected" if score >= TILE_THRESHOLD else "not_detected",
|
||||
width=width,
|
||||
height=height,
|
||||
score=score,
|
||||
threshold=TILE_THRESHOLD,
|
||||
)
|
||||
@@ -18,8 +18,12 @@ CHATGPT = SAMPLES / "chatgpt-1.png"
|
||||
|
||||
class TestTopLevelExports:
|
||||
def test_lazy_reexports_resolve(self):
|
||||
from remove_ai_watermarks import synthid_detector
|
||||
|
||||
assert raiw.remove_visible is api.remove_visible
|
||||
assert raiw.visible_provenance is api.visible_provenance
|
||||
assert raiw.detect_synthid is synthid_detector.detect_synthid
|
||||
assert raiw.SynthIDDetection is synthid_detector.SynthIDDetection
|
||||
|
||||
def test_unknown_attribute_raises(self):
|
||||
with pytest.raises(AttributeError):
|
||||
|
||||
@@ -736,6 +736,28 @@ class TestIdentifyCommand:
|
||||
assert result.exit_code != 0
|
||||
|
||||
|
||||
class TestDetectSynthIDCommand:
|
||||
def test_help(self, runner):
|
||||
result = runner.invoke(main, ["detect-synthid", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "calibrated image sizes" in result.output
|
||||
|
||||
def test_unsupported_geometry_is_machine_readable(self, runner, tmp_clean_png):
|
||||
result = runner.invoke(main, ["detect-synthid", str(tmp_clean_png), "--json"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
payload = json.loads(result.output)
|
||||
assert payload["status"] == "unsupported"
|
||||
assert payload["score"] is None
|
||||
|
||||
def test_non_json_output_preserves_negative_scope(self, runner, tmp_clean_png):
|
||||
result = runner.invoke(main, ["detect-synthid", str(tmp_clean_png)])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "unsupported" in result.output
|
||||
assert "not proof that SynthID is absent" in result.output
|
||||
|
||||
|
||||
class TestBatchCommand:
|
||||
"""Tests for the 'batch' subcommand."""
|
||||
|
||||
|
||||
@@ -886,6 +886,32 @@ class TestIdentifyVisibleTextMarks:
|
||||
# ── Caveats and serialization ───────────────────────────────────────
|
||||
|
||||
|
||||
class TestSynthIDPixelCarrier:
|
||||
def test_positive_pixel_carrier_is_high_confidence_ai_evidence(self, tmp_clean_png: Path):
|
||||
with (
|
||||
patch("remove_ai_watermarks.identify._invisible_watermark", return_value=None),
|
||||
patch("remove_ai_watermarks.identify._synthid_pixel_watermark", return_value=True),
|
||||
patch("remove_ai_watermarks.identify._trustmark", return_value=None),
|
||||
):
|
||||
report = identify(tmp_clean_png, check_visible=False, check_invisible=True)
|
||||
|
||||
assert report.is_ai_generated is True
|
||||
assert report.confidence == "high"
|
||||
assert any(signal.name == "synthid_pixel" for signal in report.signals)
|
||||
assert any("positive-only" in caveat for caveat in report.caveats)
|
||||
|
||||
def test_negative_pixel_carrier_does_not_claim_clean(self, tmp_clean_png: Path):
|
||||
with (
|
||||
patch("remove_ai_watermarks.identify._invisible_watermark", return_value=None),
|
||||
patch("remove_ai_watermarks.identify._synthid_pixel_watermark", return_value=False),
|
||||
patch("remove_ai_watermarks.identify._trustmark", return_value=None),
|
||||
):
|
||||
report = identify(tmp_clean_png, check_visible=False, check_invisible=True)
|
||||
|
||||
assert report.is_ai_generated is None
|
||||
assert not any(signal.name == "synthid_pixel" for signal in report.signals)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SAMPLES_DIR.exists(), reason="data/fixtures/provenance not present")
|
||||
class TestIdentifyCaveats:
|
||||
def test_legacy_openai_has_no_synthid_claim(self):
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Runtime tests for the positive-only SynthID periodic carrier detector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import remove_ai_watermarks.synthid_detector as detector
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def supported_images(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]:
|
||||
"""Create supported-geometry positive and negative synthetic fixtures."""
|
||||
directory = tmp_path_factory.mktemp("synthid-detector")
|
||||
template, *_model = detector._load_template()
|
||||
scaled_tile = np.rint(template / np.max(np.abs(template)))
|
||||
marked = np.full((detector.MODEL_HEIGHT, detector.MODEL_WIDTH, 3), 128, dtype=np.float64)
|
||||
marked += np.tile(scaled_tile, (128, 128, 1))
|
||||
|
||||
positive = directory / "positive.png"
|
||||
negative = directory / "negative.png"
|
||||
Image.fromarray(np.clip(np.rint(marked), 0, 255).astype(np.uint8), "RGB").save(positive)
|
||||
Image.new("RGB", (detector.MODEL_WIDTH, detector.MODEL_HEIGHT), (128, 128, 128)).save(negative)
|
||||
return positive, negative
|
||||
|
||||
|
||||
def test_bundled_model_is_the_frozen_calibrated_artifact() -> None:
|
||||
model = Path(detector.__file__).parent / "assets" / detector.MODEL_FILENAME
|
||||
|
||||
assert hashlib.sha256(model.read_bytes()).hexdigest() == (
|
||||
"ee7838da8542c206c3403284b68e98f0ac99429e82f262c1a438f50a638b488b"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("width", "height"),
|
||||
[(1000, 1000), (1001, 1000), (3000, 6000), (768, 1364)],
|
||||
)
|
||||
def test_supported_geometry_uses_the_challenged_pixel_count_range(width: int, height: int) -> None:
|
||||
assert detector._geometry_supported(width, height)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("width", "height"),
|
||||
[(999, 1000), (3001, 6000), (64, 32)],
|
||||
)
|
||||
def test_geometry_outside_the_challenged_pixel_count_range_is_unsupported(
|
||||
width: int,
|
||||
height: int,
|
||||
) -> None:
|
||||
assert not detector._geometry_supported(width, height)
|
||||
|
||||
|
||||
def test_detects_supported_periodic_carrier(supported_images: tuple[Path, Path]) -> None:
|
||||
positive, _negative = supported_images
|
||||
|
||||
result = detector.detect_synthid(positive)
|
||||
|
||||
assert result.status == "detected"
|
||||
assert result.detected is True
|
||||
assert result.score is not None
|
||||
assert result.score > result.threshold
|
||||
assert result.to_dict()["detector"] == detector.DETECTOR_ID
|
||||
|
||||
|
||||
def test_detects_unregistered_non_divisible_geometry_in_size_range(tmp_path: Path) -> None:
|
||||
width, height = 1001, 1000
|
||||
template, *_model = detector._load_template()
|
||||
scaled_tile = np.rint(template / np.max(np.abs(template)))
|
||||
repeats_y = (height + scaled_tile.shape[0] - 1) // scaled_tile.shape[0]
|
||||
repeats_x = (width + scaled_tile.shape[1] - 1) // scaled_tile.shape[1]
|
||||
carrier = np.tile(scaled_tile, (repeats_y, repeats_x, 1))[:height, :width]
|
||||
pixels = np.clip(np.rint(carrier + 128.0), 0, 255).astype(np.uint8)
|
||||
path = tmp_path / "non-divisible-positive.png"
|
||||
Image.fromarray(pixels, "RGB").save(path)
|
||||
|
||||
result = detector.detect_synthid(path)
|
||||
|
||||
assert result.status == "detected"
|
||||
assert (result.width, result.height) == (width, height)
|
||||
assert result.score is not None
|
||||
assert result.score > result.threshold
|
||||
|
||||
|
||||
def test_supported_negative_does_not_claim_clean(supported_images: tuple[Path, Path]) -> None:
|
||||
_positive, negative = supported_images
|
||||
|
||||
result = detector.detect_synthid(negative)
|
||||
|
||||
assert result.status == "not_detected"
|
||||
assert result.detected is False
|
||||
assert result.score == pytest.approx(0.0)
|
||||
|
||||
|
||||
def test_threshold_mutation_changes_the_real_verdict(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
supported_images: tuple[Path, Path],
|
||||
) -> None:
|
||||
positive, _negative = supported_images
|
||||
baseline = detector.detect_synthid(positive)
|
||||
assert baseline.score is not None
|
||||
assert baseline.status == "detected"
|
||||
mutated_threshold = float(np.nextafter(baseline.score, np.inf))
|
||||
assert mutated_threshold > baseline.score
|
||||
|
||||
monkeypatch.setattr(detector, "TILE_THRESHOLD", mutated_threshold)
|
||||
mutated = detector.detect_synthid(positive)
|
||||
|
||||
assert mutated.status == "not_detected"
|
||||
assert mutated.threshold == mutated_threshold
|
||||
|
||||
|
||||
def test_unsupported_geometry_is_distinct_from_negative(tmp_path: Path) -> None:
|
||||
path = tmp_path / "small.png"
|
||||
Image.new("RGB", (64, 32), "white").save(path)
|
||||
|
||||
result = detector.detect_synthid(path)
|
||||
|
||||
assert result.status == "unsupported"
|
||||
assert result.score is None
|
||||
assert (result.width, result.height) == (64, 32)
|
||||
|
||||
|
||||
def test_shared_bgr_decode_matches_file_decode(supported_images: tuple[Path, Path]) -> None:
|
||||
import cv2
|
||||
|
||||
positive, _negative = supported_images
|
||||
bgr = cv2.imread(str(positive))
|
||||
assert bgr is not None
|
||||
|
||||
from_file = detector.detect_synthid(positive)
|
||||
from_array = detector.detect_synthid(positive, image=bgr)
|
||||
|
||||
assert from_array == from_file
|
||||
|
||||
|
||||
def test_supported_geometry_requires_pixel_dependencies(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
supported_images: tuple[Path, Path],
|
||||
) -> None:
|
||||
_positive, negative = supported_images
|
||||
monkeypatch.setattr(detector, "is_available", lambda: False)
|
||||
|
||||
with pytest.raises(RuntimeError, match="pixel extra"):
|
||||
detector.detect_synthid(negative)
|
||||
|
||||
|
||||
def test_fold_accepts_non_divisible_geometry_without_resampling() -> None:
|
||||
rng = np.random.default_rng(20260810)
|
||||
tile = rng.normal(0.0, 8.0, size=(16, 16, 3))
|
||||
repeated = np.tile(tile, (19, 20, 1)) + 128.0
|
||||
|
||||
divisible = detector.fold_residual_template(
|
||||
repeated,
|
||||
tile_height=16,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
non_divisible = detector.fold_residual_template(
|
||||
repeated[:299, :317],
|
||||
tile_height=16,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
divisible_unit, _ = detector.unit_tile(divisible)
|
||||
non_divisible_unit, _ = detector.unit_tile(non_divisible)
|
||||
|
||||
assert non_divisible.shape == (16, 16, 3)
|
||||
assert float(np.sum(divisible_unit * non_divisible_unit)) > 0.999
|
||||
|
||||
|
||||
def test_non_divisible_fold_matches_modulo_cell_means() -> None:
|
||||
import cv2
|
||||
|
||||
rng = np.random.default_rng(44041)
|
||||
pixels = rng.integers(0, 256, size=(53, 71, 3), dtype=np.uint8)
|
||||
source = pixels.astype(np.float32)
|
||||
residual = source - cv2.GaussianBlur(
|
||||
source,
|
||||
(0, 0),
|
||||
sigmaX=1.25,
|
||||
sigmaY=1.25,
|
||||
borderType=cv2.BORDER_REFLECT_101,
|
||||
)
|
||||
expected = np.empty((16, 16, 3), dtype=np.float64)
|
||||
for tile_y in range(16):
|
||||
for tile_x in range(16):
|
||||
expected[tile_y, tile_x] = residual[tile_y::16, tile_x::16].mean(
|
||||
axis=(0, 1),
|
||||
dtype=np.float64,
|
||||
)
|
||||
expected -= np.mean(expected, axis=(0, 1), keepdims=True)
|
||||
|
||||
actual = detector.fold_residual_template(
|
||||
pixels,
|
||||
tile_height=16,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.25,
|
||||
)
|
||||
|
||||
np.testing.assert_allclose(actual, expected, rtol=0.0, atol=0.0)
|
||||
|
||||
|
||||
def test_fold_rejects_tile_larger_than_image() -> None:
|
||||
pixels = np.zeros((15, 16, 3), dtype=np.uint8)
|
||||
|
||||
with pytest.raises(ValueError, match="at least as large"):
|
||||
detector.fold_residual_template(
|
||||
pixels,
|
||||
tile_height=16,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
@@ -0,0 +1,212 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import synthid_oracle_batch as batch
|
||||
from synthid_periodic_tile import fold_residual_template, unit_tile
|
||||
from synthid_periodic_tile_probe import PeriodicTileModel
|
||||
from synthid_periodic_tile_probe import save_model as save_tile_model
|
||||
from synthid_phase_carrier import PhaseCarrierModel
|
||||
from synthid_phase_carrier import save_model as save_phase_model
|
||||
|
||||
|
||||
def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
|
||||
rng = np.random.default_rng(23)
|
||||
raw_tile = rng.normal(size=(8, 8, 3))
|
||||
raw_tile -= np.mean(raw_tile, axis=(0, 1), keepdims=True)
|
||||
raw_tile, _ = unit_tile(raw_tile)
|
||||
source = np.clip(np.rint(128.0 + 24.0 * np.tile(raw_tile, (8, 8, 1))), 0, 255).astype(np.uint8)
|
||||
source_path = tmp_path / "source.png"
|
||||
Image.fromarray(source, mode="RGB").save(source_path)
|
||||
|
||||
folded = fold_residual_template(
|
||||
source,
|
||||
tile_height=8,
|
||||
tile_width=8,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
template, expected_norm = unit_tile(folded)
|
||||
tile_model = PeriodicTileModel(64, 64, 8, 8, 1.0, template, expected_norm)
|
||||
tile_model_path = tmp_path / "tile-model.npz"
|
||||
save_tile_model(tile_model_path, tile_model)
|
||||
|
||||
spectra = np.stack([np.fft.rfft2(source[:, :, channel]) for channel in range(3)], axis=2)
|
||||
magnitude = np.abs(spectra)
|
||||
magnitude[0, 0, :] = 0.0
|
||||
row, column, channel = np.unravel_index(int(np.argmax(magnitude)), magnitude.shape)
|
||||
phase_model = PhaseCarrierModel(
|
||||
64,
|
||||
64,
|
||||
np.asarray([row], dtype=np.int32),
|
||||
np.asarray([column], dtype=np.int32),
|
||||
np.asarray([channel], dtype=np.int8),
|
||||
np.asarray([np.angle(spectra[row, column, channel])]),
|
||||
np.asarray([1.0]),
|
||||
np.asarray([magnitude[row, column, channel]]),
|
||||
)
|
||||
phase_model_path = tmp_path / "phase-model.npz"
|
||||
save_phase_model(phase_model_path, phase_model)
|
||||
return source_path, tile_model_path, phase_model_path
|
||||
|
||||
|
||||
def _build(tmp_path: Path) -> tuple[Path, Path]:
|
||||
source, tile_model, phase_model = _fixture(tmp_path)
|
||||
output_dir = tmp_path / "oracle-batch"
|
||||
manifest_path = batch.build_batch(
|
||||
[source],
|
||||
output_dir=output_dir,
|
||||
tile_model_path=tile_model,
|
||||
phase_model_path=phase_model,
|
||||
tile_threshold=0.5,
|
||||
phase_threshold=0.5,
|
||||
active_threshold=0.0,
|
||||
strength=2.0,
|
||||
seed=20260810,
|
||||
provider="google",
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
return output_dir, manifest_path
|
||||
|
||||
|
||||
def test_build_preregisters_fixed_request_order_without_copying_source(tmp_path: Path) -> None:
|
||||
output_dir, manifest_path = _build(tmp_path)
|
||||
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
assert manifest["status"] == "preregistered_unsubmitted"
|
||||
assert manifest["request_count"] == 5
|
||||
assert [row["role"] for row in manifest["rows"]] == list(batch.ROLE_ORDER)
|
||||
assert manifest["rows"][0]["in_batch"] is False
|
||||
assert not (output_dir / "source.png").exists()
|
||||
assert manifest["provider"] == "google"
|
||||
assert "shifted, and orthogonal_random are detected" in manifest["decision_rule"]
|
||||
assert all(row["synthid_result"] is None for row in manifest["rows"])
|
||||
assert all(row["c2pa_result"] is None for row in manifest["rows"])
|
||||
template = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8"))
|
||||
assert template["manifest_sha256"] == batch.artifact_sha256(manifest_path)
|
||||
assert [row["artifact_sha256"] for row in template["rows"]] == [row["artifact_sha256"] for row in manifest["rows"]]
|
||||
assert (
|
||||
batch.verify_batch(
|
||||
manifest_path,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)["source_count"]
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_verify_rejects_derivative_mutation(tmp_path: Path) -> None:
|
||||
output_dir, manifest_path = _build(tmp_path)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
aligned_row = next(row for row in manifest["rows"] if row["role"] == "aligned")
|
||||
aligned_path = output_dir / aligned_row["path"]
|
||||
with Image.open(aligned_path) as image:
|
||||
pixels = np.asarray(image.convert("RGB"), dtype=np.uint8).copy()
|
||||
pixels[0, 0, 0] ^= 1
|
||||
Image.fromarray(pixels, mode="RGB").save(aligned_path)
|
||||
assert batch.artifact_sha256(aligned_path) != aligned_row["artifact_sha256"]
|
||||
|
||||
with pytest.raises(ValueError, match="artifact hash mismatch"):
|
||||
batch.verify_batch(
|
||||
manifest_path,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
|
||||
|
||||
def test_build_rejects_output_inside_repository(tmp_path: Path) -> None:
|
||||
source, tile_model, phase_model = _fixture(tmp_path)
|
||||
repository_root = Path(__file__).resolve().parent.parent
|
||||
|
||||
with pytest.raises(ValueError, match="outside the repository"):
|
||||
batch.build_batch(
|
||||
[source],
|
||||
output_dir=repository_root / ".local-eval/oracle-batch-test",
|
||||
tile_model_path=tile_model,
|
||||
phase_model_path=phase_model,
|
||||
tile_threshold=0.5,
|
||||
phase_threshold=0.5,
|
||||
active_threshold=0.0,
|
||||
strength=2.0,
|
||||
seed=20260810,
|
||||
provider="google",
|
||||
repository_root=repository_root,
|
||||
)
|
||||
|
||||
|
||||
def test_build_rejects_nonempty_output_directory(tmp_path: Path) -> None:
|
||||
source, tile_model, phase_model = _fixture(tmp_path)
|
||||
output_dir = tmp_path / "existing"
|
||||
output_dir.mkdir()
|
||||
(output_dir / "marker.txt").write_text("occupied", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="must not already contain"):
|
||||
batch.build_batch(
|
||||
[source],
|
||||
output_dir=output_dir,
|
||||
tile_model_path=tile_model,
|
||||
phase_model_path=phase_model,
|
||||
tile_threshold=0.5,
|
||||
phase_threshold=0.5,
|
||||
active_threshold=0.0,
|
||||
strength=2.0,
|
||||
seed=20260810,
|
||||
provider="google",
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
|
||||
|
||||
def test_evaluate_requires_controls_and_aligned_outcome(tmp_path: Path) -> None:
|
||||
output_dir, manifest_path = _build(tmp_path)
|
||||
results = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8"))
|
||||
for row in results["rows"]:
|
||||
row["synthid_result"] = "not_detected" if row["role"] == "aligned" else "detected"
|
||||
row["c2pa_result"] = "unavailable"
|
||||
row["raw_response"] = f"verbatim {row['role']} result"
|
||||
row["submitted_at"] = "2026-08-10T20:00:00Z"
|
||||
results_path = output_dir / "results.json"
|
||||
results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
report = batch.evaluate_results(
|
||||
manifest_path,
|
||||
results_path,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
|
||||
assert report["counts"] == {
|
||||
"causal_success": 1,
|
||||
"aligned_still_detected": 0,
|
||||
"control_failed": 0,
|
||||
"indeterminate": 0,
|
||||
}
|
||||
|
||||
shifted = next(row for row in results["rows"] if row["role"] == "shifted")
|
||||
shifted["synthid_result"] = "not_detected"
|
||||
results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
control_report = batch.evaluate_results(
|
||||
manifest_path,
|
||||
results_path,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
assert control_report["counts"]["control_failed"] == 1
|
||||
|
||||
|
||||
def test_evaluate_rejects_incomplete_result(tmp_path: Path) -> None:
|
||||
output_dir, manifest_path = _build(tmp_path)
|
||||
results = json.loads((output_dir / "results-template.json").read_text(encoding="utf-8"))
|
||||
results["rows"].pop()
|
||||
results_path = output_dir / "incomplete-results.json"
|
||||
results_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="cover every"):
|
||||
batch.evaluate_results(
|
||||
manifest_path,
|
||||
results_path,
|
||||
repository_root=Path(__file__).resolve().parent.parent,
|
||||
)
|
||||
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
import synthid_periodic_tile_ablation as ablation
|
||||
from synthid_periodic_tile import fold_residual_template, unit_tile
|
||||
from synthid_periodic_tile_probe import PeriodicTileModel
|
||||
from synthid_phase_carrier import PhaseCarrierModel
|
||||
|
||||
|
||||
def test_exact_sign_test_detects_one_sided_direction() -> None:
|
||||
assert ablation.exact_sign_test(30, 0) == pytest.approx(1.862645149230957e-9)
|
||||
assert ablation.exact_sign_test(0, 0) == 1.0
|
||||
|
||||
|
||||
def test_control_templates_are_norm_matched_and_random_control_is_orthogonal() -> None:
|
||||
rng = np.random.default_rng(7)
|
||||
template, _ = unit_tile(rng.normal(size=(8, 8, 3)))
|
||||
|
||||
controls = ablation.control_templates(template, seed=11)
|
||||
|
||||
assert set(controls) == {"aligned", "shifted", "orthogonal_random"}
|
||||
assert all(np.linalg.norm(control) == pytest.approx(1.0) for control in controls.values())
|
||||
assert np.sum(controls["orthogonal_random"] * template) == pytest.approx(0.0, abs=1e-12)
|
||||
assert not np.array_equal(controls["shifted"], template)
|
||||
|
||||
|
||||
def test_aligned_subtraction_controls_both_synthetic_representations(tmp_path: Path) -> None:
|
||||
rng = np.random.default_rng(17)
|
||||
raw_tile = rng.normal(size=(8, 8, 3))
|
||||
raw_tile -= np.mean(raw_tile, axis=(0, 1), keepdims=True)
|
||||
raw_tile, _ = unit_tile(raw_tile)
|
||||
source = np.clip(np.rint(128.0 + 24.0 * np.tile(raw_tile, (8, 8, 1))), 0, 255).astype(np.uint8)
|
||||
source_path = tmp_path / "source.png"
|
||||
Image.fromarray(source, mode="RGB").save(source_path)
|
||||
|
||||
folded = fold_residual_template(
|
||||
source,
|
||||
tile_height=8,
|
||||
tile_width=8,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
template, expected_norm = unit_tile(folded)
|
||||
tile_model = PeriodicTileModel(
|
||||
height=64,
|
||||
width=64,
|
||||
tile_height=8,
|
||||
tile_width=8,
|
||||
denoise_sigma=1.0,
|
||||
template=template,
|
||||
expected_norm=expected_norm,
|
||||
)
|
||||
|
||||
spectra = np.stack([np.fft.rfft2(source[:, :, channel]) for channel in range(3)], axis=2)
|
||||
magnitude = np.abs(spectra)
|
||||
magnitude[0, 0, :] = 0.0
|
||||
row, column, channel = np.unravel_index(int(np.argmax(magnitude)), magnitude.shape)
|
||||
phase_model = PhaseCarrierModel(
|
||||
height=64,
|
||||
width=64,
|
||||
rows=np.asarray([row], dtype=np.int32),
|
||||
columns=np.asarray([column], dtype=np.int32),
|
||||
channels=np.asarray([channel], dtype=np.int8),
|
||||
phases=np.asarray([np.angle(spectra[row, column, channel])]),
|
||||
weights=np.asarray([1.0]),
|
||||
expected_magnitudes=np.asarray([magnitude[row, column, channel]]),
|
||||
)
|
||||
|
||||
report = ablation.run_ablation(
|
||||
[source_path],
|
||||
tile_model=tile_model,
|
||||
phase_model=phase_model,
|
||||
tile_threshold=0.5,
|
||||
phase_threshold=0.5,
|
||||
active_threshold=0.0,
|
||||
strengths=(1.0, 2.0),
|
||||
phase_strength=2.0,
|
||||
seed=20260810,
|
||||
)
|
||||
|
||||
assert report["original"] == {"tile_accepted": 1, "phase_accepted": 1}
|
||||
aligned = next(row for row in report["phase_summaries"] if row["control"] == "aligned")
|
||||
assert aligned["accepted"] == 0
|
||||
comparisons = {(row["aligned_minus"], row["metric"]): row for row in report["paired_comparisons"]}
|
||||
assert comparisons[("shifted", "tile_delta")]["difference"]["median"] < 0.0
|
||||
assert comparisons[("orthogonal_random", "tile_delta")]["difference"]["median"] < 0.0
|
||||
|
||||
|
||||
def test_phase_strength_must_be_part_of_sweep() -> None:
|
||||
template = np.zeros((8, 8, 3), dtype=np.float64)
|
||||
template[0, 0, 0] = 1.0
|
||||
tile_model = PeriodicTileModel(64, 64, 8, 8, 1.0, template, 1.0)
|
||||
phase_model = PhaseCarrierModel(
|
||||
64,
|
||||
64,
|
||||
np.asarray([1]),
|
||||
np.asarray([1]),
|
||||
np.asarray([0]),
|
||||
np.asarray([0.0]),
|
||||
np.asarray([1.0]),
|
||||
np.asarray([1.0]),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="phase strength"):
|
||||
ablation.run_ablation(
|
||||
[Path("unused.png")],
|
||||
tile_model=tile_model,
|
||||
phase_model=phase_model,
|
||||
tile_threshold=0.0,
|
||||
phase_threshold=0.0,
|
||||
active_threshold=0.0,
|
||||
strengths=(1.0,),
|
||||
phase_strength=2.0,
|
||||
seed=1,
|
||||
)
|
||||
@@ -80,6 +80,25 @@ def test_registration_recovers_cyclic_tile_shift(tmp_path: Path) -> None:
|
||||
assert (registered.row_shift, registered.column_shift) == (7, 6)
|
||||
|
||||
|
||||
def test_array_scoring_matches_file_scoring(tmp_path: Path) -> None:
|
||||
carrier = _carrier(12)
|
||||
positives = []
|
||||
for index in range(3):
|
||||
path = tmp_path / f"positive-{index}.png"
|
||||
_write_image(path, carrier, seed=index)
|
||||
positives.append(path)
|
||||
model = probe.discover_model(positives, tile_height=8, tile_width=8)
|
||||
with Image.open(positives[0]) as image:
|
||||
pixels = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
|
||||
file_score = probe.score_image(positives[0], model)
|
||||
array_score = probe.score_pixels(pixels, model)
|
||||
|
||||
assert array_score.score == pytest.approx(file_score.score)
|
||||
assert array_score.active_support == pytest.approx(file_score.active_support)
|
||||
assert array_score.path == "<array>"
|
||||
|
||||
|
||||
def test_fft_correlations_match_explicit_cyclic_shifts() -> None:
|
||||
rng = np.random.default_rng(30)
|
||||
template = rng.normal(size=(4, 5, 3))
|
||||
|
||||
@@ -104,6 +104,24 @@ def test_scoring_rejects_geometry_mismatch(tmp_path: Path) -> None:
|
||||
carrier.score_image(mismatch, model)
|
||||
|
||||
|
||||
def test_array_scoring_matches_file_scoring(tmp_path: Path) -> None:
|
||||
positives: list[Path] = []
|
||||
for index in range(3):
|
||||
path = tmp_path / f"positive-{index}.png"
|
||||
_write_image(path, phase=0.4, seed=index)
|
||||
positives.append(path)
|
||||
model = carrier.discover_model(positives, peak_count=4, min_radius=1.0)
|
||||
with Image.open(positives[0]) as image:
|
||||
pixels = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
|
||||
file_score = carrier.score_image(positives[0], model)
|
||||
array_score = carrier.score_pixels(pixels, model)
|
||||
|
||||
assert array_score.score == pytest.approx(file_score.score)
|
||||
assert array_score.active_weight_fraction == pytest.approx(file_score.active_weight_fraction)
|
||||
assert array_score.path == "<array>"
|
||||
|
||||
|
||||
def test_scoring_can_canonicalize_geometry(tmp_path: Path) -> None:
|
||||
positives: list[Path] = []
|
||||
for index in range(3):
|
||||
|
||||
@@ -4,7 +4,6 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
|
||||
@@ -44,13 +43,15 @@ def test_subtraction_reduces_repeated_tile_energy() -> None:
|
||||
assert after < before
|
||||
|
||||
|
||||
def test_folding_rejects_nondivisible_geometry() -> None:
|
||||
def test_folding_accepts_nondivisible_geometry() -> None:
|
||||
pixels = np.zeros((63, 64, 3), dtype=np.uint8)
|
||||
|
||||
with pytest.raises(ValueError, match="divisible"):
|
||||
fold_residual_template(
|
||||
pixels,
|
||||
tile_height=8,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
folded = fold_residual_template(
|
||||
pixels,
|
||||
tile_height=8,
|
||||
tile_width=16,
|
||||
denoise_sigma=1.0,
|
||||
)
|
||||
|
||||
assert folded.shape == (8, 16, 3)
|
||||
assert np.count_nonzero(folded) == 0
|
||||
|
||||
Reference in New Issue
Block a user