From b011f0f9627b3f4c3177572d5f5dce5a0605c5eb Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Sun, 9 Aug 2026 18:40:45 -0700 Subject: [PATCH] Add rigorous SynthID research and evaluation harness --- data/synthid/README.md | 5 +- data/synthid/research-manifest.md | 91 +++ docs/index.md | 1 + docs/synthid-detector-removal-plan.md | 834 ++++++++++++++++++++++ docs/synthid.md | 119 ++- docs/verification-plan.md | 19 +- scripts/synthid_color_space_probe.py | 352 +++++++++ scripts/synthid_confound_probe.py | 478 +++++++++++++ scripts/synthid_consensus_probe.py | 259 +++++++ scripts/synthid_ensemble_attack.py | 223 ++++++ scripts/synthid_ensemble_detector.py | 188 +++++ scripts/synthid_fragment_attack.py | 243 +++++++ scripts/synthid_hybrid_attack.py | 140 ++++ scripts/synthid_phase_carrier.py | 312 ++++++++ scripts/synthid_pixel_attack.py | 199 ++++++ scripts/synthid_research_inventory.py | 170 +++++ scripts/synthid_research_manifest.py | 347 +++++++++ scripts/synthid_spectral_probe.py | 312 ++++++++ scripts/synthid_tile_attack.py | 166 +++++ scripts/synthid_v3_carrier_subtract.py | 148 ++++ scripts/synthid_v3_codebook_probe.py | 172 +++++ tests/test_synthid_color_space_probe.py | 105 +++ tests/test_synthid_confound_probe.py | 243 +++++++ tests/test_synthid_consensus_probe.py | 73 ++ tests/test_synthid_ensemble_attack.py | 90 +++ tests/test_synthid_ensemble_detector.py | 86 +++ tests/test_synthid_fragment_attack.py | 92 +++ tests/test_synthid_hybrid_attack.py | 50 ++ tests/test_synthid_phase_carrier.py | 120 ++++ tests/test_synthid_pixel_attack.py | 62 ++ tests/test_synthid_research_inventory.py | 107 +++ tests/test_synthid_research_manifest.py | 252 +++++++ tests/test_synthid_spectral_probe.py | 103 +++ tests/test_synthid_tile_attack.py | 55 ++ tests/test_synthid_v3_carrier_subtract.py | 66 ++ tests/test_synthid_v3_codebook_probe.py | 59 ++ 36 files changed, 6307 insertions(+), 34 deletions(-) create mode 100644 data/synthid/research-manifest.md create mode 100644 docs/synthid-detector-removal-plan.md create mode 100644 scripts/synthid_color_space_probe.py create mode 100644 scripts/synthid_confound_probe.py create mode 100644 scripts/synthid_consensus_probe.py create mode 100644 scripts/synthid_ensemble_attack.py create mode 100644 scripts/synthid_ensemble_detector.py create mode 100644 scripts/synthid_fragment_attack.py create mode 100644 scripts/synthid_hybrid_attack.py create mode 100644 scripts/synthid_phase_carrier.py create mode 100644 scripts/synthid_pixel_attack.py create mode 100644 scripts/synthid_research_inventory.py create mode 100644 scripts/synthid_research_manifest.py create mode 100644 scripts/synthid_spectral_probe.py create mode 100644 scripts/synthid_tile_attack.py create mode 100644 scripts/synthid_v3_carrier_subtract.py create mode 100644 scripts/synthid_v3_codebook_probe.py create mode 100644 tests/test_synthid_color_space_probe.py create mode 100644 tests/test_synthid_confound_probe.py create mode 100644 tests/test_synthid_consensus_probe.py create mode 100644 tests/test_synthid_ensemble_attack.py create mode 100644 tests/test_synthid_ensemble_detector.py create mode 100644 tests/test_synthid_fragment_attack.py create mode 100644 tests/test_synthid_hybrid_attack.py create mode 100644 tests/test_synthid_phase_carrier.py create mode 100644 tests/test_synthid_pixel_attack.py create mode 100644 tests/test_synthid_research_inventory.py create mode 100644 tests/test_synthid_research_manifest.py create mode 100644 tests/test_synthid_spectral_probe.py create mode 100644 tests/test_synthid_tile_attack.py create mode 100644 tests/test_synthid_v3_carrier_subtract.py create mode 100644 tests/test_synthid_v3_codebook_probe.py diff --git a/data/synthid/README.md b/data/synthid/README.md index 55cf3b5..96c01fe 100644 --- a/data/synthid/README.md +++ b/data/synthid/README.md @@ -54,7 +54,10 @@ versions, or runtime settings. Ground-truth quality, strongest first: - `gemini-app` — checked via the Gemini app "Verify with SynthID" feature. Gold standard for the pixel watermark (Google models). -- `openai-verify` — checked via openai.com/verify (gold standard for OpenAI ChatGPT/Codex/API images). +- `openai-verify` — checked through the OpenAI web verifier. This is the + historical manifest label for existing rows. OpenAI now also documents a + Content Provenance API with a separate SynthID result; add a distinct manifest + value before recording API-derived labels rather than silently reusing this one. - `synthid-portal` — checked via Google's SynthID Detector portal. - `c2pa-metadata` — supported provenance evidence (Google AI C2PA, or OpenAI C2PA with an explicit `c2pa.watermarked.*` action). Weaker than a provider diff --git a/data/synthid/research-manifest.md b/data/synthid/research-manifest.md new file mode 100644 index 0000000..1e1ac27 --- /dev/null +++ b/data/synthid/research-manifest.md @@ -0,0 +1,91 @@ +# Private SynthID research manifest + +The research manifest is a local CSV for detector and pixel-only removal +experiments. It is intentionally separate from `manifest.csv`, which describes +the small public regression corpus. + +Store the CSV and its media below `.local-eval/synthid/`. Keep one manifest per +detector target, for example `google-manifest.csv` and `openai-manifest.csv`. +Separate manifests may reference the same external negative bytes without +duplicating an artifact inside either provider's split graph. Do not commit +private media, corpus sizes, oracle sessions, or provider-access details. + +Before assigning labels, build a label-free inventory. It records byte and +decoded-pixel hashes, exact duplicates, geometry, and format, but deliberately +has no provider, oracle, outcome, or split columns: + +```bash +uv run python scripts/synthid_research_inventory.py \ + --root .local-eval/synthid negatives google openai \ + --inventory-out .local-eval/synthid/inventory.csv +``` + +An inventory row becomes a manifest row only after its source, detector target, +label evidence, group lineage, and split are established independently. A +directory name is not evidence and must not be promoted mechanically. + +Audit each local manifest with: + +```bash +uv run python scripts/synthid_research_manifest.py \ + .local-eval/synthid/google-manifest.csv --verify-files +``` + +Once a provider manifest has ordinary positives and negatives in train, +validation, and locked-test splits, run the D1 confound challenge: + +```bash +uv run --extra pixels python scripts/synthid_confound_probe.py \ + .local-eval/synthid/google-manifest.csv \ + --target-provider google \ + --report-out .local-eval/synthid/google-d1-confounds.json +``` + +D1 excludes candidate, sham, and source-control rows. Its `container`, +`thumbnail`, and `canonical` baselines intentionally measure how well export +fields, geometry, and coarse generator/content style can imitate detection. A +report is evidence-ready only when the locked test includes a same-provider hard +negative and the temporal split contains both labels. Evidence readiness does +not mean the D1 gate passed; a candidate signal still has to beat the frozen +canonical baseline on those controls. + +## Columns + +| Column | Meaning | +| --- | --- | +| `artifact_sha256` | SHA-256 of the exact file submitted or measured. | +| `pixel_sha256` | SHA-256 of decoded RGB bytes, used to catch lossless duplicates. | +| `artifact_path` | Safe path relative to the manifest. | +| `parent_sha256` | Exact parent artifact for a derivative, empty for an original. | +| `group_id` | Leakage boundary shared by an original and all semantic or transformed siblings. | +| `target_provider` | Detector target, `openai` or `google`. | +| `source_provider` | Actual source family: `openai`, `google`, `camera`, `other_ai`, `synthetic`, or `editor`. | +| `surface` | Product or export surface. | +| `model_epoch` | Model family plus a dated epoch when the exact version is unavailable. | +| `generation_session` | Groups outputs created in one provider session. | +| `content_stratum` | Predeclared content class. | +| `width`, `height`, `format` | Decoded geometry and file format. | +| `transform` | `original` or the reproducible transform applied to `parent_sha256`. | +| `split` | `discovery`, `train`, `validation`, `test`, or `temporal`. | +| `c2pa_outcome` | C2PA result, recorded independently from the pixel signal. | +| `synthid_outcome` | `detected`, `not_detected`, `indeterminate`, `refused`, or `not_checked`. | +| `verified_via` | Matching provider oracle, external source evidence, or `none`. | +| `evidence_reference` | Stable URL or local evidence-record reference required for `source-evidence`; never infer it from a directory name. | +| `oracle_session` | Groups allowed checks made in one session. | +| `oracle_role` | `ordinary`, `source_control`, `candidate`, or `sham`. | +| `captured_at`, `oracle_checked_at` | Timezone-aware ISO-8601 timestamps. | +| `notes` | Verbatim context that does not fit a structured column. | + +`source-evidence` may establish an ordinary external negative, but never a +positive or a same-provider negative. Provider positives and same-provider hard +negatives require the matching provider verifier. An `indeterminate`, `refused`, +or unchecked row may remain in `discovery`; it cannot enter a train, validation, +test, or temporal split. Every `source-evidence` row must retain the source URL +or evidence-record reference that establishes the claim. + +The auditor also rejects duplicate artifact hashes, identical decoded pixels in +different groups, derivatives without parents, cross-provider parentage, +lineage cycles, and any group crossing split boundaries. A `not_detected` +candidate or sham is valid only when the same provider, group, and oracle +session contains a detected `source_control`. A session that misses its control +cannot establish removal, even when its candidate response is negative. diff --git a/docs/index.md b/docs/index.md index d73188b..4156b41 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,6 +25,7 @@ to run the tool. Use the maintainer references only when changing the code. | [Release and distribution](release-and-distribution.md) | PyPI, Homebrew, Hugging Face Space, and release workflow. | | [Watermarking landscape](watermarking-landscape.md) | Vendor signals and detection approaches. | | [SynthID technical reference](synthid.md) | Mechanism, detector access, robustness, and implications for this project. | +| [SynthID detector and removal plan](synthid-detector-removal-plan.md) | Gated research program for provider-specific local detection and pixel-only removal. | ## Research archive diff --git a/docs/synthid-detector-removal-plan.md b/docs/synthid-detector-removal-plan.md new file mode 100644 index 0000000..12aa426 --- /dev/null +++ b/docs/synthid-detector-removal-plan.md @@ -0,0 +1,834 @@ +# SynthID detector and pixel-only removal research plan + +> Research plan, not a statement of current product capability. The shipped +> behavior remains documented in [supported signals](supported-signals.md), +> [known limitations](known-limitations.md), and +> [module internals](module-internals.md). + +## Objective + +Build two independent provider tracks with four capabilities: + +1. a local, metadata-independent detector for the OpenAI SynthID image signal; +2. a local, metadata-independent detector for the Google SynthID image signal; +3. a pixel-only remover for the OpenAI signal that does not use diffusion, + VAE reconstruction, semantic regeneration, or generative inpainting; +4. an independently calibrated pixel-only remover for the Google signal under + the same constraints. + +OpenAI is the first research track because it has a documented remote verifier. +Google follows with the same experimental protocol but its own corpus, labels, +model, thresholds, and oracle. No carrier, feature, score, or operating point +transfers between providers until a held-out experiment demonstrates that it +does. + +The detector target is signal presence, not payload recovery, provider +classification, or general AI-image classification. The removal target is a +minimal pixel residual that makes a source-positive image negative in the +matching provider oracle while preserving the image's geometry and semantics. + +## Non-negotiable evidence rules + +1. **Detector before remover.** A remover may be prototyped against synthetic + carriers, but no real-image removal claim is made until the local detector + passes its own held-out gate. +2. **Provider-specific ground truth.** OpenAI and Google labels come from their + matching verifier. C2PA is recorded separately and is never the pixel label. +3. **No metadata or export leakage.** Training and evaluation operate on decoded + pixels after controlled metadata stripping and matched re-encoding. Geometry, + filename, file size, chunks, encoder settings, and source directories cannot + be model inputs. +4. **Signal identifiability is a gate.** A classifier trained only on provider + positives and unrelated negatives can learn the provider's generator or + export fingerprint. It is called a provider classifier, not a SynthID + detector, until at least one causal control succeeds. +5. **The oracle is held out from optimization.** Candidate algorithms and + hyperparameters are selected locally. Oracle batches are immutable and + registered before submission. A remote binary verdict is never used as an + online loss or hill-climbing signal. +6. **A local score decrease is not removal evidence.** Removal requires a + source-positive, output-negative result from the matching provider oracle. +7. **One hypothesis does not become several signals.** Correlated spatial, + spectral, and color statistics derived from one residual are reported as one + line of evidence unless independent controls separate them. +8. **Every result is reproducible.** Input and output hashes, code revision, + model artifact hash, preprocessing, transform lineage, seed, score, threshold, + quality metrics, oracle result, session, and timestamp are retained. + +## Oracle boundary + +OpenAI documents a synchronous Content Provenance API at +`POST /v1/content_provenance_checks`. For images it returns separate `c2pa` and +`synthid` entries and supports PNG, JPEG, and WebP. It remains a remote, +OpenAI-scoped verifier, not a released local decoder. The same documentation +explicitly says not to use repeated queries to reverse-engineer, remove, or +evade a watermark. Adaptive detector or remover research against that endpoint +therefore requires explicit OpenAI authorization or a separate research oracle +whose terms permit the work. Without that authorization, the OpenAI track may +develop local hypotheses but stops before oracle-driven calibration and removal +certification. + +Google's Gemini verification flow is a Google-scoped oracle. Its documented +result can be detected, not detected, or unclear, and the consumer flow has a +small rolling quota. An unclear result is `indeterminate`, never a negative. + +For every permitted oracle batch: + +- verify the untouched source first; +- submit one file per request; +- record C2PA and SynthID outcomes independently when both are returned; +- preserve the exact submitted bytes and SHA-256 outside the public repository; +- retry only a transient failure on the same bytes; +- record `detected`, `not_detected`, `indeterminate`, or `refused` verbatim; +- submit a matched transform-only control before attributing a negative result + to an experimental edit; +- reserve a final temporal holdout that no feature, threshold, or remover has + seen. + +## Data design + +### Corpus layers + +Each provider gets a separate corpus with five layers: + +| Layer | Purpose | Required controls | +| --- | --- | --- | +| Verified positives | Learn and evaluate the real signal | Matching provider oracle, original bytes | +| Same-provider hard negatives | Separate watermark from generator identity | Same surface or model, oracle-negative | +| External hard negatives | Measure false positives | Cameras, scans, edited photos, other generators, synthetic graphics | +| Low-texture probes | Expose weak shared structure | Solid colors, gradients, ramps, checkerboards, sparse edges | +| Causal pairs | Attribute a residual to the watermark | Same underlying pixels, positive and confirmed negative | + +The causal-pair layer is the most valuable and the hardest to obtain. An +authorized encoder-off pair is ideal. A provider output and a pixel-only +processed version become a usable pair only after the original is positive and +the processed bytes are negative in the provider oracle. Public third-party +pairs are discovery material until their provenance and both labels are +independently verified. + +A negative created by a remover trained against detector A cannot train or +validate detector A in the same experiment. Reserve it for detector B or a later +model epoch after closing the originating experiment. Otherwise detector and +remover can certify each other's shared blind spot. + +If no same-provider hard negatives or causal pairs can be obtained, learned +real-image models stay explicitly labeled as provider classifiers. Spectral +repeatability on solid fills alone does not clear this gate. + +### Strata + +Record and split by: + +- provider, product surface, model family, and generation date window; +- native width, height, aspect ratio, file format, and color mode; +- photoreal, face, text-heavy, flat graphic, illustration, low texture, and + high texture content; +- untouched, metadata-stripped, lossless-normalized, JPEG/WebP, resized, + cropped, and color-adjusted lineage; +- generation session and prompt family; +- parent hash for every derivative. + +Store research media under `.local-eval/synthid/`, not in the public repository. +Only cleared fixtures may enter `data/synthid/`. Tracked files contain schemas, +scripts, synthetic fixtures, aggregate verdicts, and non-sensitive hashes. + +### Split discipline + +Deduplicate by decoded-pixel hash and perceptual similarity before splitting. +Keep every derivative, prompt sibling, and semantic near-duplicate in one hash +group. Split by group into train, validation, locked test, and a later temporal +test collected after the detector is frozen. A random image-level split is +invalid because it leaks transform and generation-family fingerprints. + +The negative set must be large enough to support the claimed operating point. +At zero observed false positives, roughly 3,000 independent negatives are +needed merely to put the one-sided 95% upper bound near 0.1% by the rule of +three. The final evaluation should prefer at least 10,000 hard and ordinary +negatives per provider, report confidence intervals, and report every negative +stratum separately rather than hiding a weak stratum in an aggregate. + +## Detector program + +### Experiment D0: oracle and corpus integrity + +Goal: prove that labels and bytes mean what the manifest says. + +- Separate SynthID from C2PA in every response. +- Confirm that metadata stripping changes C2PA but does not silently define the + SynthID label. +- Confirm every positive used for evaluation on its matching oracle. +- Mutation-test manifest ingestion with swapped provider, duplicate bytes, + derivative leakage, an `indeterminate` result mislabeled as negative, and a + changed file after hashing. +- Measure permitted same-byte verifier reproducibility on a small preregistered + sample. Do not turn retries into adaptive querying. + +Gate: no ambiguous label path, no cross-provider oracle substitution, and no +train/test family leakage. + +### Experiment D1: export and generator confounds + +Goal: determine how accurately SynthID can appear to be detected when no +watermark-specific evidence is available. + +Train deliberately confounded baselines on file/container fields, dimensions, +RGB thumbnails, and generator-vs-camera content. Then repeat after canonical +decode, metadata removal, resolution matching, and hard-negative balancing. + +Gate: any proposed signal feature must beat the canonicalized confound baseline +on same-provider hard negatives and on a temporal holdout. Otherwise the result +is generator attribution. + +### Experiment D2: low-texture carrier discovery + +Goal: test whether a repeatable carrier component is observable when scene +texture is suppressed. + +Extend the existing `scripts/synthid_pixel_probe.py` measurements from grayscale +NCC to: + +- per-channel and opponent-color residuals; +- two-dimensional FFT magnitude and circular phase coherence; +- wavelet bands and multi-scale autocorrelation; +- resolution and aspect-ratio registration; +- cross-color, cross-session, cross-model, and cross-date agreement; +- matched clean synthetic fills and same-provider oracle-negative probes. + +Use leave-one-color, leave-one-session, and leave-one-resolution-out tests. +Fixed bins discovered and evaluated on the same images are descriptive only. + +Gate: a template learned on one subset must detect held-out positive probes above +matched negatives and survive a later collection window. Failure kills the +fixed-carrier branch but not the content-dependent branch. + +### Experiment D3: classical real-image detector + +Goal: establish the strongest interpretable baseline before a neural model. + +Candidate features include locally normalized high-pass residuals, FFT and +wavelet energy ratios, circular phase coherence, color-channel agreement, +block periodicity, and correlations against provider/resolution templates. +Fit regularized logistic regression and a shallow tree model. Calibration uses +only validation data. + +Report TPR at 0.1% FPR as the primary metric, with bootstrap confidence +intervals. AUROC, average precision, and an arbitrary accuracy percentage are +secondary. Also report worst-stratum FPR, temporal-holdout TPR, and score drift. + +Gate: advance the classical detector only if the locked test shows a stable +watermark-specific advantage over confound baselines. Do not choose a threshold +from the locked test. + +### Experiment D4: learned residual detector + +Goal: learn content-dependent evidence that a fixed template misses. + +Start with a small two-view model: + +1. a spatial view of locally normalized high-pass RGB residuals; +2. a frequency view containing log magnitude and phase-derived channels. + +Fuse only late features so each view can be ablated. Use aggressive content and +export balancing, group-aware sampling, and augmentations drawn from the paper's +transformation families. Never expose metadata, path, dimensions without +normalization, or encoder-specific byte patterns to the network. + +Train provider-specific models first. A shared backbone with provider-specific +heads is a later ablation, not the default architecture. Keep a second detector +family completely outside remover training so it can reveal surrogate overfit. + +Gate for a detector release candidate: + +- empirical FPR at or below 0.1% on the locked negative set; +- one-sided 95% TPR lower bound at or above 90% on untouched positives; +- no declared hard-negative stratum above 0.5% FPR; +- useful TPR on the temporal holdout and transformation suite; +- meaningful discrimination on causal pairs or same-provider hard negatives; +- calibrated `abstain` behavior outside supported providers and strata. + +These are research gates, not promises that the proprietary decoder's operating +point has been reproduced. + +### Experiment D5: robustness and drift + +Evaluate identity, JPEG and WebP, resize, crop, padding, rotation, color changes, +blur, noise, overlays, screenshots, and combinations. Preserve a matched +transform-only positive control for every attack family. Report both average +and worst-transform TPR at the frozen threshold. + +Repeat a small fixed collection after provider model or surface changes. A +shift in score distribution opens a new model epoch; it does not silently +recalibrate the old threshold. + +## Localization program + +A whole-image detector is not automatically a useful removal loss. Test whether +its evidence is spatially causal with three independent methods: + +- tile occlusion and replacement with distortion-matched controls; +- detector-gradient attribution for the learned model; +- phase or template energy mapped back to spatial blocks. + +For each source, create top-ranked, random, and bottom-ranked edits with equal +pixel norm and the same codec path. The oracle batch is registered before any +results return. + +Gate: top-ranked edits must reduce matching-oracle detection more often than +random edits under paired analysis. Aggregate per-image differences and confirm +the direction with a sign test. If localization does not transfer, do not build +a region remover around it. + +## Pixel-only removal program + +All candidates preserve image dimensions and avoid a generative decoder. Run +the following ladder in order. + +### Experiment R1: analytical carrier subtraction + +Estimate provider, model-epoch, and geometry-specific residual components from +verified probes or causal pairs. Test: + +- complex FFT projection with conjugate symmetry; +- wavelet-band projection; +- per-channel and opponent-color residual subtraction; +- spatially varying strength based on local texture and detector attribution. + +Sweep signed amplitude, not only attenuation. Include a sham edit with identical +pixel norm outside candidate bins. This is the highest-value path because a real +shared post-hoc residual could be removed at very high fidelity. + +### Experiment R2: constrained per-image optimization + +Optimize pixels against an ensemble of frozen local detectors. The objective +combines detector margin with L-infinity and L2 bounds, LPIPS or DISTS, MS-SSIM, +edge consistency, OCR preservation, and face-embedding preservation where +applicable. Expectation over transformations covers lossless export, JPEG/WebP, +small resize, and color conversion so the result is not a fragile local +adversarial example. + +The optimization may read only local detectors. The provider oracle evaluates +a frozen candidate batch afterward and never supplies gradients, search +direction, or per-step feedback. + +### Experiment R3: feed-forward residual remover + +If R2 transfers to the provider oracle, distill successful minimal residuals +into a compact image-to-residual network. Constrain the output amplitude and +frequency distribution explicitly. Train on one detector ensemble and select +on the held-out detector family. Preserve the per-image optimizer as the +reference implementation. + +### Experiment R4: detector-remover co-evaluation + +Evaluate four combinations separately: + +1. local detector positive, provider oracle positive; +2. local detector negative, provider oracle positive; +3. local detector positive, provider oracle negative; +4. both negative. + +Case 2 is the critical surrogate-overfit failure. Add every permitted example +to a future hard set only after the current experiment is closed; never tune and +report on the same oracle failure. + +## Removal success gate + +A provider-specific remover is a release candidate only when a locked, +source-positive evaluation shows all of the following: + +- at least 90% matching-oracle `not_detected` results, with `indeterminate` + counted as failure; +- a positive matched transform-only control for every evaluated source; +- exact-size output with no semantic regeneration stage; +- median PSNR at least 40 dB and fifth-percentile PSNR at least 35 dB; +- median SSIM at least 0.99, plus LPIPS or DISTS reported rather than optimized + silently; +- no OCR regression on text strata and no material face-identity regression on + face strata under preregistered thresholds; +- no worse oracle-negative rate after a standard downstream JPEG/WebP/resize + suite; +- byte-identical pass-through for detector-negative inputs by default; +- a significant paired advantage over random and norm-matched sham edits; +- no claim of forensic cleanliness without a separately trained removal-artifact + detector and a held-out evaluation. + +If pixel-only methods fail this gate, retain diffusion regeneration as the +explicit fallback. Do not combine its success with pixel-only results. + +## Provider sequence + +### OpenAI first + +1. Resolve the oracle authorization gate. +2. Build the corpus schema and confound challenge. +3. Verify or reject the shared-carrier hypothesis on low-texture probes. +4. Train classical and learned detectors. +5. Freeze the detector and temporal test. +6. Run R1, then R2, then R3 only after each preceding gate passes. +7. Submit one preregistered final oracle batch. + +OpenAI work establishes the experimental machinery, not parameters for Google. + +### Google second + +Repeat the full sequence with Google-native positives, Google hard negatives, +and the Gemini oracle. Spend the small manual oracle budget on controls and +decisive boundary points, not uniform sweeps. Use local detector uncertainty to +choose a batch before submission, then freeze it. Test Gemini app and AI Studio +surfaces as separate strata because their export and metadata paths differ. + +## Implementation order + +The research harness remains outside the public API until the gates pass. + +1. **Implemented:** use the private corpus schema and auditor documented in + [`data/synthid/research-manifest.md`](../data/synthid/research-manifest.md) to + record provider, surface, model epoch, session, content stratum, parent hash, + transform lineage, separate C2PA and SynthID outcomes, oracle session, and + artifact hashes. +2. **Implemented:** build a label-free local inventory before promotion so byte-identical files, + decoded-pixel duplicates, and unsupported formats are visible without + inferring evidence from directory names. +3. **Implemented:** add a corpus auditor that rejects hash-group leakage, missing parent links, + ambiguous labels, and unsupported oracle-provider pairs. +4. **Harness implemented; evidence run pending:** run the manifest-driven D1 + challenge over container, thumbnail, and canonical decoded-content + baselines. Freeze its validation threshold and report same-provider negative + cohorts separately. +5. Generalize `scripts/synthid_pixel_probe.py` into reusable feature extraction + while preserving its current synthetic tests. +6. Add reproducible train/evaluate commands whose output is a versioned model + card and metrics snapshot, never an unversioned console claim. +7. Add analytical and optimization removal harnesses with norm-matched controls. +8. Reuse the existing fidelity scripts for PSNR, SSIM, OCR, face, and edge + measurements, adding only missing metrics. +9. Package provider-specific detector weights behind an optional dependency only + after the detector gate passes. +10. Add a runtime remover and CLI surface only after the removal gate passes. + +Pure feature, manifest, split, threshold, and residual-constraint logic must be +unit-tested without model downloads. Real model and oracle runs stay explicit +research jobs. + +The inventory, manifest auditor, and D1 confound harness now exist as local +research tools. D1 has not produced a real-corpus metric yet because existing +artifacts have not been promoted into evidence-bearing provider manifests. This +is an evidence gap, not permission to infer labels from their paths. The next +real D1 run begins only after ordinary rows cover train, validation, locked test, +same-provider hard negatives, and a two-class temporal holdout. + +## Empirical log + +### 2026-08-09: fixed spectral-template baseline rejected + +An exploratory Google template was reconstructed from four public, purported +clean/marked pairs. Their provenance and oracle status could not be established, +so they were used for discovery only. A scalar phase-consensus score and its +threshold were selected on those pairs, four older Google-oracle positives, and +30 external negatives. + +The frozen threshold then produced six false positives on a new 100-image +external holdout. It also detected only one of four newly generated Gemini +images collected after threshold selection. Duplicate-image removal cannot +reduce the false-positive rate enough to approach the 0.1% detector gate, and +the temporal result is far below the required sensitivity. The baseline is +therefore rejected, not recalibrated on the holdout. + +This result rules out the fixed phase template as a detector or removal loss. +It does not rule out a content-adaptive or model-epoch-specific signal. The +next detector must learn from independently labeled provider data, must retain +the failed holdout unchanged, and must demonstrate discrimination from export +format and generator identity. + +### 2026-08-09: cross-color low-texture consensus rejected + +A polarity-invariant consensus template was trained on black, white, and red +Gemini low-texture probes and frozen before evaluation. Its median score fell +from 0.574 on the training groups to 0.354 on held-out probes of the same +colors and 0.011 on unseen blue, green, and gray probes. External negatives had +a median score of 0.0047. The separation therefore depended on the training +colors and did not generalize across the intended low-texture stratum. + +This branch is rejected as a current Google detector. It remains useful as a +negative control demonstrating why high training coherence is not evidence of +a shared carrier. + +### 2026-08-09: external V3 codebook detector pilot + +The numeric format-v2 V3 artifact from `reverse-SynthID` was loaded with pickle +disabled and evaluated independently; no third-party code was imported or +executed. A 256-bin phase score with a discovery-frozen threshold of 0.5 +detected five of five Google-oracle-positive images and produced zero false +positives on 194 external images collected before and after threshold freeze. +The pilot includes four older positives and one newly generated temporal +positive. It is promising discovery evidence, not a released detector: + +- zero errors on 194 negatives cannot support a 0.1% FPR claim; +- the set lacks enough same-provider, oracle-negative hard controls; +- the phase profile is a third-party artifact whose positive provenance is not + independently reproducible from our current corpus; +- a secondary `active_weight_fraction` shortcut failed on an exact-resolution + holdout, firing on 17 of 50 new negatives, and is rejected as a + native-resolution/content confound. + +The next detector milestone is to reproduce the phase carrier from our own +deduplicated positives, expand the negative set to at least the rule-of-three +minimum, and add same-provider hard negatives before fitting or publishing an +operating point. + +### 2026-08-09: independently fitted phase-carrier pilot + +An exact-geometry phase model was then fitted from four of our own +Google-oracle-positive images. The first unconstrained search was a clear +multiple-comparisons failure: training scores were 0.983-0.991, while the +newly confirmed fern positive scored 0.061 and an exact-resolution negative +reached 0.105. Leave-one-out discovery detected only two of four older +positives. That unrestricted model is rejected. + +A second model used only the 256 frequency coordinates proposed by the numeric +V3 artifact, while relearning phase, weight, and expected magnitude entirely +from our four positives. This restricted model scored the fern validation +positive at 0.740 and the four fitting positives at 0.936-0.963. A threshold of +0.43 was frozen after 50 validation negatives whose maximum was 0.305. + +The phase-only epoch failed its first new holdout with three false positives in +121 exact-size images. All three had active carrier weight between 0.07 and +0.14, so the failed epoch was closed and a new rule was frozen: phase score at +least 0.43 and active weight at least 0.50. That second epoch produced zero +false positives on 279 newly downloaded exact-size images. + +A separate 3,000-image COCO check also produced zero joint-rule positives, but +it is not counted toward the supported-stratum FPR bound. Every COCO image had +been upscaled to the model geometry and every active weight was at most 0.201. +The rule therefore abstained because the inputs were outside its measured +carrier-support distribution. Treat active weight as an `abstain` gate, not as +negative SynthID evidence. + +This is the first positive local-detector pilot derived from our labels, but it +does not clear the detector gate. It still depends on third-party candidate +coordinates, has only five confirmed positives including the fitting images, +has only 279 fresh negatives inside the current support stratum, and has no +same-provider hard negatives. The next epoch requires a larger independently +labeled positive set, at least 3,000 native-support negatives, same-provider +oracle negatives, and a new temporal positive that has not influenced feature +or threshold selection. + +### 2026-08-09: color-space spectral comparison + +The restricted phase-carrier experiment was repeated in RGB, full-range +YCbCr, YCoCg, an orthogonal opponent basis, CIE Lab, and HSV. Every branch +started from the same 102 spatial frequencies: the top 256 external codebook +coordinates contained 102 unique `(row, column)` pairs, which were expanded +over all three components. Each branch then independently selected 256 of the +306 component-frequency candidates and relearned phase, expected magnitude, +and weight from the same four oracle-positive fitting images. The score was +the phase score multiplied by active carrier weight, so weak spectral support +reduced rather than merely qualified the evidence. + +Thresholds were frozen from the fern validation positive and 50 exact-size +validation negatives before scoring the 279-image comparison set. The +comparison set was locked for this color-space branch, although it had already +served as the second epoch's RGB negative set and is therefore not a globally +virgin corpus. + +| Space | Validation gap | Comparison negative max | Fern minus negative max | False positives | +|---|---:|---:|---:|---:| +| RGB | 0.509 | 0.097 | 0.459 | 0/279 | +| YCbCr | 0.330 | 0.193 | 0.312 | 0/279 | +| YCoCg | 0.437 | 0.163 | 0.390 | 0/279 | +| Opponent | 0.351 | 0.167 | 0.339 | 0/279 | +| Lab | 0.408 | 0.126 | 0.382 | 0/279 | +| HSV | 0.530 | 0.084 | 0.495 | 0/279 | + +HSV had the best observed worst-negative margin, narrowly ahead of RGB, but +did not generally shift paired negatives below RGB. After normalizing each +space by its fern score, the median paired HSV-minus-RGB difference was 0.0015 +and a two-sided sign test gave `p=0.632` (144 higher, 135 lower). The result is +therefore a tail observation, not evidence that HSV dominates RGB. + +Channel decomposition localized the useful effect. HSV hue contributed only +0.012 of fern evidence and its fern-minus-holdout-maximum channel gap was +-0.024; saturation and value contributed 0.333 and 0.233, with positive gaps +of 0.273 and 0.194. In YCbCr, YCoCg, opponent, and Lab, the luminance-like +channel supplied the largest and most stable separation, while chroma-channel +gaps were much smaller. This supports a carrier expressed across intensity and +contrast structure, with useful nonlinear saturation/value projections, not a +hue-specific watermark. + +This experiment does not raise the detector's evidence grade. It still has +only one held-out confirmed positive, reuses the same third-party spatial +frequency prior, lacks same-provider hard negatives, and evaluates only one +native geometry. A future HSV S+V ablation must freeze its own operating point +and use a new negative epoch and new oracle-positive images; dropping hue now +would be a post-holdout model choice. + +### 2026-08-09: pixel-only R1 pilot did not transfer + +Three non-generative candidate families were frozen locally before Gemini +verification: + +1. local visible-region inpainting changed 0.087% of pixels at 47.64 dB PSNR + and 0.99963 SSIM; +2. sparse complex-spectrum subtraction used 256, 4,096, or 16,384 numeric V3 + bins while preserving native 2816x1536 geometry; +3. deterministic spatial fragmentation combined smooth local warps, one global + resampling operation, mild color changes, and codec round-trips. + +Local inpainting left the 256-bin phase score effectively unchanged. Healthy +Gemini sessions continued to detect both the target and its spatially shifted +sham, refuting an earlier one-off `not detected` response. + +The widest spectral candidate reduced the frozen 256-bin phase score from +0.639 to 0.124 while retaining 51.94 dB PSNR and 0.99854 SSIM. A +pixel-identical, metadata-normalized source control was detected in two healthy +oracle sessions. The 16,384-bin candidate remained detected in one of them and +the second hit quota before returning a verdict. The 4,096-bin candidates also +remained detected. Therefore the external phase carrier is correlated with the +Google signal but is not a sufficient removal loss. + +One additional profile returned `not detected` for the 16,384-bin candidate, +then also returned `not detected` for the untouched source. That entire session +is invalidated rather than counted as removal success. This failure adds a new +oracle-health rule: every account/session batch starts with the same +pixel-identical positive control; candidate verdicts are discarded when that +control is not positive. Quota and tool-unavailable responses remain +`indeterminate`. + +The frozen fragmentation batch is not yet oracle-resolved because healthy +profiles exhausted their rolling quota. Its balanced and strong variants also +fall below the release fidelity gate locally, so even a future negative oracle +result would be a mechanism-discovery result, not a releasable remover. + +A follow-up bounded-warp batch replaced RMS-normalized displacement with strict +per-axis maxima of 1.0, 1.8, and 2.8 pixels. The 1.0-pixel variant retained +37.79 dB PSNR and 0.9784 SSIM but left the local phase score essentially +unchanged at 0.643. The 2.8-pixel variant reduced the score only to 0.503 while +falling to 28.21 dB and 0.8570 SSIM. A multi-transform bounded variant reached +0.159 locally only at 21.97 dB and 0.6749 SSIM. Spatial fragmentation therefore +does not approach the release fidelity gate in this implementation and remains +a preregistered mechanism probe for the next healthy oracle window, not a +product candidate. + +### 2026-08-09: positive-only RGB plus S/V detector epoch + +The color-space result was converted into a deliberately one-sided research +detector. It requires exact 2816x1536 geometry, RGB evidence of at least 0.302, +HSV saturation-plus-value evidence of at least 0.302, and active carrier +support of at least 0.50 in both branches. It emits `positive` only when every +condition passes and emits `abstain` otherwise; it never claims that SynthID is +absent. + +Leave-one-positive-out fitting passed all four older Google positives. Their +minimum held-out RGB evidence was 0.527, minimum held-out S+V evidence was +0.452, and minimum active support was 0.600. The final frozen models detected +all five available oracle-positive controls. A new Picsum epoch downloaded +IDs 600-999; after exact-geometry validation and content-hash deduplication +against all earlier epochs, 330 images remained. None produced a positive +verdict. However, 328 of 330 abstained for insufficient carrier support and +only two reached both support gates before falling below the evidence +thresholds. This validates a positive-only signal catcher inside one geometry, +not a two-sided detector or a general FPR claim. + +### 2026-08-09: ensemble phase projection is not a sufficient removal loss + +An analytical pixel-only attack removed the positive complex projection onto +the learned RGB phases and HSV S/V phases. One iteration at strength 0.5 +cleared the local ensemble at 56.03 dB PSNR, 0.99909 SSIM, and a maximum pixel +change of two levels. A norm-matched random-noise control remained locally +positive. + +The provider result rejected this loss as sufficient. In a Gemini session +where the untouched fern control returned `SynthID Status: Detected`, all +three preregistered projection candidates at strengths 0.5, 0.75, and 1.0 also +returned detected. The strongest candidate had negative local RGB and S+V +evidence at 51.97 dB PSNR. The ensemble therefore tracks a real correlate but +does not cover all evidence used by Google's decoder. + +### 2026-08-09: 16x32 periodic-tile carrier hypothesis + +The failed global projection exposed a more specific spatial structure. Of +the top 256 external V3 bins at 1536x2816, 208 (81.25%) lie exactly on an FFT +lattice spaced by 96 rows and 88 columns. This corresponds to a spatial cell +of `1536 / 96 = 16` rows by `2816 / 88 = 32` columns. Instead of subtracting a +small list of harmonics, a new pixel-only probe high-pass filters the image, +folds the residual modulo 16x32 over 8,448 repeats, and subtracts the complete +zero-mean folded template. + +On the fern positive with its visible sparkle already removed, the mildest +locally clearing candidate used Gaussian sigma 0.6 and strength 1.5. The +phase-aligned template cleared the ensemble at 57.03 dB PSNR and 0.99911 SSIM; +the same template shifted by one row and one column remained positive at +57.02 dB and 0.99912 SSIM. Two additional aligned/shifted pairs reproduced the +direction at 55.96 and 54.19 dB. + +The per-image estimator was then run on the four older positives with a frozen +strength ladder. The minimum locally clearing strength was 2.0 for three +images and 1.5 for one. All four phase-aligned outputs abstained, while all +four same-strength one-pixel-shifted controls remained positive. Selected +outputs retained at least 55.67 dB PSNR and 0.99874 SSIM. This is the strongest +local causal evidence so far because effect direction follows exact tile +phase at matched distortion across five contents. + +It is not a removal result. In a healthy Google verifier session, the fern +source control was detected and the mild aligned sigma-0.6, strength-1.5 +candidate was also detected. The stronger local variants therefore remain +unresolved rather than inheriting a negative label from the local ensemble. +Cross-resolution tile discovery and same-provider hard negatives also remain +open. + +### 2026-08-09: provider-specific phase losses failed both remote verifiers + +The Google result above and a separate OpenAI experiment closed the first +analytical-loss epoch. The OpenAI branch fitted an 8-pixel phase pattern from +OpenAI low-texture outputs and produced a 46.70 dB candidate whose local phase +score changed sign. A single frozen submission to OpenAI Verify still returned +`SynthID detected`; C2PA was absent because the candidate had been re-encoded. +This confirms that metadata removal and a local phase-score reversal do not +establish pixel-watermark removal. + +The experiments are not cross-provider replications. Google images were sent +only to Google Verify AI and the OpenAI candidate only to OpenAI Verify. No +carrier, threshold, response, or verdict was transferred between tracks. +OpenAI's published verifier guidance prohibits repeated queries for watermark +reverse engineering or evasion, so that one failed query did not start an +adaptive loop. Further OpenAI candidates remain local hypotheses unless an +authorized research oracle becomes available. + +### 2026-08-09: JPEG-robust periodic EOT surrogate pilot + +Small residual CNNs were trained separately for each provider as differentiable +attack surfaces. They are provider classifiers, not SynthID detectors: their +corpora lack current same-provider oracle-negative controls, and several +external negatives approach the positive range. Two same-architecture models +were used for optimization and a third model with a different first kernel and +channel schedule was kept outside the attack. + +For Google, all three models scored the temporal fern positive at a 75th-patch +percentile between 0.938 and 0.962. A patch-local L-infinity-8 attack initially +cleared the models and survived JPEG, but visual inspection exposed rectangular +grid boundaries; that candidate was rejected despite favorable scalar metrics. +A replacement optimized one 16x32 residual cell repeated over the full image, +with differentiable JPEG-95 and JPEG-90 branches. At half strength it retained +36.80 dB PSNR and 0.9241 SSIM against the icon-clean source. The three PNG +scores fell to 0.024, 0.047, and 0.016; after an actual JPEG-90 round trip they +were 0.182, 0.243, and 0.016. The independent RGB and HSV S+V phase ensemble +also abstained, and remained below threshold after JPEG-90, JPEG-95, WebP-95, +and a 0.98 resize round trip. + +For OpenAI, the held-out complex market image scored 0.936, 0.929, and 0.813 +on its three provider classifiers. An independently optimized 8x8 periodic EOT +residual was then attenuated in low-texture regions. The selected local +candidate retained 34.97 dB PSNR and 0.9356 SSIM. Its PNG scores were 0.030, +0.053, and 0.099; after actual JPEG-90 they were 0.042, 0.063, and 0.178. +WebP-95 behaved similarly, while a 0.98 resize raised the independent model to +0.692 and remains a measured weakness. A second optimization that included a +resize branch improved the two optimized models but transferred worse to the +independent model, so it was rejected. + +The periodic residuals were then tested for cross-image transfer without +further optimization. The OpenAI 8x8 cell learned from the market image was +applied with the same low-texture attenuation to 11 other OpenAI positives. +Every image moved downward on all three models, both as PNG and after an actual +JPEG-90 round trip. Median PNG scores changed from 0.971, 0.970, and 0.931 to +0.016, 0.033, and 0.036. Each model had 11 of 11 paired decreases, giving a +two-sided exact sign-test `p=0.00098`; per-image PSNR ranged from 33.50 to +35.99 dB. Two complex scenes retained materially higher scores on the +independent model, so the transfer is strong but not uniform. + +The Google 16x32 EOT cell was likewise applied to the four older images after +their own locally clearing phase-tile subtraction. All four remained phase +ensemble abstentions, and every CNN score decreased in the lossless outputs. +JPEG-90 preserved the direction but restored the two optimized models to +roughly 0.58-0.78 on the hardest images. Fidelity was approximately 37.25 dB +PSNR. With only four paired images this is descriptive transfer evidence, not a +statistical or oracle gate. + +These are positive local robustness results, not watermark-removal results. +Neither candidate meets the release fidelity gate, neither CNN ensemble has +cleared the detector-identifiability gate, and neither selected EOT candidate +has a matching-provider negative oracle verdict. A three-file Google batch is +hash-frozen outside the repository for the next healthy quota window: the +positive source, the stronger phase-only ablation, and the combined periodic +EOT candidate. Results will be accepted only in that fixed order without +between-query adaptation. The first later attempt detected the positive source, +then returned a usage-limit response for the phase-only ablation; that response +is `indeterminate`, and the combined candidate was not submitted out of order. +The OpenAI candidate is not scheduled for the public verifier under the current +usage restriction. + +### 2026-08-09: OpenAI fidelity and architecture-transfer frontier + +Further local experiments tested whether the OpenAI periodic result could move +above the release fidelity gate without losing transport robustness. Chroma-only +projection reached 47.43 dB PSNR, but an actual JPEG round trip removed the +perturbation and restored the two classifier scores to approximately 0.9. Mixed +luminance/chroma projections and stronger texture masks did not improve that +frontier. A regularized Adam attack reached 41.21 dB on the source image, but +JPEG-90 restored the optimized classifiers to 0.90 and 0.91, and the independent +architecture scored 0.89. This rejected post-hoc color projection and simple +regularization as fidelity remedies. + +A second experiment put actual OpenCV JPEG and WebP results in the forward pass +and used a straight-through gradient. At 40.08 dB, actual JPEG-90 still scored +0.89 and 0.91 on the optimized models. Removing the luminance penalty improved +the first model to 0.76 but left the second at 0.88 and reduced fidelity to +38.78 dB. The exact-codec forward pass therefore corrected an invalid transport +approximation, but did not produce a new Pareto candidate. + +An architecture-diverse fourth classifier used fine RGB residuals, broader RGB +residuals, and luminance/opponent-color residuals. The market image remained +outside its training split. Its 75th-patch percentile ranged from 0.983 to 0.988 +on four held-out OpenAI positives; the median across 120 held-out COCO negatives +was 0.041 and the maximum was 0.521. This is useful attack-surface diversity, not +a certified detector, because no current same-provider oracle-negative corpus is +available. + +Adding that model to periodic optimization did not improve architecture transfer. +The preregistered local winner retained 34.86 dB and scored 0.057, 0.197, and +0.021 after JPEG-90 on the three optimized models, but the independent model +scored 0.573. A per-image half-resolution perturbation produced a higher-fidelity +41.86 dB candidate; JPEG-90 scores on its optimized models were 0.587, 0.572, +and 0.471, while the independent model remained at 0.875. Optimizing only the +two original models produced the same failure on both independent architectures. + +The measured local frontier is therefore unchanged. The 34.97 dB periodic +candidate remains the strongest transport-robust and cross-architecture result; +the candidates above 40 dB are architecture-specific and cannot support a +removal claim. No additional public OpenAI verifier requests were made during +these experiments. + +## Decision record + +The program has four possible honest outcomes per provider: + +| Outcome | Product consequence | +| --- | --- | +| Causal signal and detector both generalize | Continue to pixel-only removal | +| Detector works but causal attribution fails | Ship no SynthID detector claim; retain as provenance research | +| Detector generalizes but pixel-only removal does not transfer | Keep local detection, retain regeneration fallback | +| Pixel-only removal clears the oracle with quality gates | Productize provider-specific detector and remover | + +Stopping at a failed gate is a result. It prevents a local surrogate, export +fingerprint, or quality metric from being mistaken for control over SynthID. + +## Immediate first milestone + +The first milestone produces no shipping code. Harness code already exists for +items 1 and 5; milestone delivery means a completed evidence-bearing artifact, +not merely an available script. It delivers: + +1. the private-corpus schema and auditor; +2. an OpenAI authorization decision for use of the remote provenance verifier; +3. an independently verified status for candidate causal pairs; +4. a canonicalized OpenAI pilot set with hard negatives; +5. the D1 confound report; +6. the D2 low-texture carrier report with leave-one-group-out results; +7. a go or no-go decision for real-image detector training. + +Only after this milestone should implementation of D3 and D4 begin. + +## Primary sources + +- OpenAI, [Content provenance](https://developers.openai.com/api/docs/guides/content-provenance). +- Google, [Verify AI-generated images, videos, and audio](https://support.google.com/gemini/answer/16722517?hl=en). +- Gowal et al., [SynthID-Image: Image watermarking at internet scale](https://arxiv.org/abs/2510.09263). diff --git a/docs/synthid.md b/docs/synthid.md index 35c9caf..8fd023e 100644 --- a/docs/synthid.md +++ b/docs/synthid.md @@ -176,6 +176,62 @@ conversion (the body text of Section 10 is absent from the HTML). **What is known empirically from our own oracle-verified testing.** +A 2026-08-09 non-generative pilot found a promising Google phase-correlate, +but did not establish a releasable local detector or pixel-only remover. The +best independently fitted model relearned phase and magnitude from four of our +positives while using third-party candidate coordinates; its second frozen +epoch had zero false positives on 279 new exact-size external images and +detected the one new confirmed positive used for validation. An additional +3,000 upscaled photographs all fell outside the measured active-carrier support +and therefore count as abstentions, not negatives. The corpus is still far too +small and lacks same-provider hard negatives needed for a 0.1% FPR claim. +Repeating the spectral fit in six color spaces produced zero false positives +on the same 279-image comparison set for every branch. HSV had the best +observed worst-negative margin, but paired normalized negative scores did not +show a general improvement over RGB. Its apparent benefit came from saturation +and value; hue failed its channel-level separation check. Luminance-like +channels dominated YCbCr, YCoCg, opponent, and Lab. This narrows the next +hypothesis to intensity/contrast and HSV S+V projections rather than a +hue-specific carrier, but it does not add independent positives or certify an +operating point. +The resulting positive-only RGB plus S/V ensemble passed four +leave-one-positive-out checks, detected all five available positive controls, +and emitted no positive verdict on 330 newly collected exact-size images. +Almost all external images lacked sufficient measured carrier support and +therefore remained abstentions rather than proven negatives. + +Directly projecting out the ensemble phases cleared the local detector above +51 dB PSNR, but three frozen candidates remained detected by Gemini in a +healthy control session. A subsequent spatial analysis found that 81.25% of +the top 256 carrier bins lie on a lattice corresponding to a repeating 16x32 +pixel cell. Modulo-folding and subtracting the complete high-pass tile gave +phase-specific matched-control separation across all five Google positives at +at least 55.67 dB PSNR: aligned outputs cleared the local ensemble, while +one-pixel-shifted controls remained positive. The mild locally clearing tile +candidate nevertheless remained detected in a healthy Google Verify AI +session, so the tile correlate is not a sufficient removal loss. +Sparse subtraction reduced the local score sharply at more than 50 dB PSNR, +yet healthy Gemini sessions still detected SynthID. A one-off negative was +discarded because the same session also missed the source-positive control. +An OpenAI-specific 8-pixel phase candidate likewise changed its local score at +46.70 dB PSNR but remained `SynthID detected` in one frozen OpenAI Verify +check. No Google threshold or carrier was used in that experiment, and no +further public OpenAI checks were made because the documented verifier guidance +rules out repeated watermark-removal queries. + +A later local EOT pilot combined periodic full-frame residuals with JPEG-aware +optimization against provider-specific CNN surrogates. It produced candidates +that stayed below three local models after actual JPEG round trips, but the +models remain provider classifiers without same-provider oracle-negative +controls. The selected Google candidate retained 36.80 dB PSNR and 0.9241 +SSIM; the selected OpenAI candidate retained 34.97 dB and 0.9356. Both miss the +research fidelity gate. The OpenAI periodic residual also reduced all three +local model scores on 11 of 11 additional images before and after JPEG-90, but +neither provider candidate has a negative matching-provider oracle verdict, so +neither is an established remover. +The protocol, exact limitations, and next experiments are recorded in the +[`detector and removal research plan`](synthid-detector-removal-plan.md). + A controlled study (June 2026, clean v0.8.6 with text/face protection OFF, native resolution on this repo's default SDXL pipeline) measured the minimum img2img strength that removes the SynthID pixel watermark, verified per image on @@ -186,14 +242,15 @@ Generated cleaned outputs are not committed; the table below is the durable record of the historical oracle verdicts. One third-party image from issue #14 was oracle-verified but is not committed. -**Oracle validation order: start with OpenAI.** When validating removal across -vendors, run the OpenAI arm first. `openai.com/verify` is more accessible than the -Gemini app -- fewer per-check restrictions, so it gives the fastest signal and is -the strongest candidate for automation (Playwright / Chrome MCP driving -`openai.com/verify`); the Gemini "Verify with SynthID" flow is more manual. This is -an ordering/throughput choice, not a substitution: each oracle only reads its own -vendor's SynthID (`openai.com/verify` is OpenAI-scoped), so Google content still -needs the Gemini app. +**Historical oracle validation order: start with OpenAI.** The June study used +the OpenAI web verifier first because it was more accessible than the Gemini app. +OpenAI now documents a Content Provenance API, but its usage guidance explicitly +rules out repeated queries for reverse engineering or watermark removal. New +adaptive experiments require separate authorization and must follow the oracle +boundary in the +[`detector and removal research plan`](synthid-detector-removal-plan.md). This is +not a cross-provider substitution: each oracle reads only its own vendor's +SynthID, so Google content still needs the Gemini flow. | Vendor | Images | Resolution(s) | Pipeline | Removed at | |--------|--------|---------------|----------|------------| @@ -269,7 +326,7 @@ diffusion prior." ## 3. Detectability and verifier access -### 3.1 No public local detector +### 3.1 No public local decoder The SynthID decoder is proprietary and not released: @@ -278,16 +335,24 @@ The SynthID decoder is proprietary and not released: > available to trusted testers." > -- Gowal et al., arXiv:2510.09263 -There is no public API, no released decoder weights, and no reproducible -algorithm for local detection. The verification service (SynthID Detector) is: +There are no released decoder weights and no reproducible algorithm for local +detection. Google provides verification in Gemini and a limited SynthID Detector +portal. OpenAI now documents a synchronous Content Provenance API whose image +response contains separate C2PA and SynthID outcomes. That API is a remote, +OpenAI-scoped verifier, not a local decoder. Its documentation also says not to +use repeated queries to reverse-engineer, remove, or evade a watermark, so an +adaptive research loop requires separate authorization. + +Google's SynthID Detector service is: > "a verification portal" in early testing with "journalists and media > professionals" on a waitlist > -- deepmind.google/models/synthid/ The external variant SynthID-O is available "through partnerships" only. Our -tool cannot locally detect SynthID presence or absence -- this is by design, -not a gap we can fill. +tool does not currently detect SynthID pixels locally. The gated research path +for determining whether that can change is documented in +[`synthid-detector-removal-plan.md`](synthid-detector-removal-plan.md). ### 3.2 How our tool recognizes SynthID from provenance @@ -312,20 +377,18 @@ This is why: ### 3.3 Oracle scope: each vendor detects only their own -From openai.com/research/verify (verbatim, verified 2026-05-31): - -> "OpenAI generation signals will only be detected if the image was generated -> with our tools." -> "Content could also still be AI-generated by another company's model, which -> the tool currently does not detect." +OpenAI's current Content Provenance API documentation says it checks supported +OpenAI signals and is not a general-purpose AI detector. Google's current Gemini +documentation likewise says Gemini recognizes SynthID from Google AI tools, +not other companies' payloads. SynthID technology is used by multiple vendors, but each verifier is keyed to its own payload: -| Oracle | Detects | Does NOT detect | -|-------------------------------|------------------|-------------------------| -| Gemini app "Verify with SynthID" | Google SynthID | OpenAI SynthID | -| openai.com/research/verify | OpenAI SynthID | Google SynthID | +| Oracle | Detects | Does not detect | +| --- | --- | --- | +| Gemini app verification | Google SynthID | OpenAI SynthID | +| OpenAI Content Provenance API or web verifier | Supported OpenAI SynthID | Google SynthID | A Google-SynthID image reads clean on openai.com/verify. An OpenAI image reads clean in the Gemini oracle. They are different payloads within the same @@ -620,8 +683,10 @@ Two constraints on reading this: subjects.** JPEG re-encoding removes C2PA metadata but does NOT remove the SynthID pixel watermark (verified June 2026 on issue #14 pic3). Do not dismiss these as "not faithful originals" for SynthID-removal tests. -- **The correct oracle for OpenAI images is openai.com/verify**, not the Gemini - app. The two oracles detect different payloads. +- **The correct oracle for OpenAI images is an authorized OpenAI provenance + verifier**, not the Gemini app. OpenAI now documents both a web tool and a + Content Provenance API; the API's published use restrictions still apply. + The OpenAI and Google oracles detect different payloads. - **A quiet `identify` output after processing is not proof of removal.** It means the provenance evidence is gone. The pixel watermark state is unknown without an oracle check. @@ -756,8 +821,8 @@ reproducible verification requires a fixed seed. Forensic Stealth in Generative-AI Watermark Removal.** arXiv:2605.09203. https://arxiv.org/abs/2605.09203 -5. OpenAI. **Verify tool for AI-generated images.** openai.com/research/verify. - Accessed 2026-05-31. +5. OpenAI. **Content provenance.** + https://developers.openai.com/api/docs/guides/content-provenance 6. Google. **Verify AI-generated images, videos, and audio.** https://support.google.com/gemini/answer/16722517 diff --git a/docs/verification-plan.md b/docs/verification-plan.md index 99e19b5..7e069ed 100644 --- a/docs/verification-plan.md +++ b/docs/verification-plan.md @@ -175,12 +175,19 @@ Three harness rules are load-bearing and must not be relaxed: score a mark only crop's adjudication scope; take provenance from metadata, never from labels; and never report recall from the detector-sampled set. -## Tier D -- external oracles (manual, not automatable here) +## Tier D -- external oracles -SynthID removal cannot be verified locally by design -- no public decoder exists. Each -vendor has its own oracle and it covers only that vendor's content: `openai.com/verify` for -OpenAI (more accessible, the automation candidate), the Gemini app for Google (manual, -rate-limited). A quiet metadata proxy is **not** proof the pixel watermark is gone. +SynthID removal cannot currently be verified locally because no public decoder weights +exist. Each vendor has its own oracle and it covers only that vendor's content. OpenAI +documents a synchronous Content Provenance API with a distinct SynthID result, while the +Gemini app provides a manual, quota-limited Google verifier. A quiet metadata proxy is +**not** proof the pixel watermark is gone. + +OpenAI's API documentation says not to use repeated queries to reverse-engineer, remove, +or evade a watermark. Using it as an adaptive research oracle therefore requires explicit +authorization. Without that authorization it must not become a training loss, search loop, +or automated removal gate. The provider-specific detector and pixel-only removal research +protocol is in [`synthid-detector-removal-plan.md`](synthid-detector-removal-plan.md). Scope honestly: this tier certifies strength floors on a handful of images per vendor, and that is all it can do. See `docs/synthid.md`. @@ -279,7 +286,7 @@ on one file. The bar is never "handles it" but **never raises and never silently 5. **A5 contract sweep over a representative local set**. 6. **B4 resource ceilings**, **E robustness**. 7. **C recall expansion** -- gated by labelling appetite. -8. **D oracles** -- manual, per release. +8. **D oracles** -- authorized and provider-specific, per release. Every tier writes a versioned snapshot so runs are comparable over time; a run that cannot be diffed against the last one is a one-off, not a regression suite. diff --git a/scripts/synthid_color_space_probe.py b/scripts/synthid_color_space_probe.py new file mode 100644 index 0000000..baed365 --- /dev/null +++ b/scripts/synthid_color_space_probe.py @@ -0,0 +1,352 @@ +"""Compare an exact-geometry phase carrier across color spaces. + +Every model uses the same spatial-frequency candidates. Phases, expected +magnitudes, selected channels, and weights are learned independently from the +supplied positive images. The resulting scores are experimental evidence, not +a proprietary SynthID decoder or a certified production detector. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import cv2 +import numpy as np +from PIL import Image +from synthid_phase_carrier import _leave_one_out_coherence +from synthid_v3_codebook_probe import load_v3_model + +log = logging.getLogger(__name__) + +COLOR_SPACES = ("rgb", "ycbcr", "ycocg", "opponent", "lab", "hsv") +CHANNEL_NAMES = { + "rgb": ("R", "G", "B"), + "ycbcr": ("Y", "Cb", "Cr"), + "ycocg": ("Y", "Co", "Cg"), + "opponent": ("L", "R-G", "R+G-2B"), + "lab": ("L*", "a*", "b*"), + "hsv": ("H", "S", "V"), +} + + +@dataclass(frozen=True) +class ColorPhaseModel: + """Sparse phase carrier learned in one color space.""" + + color_space: str + height: int + width: int + rows: np.ndarray + columns: np.ndarray + channels: np.ndarray + phases: np.ndarray + weights: np.ndarray + expected_magnitudes: np.ndarray + + +@dataclass(frozen=True) +class ColorPhaseScore: + """Phase-carrier evidence and channel contributions for one image.""" + + path: str + color_space: str + phase_score: float + active_weight_fraction: float + evidence_score: float + channel_evidence: tuple[float, float, float] + selected_peak_counts: tuple[int, int, int] + peak_count: int + + +def transform_color_space(rgb: np.ndarray, color_space: str) -> np.ndarray: + """Transform float RGB values in [0, 255] into COLOR_SPACE.""" + if rgb.ndim != 3 or rgb.shape[2] != 3: + raise ValueError("rgb must have shape (height, width, 3)") + pixels = np.asarray(rgb, dtype=np.float64) + red, green, blue = np.moveaxis(pixels, 2, 0) + if color_space == "rgb": + transformed = pixels + elif color_space == "ycbcr": + transformed = np.stack( + ( + 0.299 * red + 0.587 * green + 0.114 * blue, + 128.0 - 0.168736 * red - 0.331264 * green + 0.5 * blue, + 128.0 + 0.5 * red - 0.418688 * green - 0.081312 * blue, + ), + axis=2, + ) + elif color_space == "ycocg": + transformed = np.stack( + ( + 0.25 * red + 0.5 * green + 0.25 * blue, + 0.5 * red - 0.5 * blue, + -0.25 * red + 0.5 * green - 0.25 * blue, + ), + axis=2, + ) + elif color_space == "opponent": + transformed = np.stack( + ( + (red + green + blue) / np.sqrt(3.0), + (red - green) / np.sqrt(2.0), + (red + green - 2.0 * blue) / np.sqrt(6.0), + ), + axis=2, + ) + elif color_space == "lab": + transformed = cv2.cvtColor((pixels / 255.0).astype(np.float32), cv2.COLOR_RGB2LAB).astype(np.float64) + elif color_space == "hsv": + transformed = cv2.cvtColor((pixels / 255.0).astype(np.float32), cv2.COLOR_RGB2HSV).astype(np.float64) + else: + raise ValueError(f"unsupported color space: {color_space}") + if not np.all(np.isfinite(transformed)): + raise ValueError(f"{color_space} transform produced non-finite values") + return transformed + + +def candidate_bins_from_codebook( + codebook_path: Path, + *, + height: int, + width: int, + source_peak_count: int = 256, +) -> np.ndarray: + """Expand the codebook's unique spatial coordinates over three channels.""" + prior = load_v3_model( + codebook_path, + height=height, + width=width, + peak_count=source_peak_count, + ) + spatial = np.unique(np.column_stack((prior.rows, prior.columns)), axis=0) + return np.asarray( + [(int(row), int(column), channel) for row, column in spatial for channel in range(3)], + dtype=np.int32, + ) + + +def _load_rgb(path: Path, *, height: int, width: int) -> np.ndarray: + """Load an exact-geometry image as float64 RGB.""" + with Image.open(path) as image: + rgb = image.convert("RGB") + if rgb.size != (width, height): + raise ValueError(f"{path}: geometry {rgb.width}x{rgb.height} does not match {width}x{height}") + return np.asarray(rgb, dtype=np.float64) + + +def _extract_values(pixels: np.ndarray, bins: np.ndarray) -> np.ndarray: + """Extract complex rFFT values at sparse BINS from three-channel PIXELS.""" + values = np.empty(len(bins), dtype=np.complex128) + for channel in range(3): + positions = np.flatnonzero(bins[:, 2] == channel) + spectrum = np.fft.rfft2(pixels[:, :, channel]) + values[positions] = spectrum[bins[positions, 0], bins[positions, 1]] + return values + + +def discover_model( + paths: list[Path], + *, + color_space: str, + candidate_bins: np.ndarray, + peak_count: int = 256, +) -> ColorPhaseModel: + """Learn one color-space phase carrier from exact-geometry PATHS.""" + if len(paths) < 3: + raise ValueError("at least three positive images are required") + if color_space not in COLOR_SPACES: + raise ValueError(f"unsupported color space: {color_space}") + with Image.open(paths[0]) as first: + width, height = first.size + bins = np.asarray(candidate_bins, dtype=np.int32) + if bins.ndim != 2 or bins.shape[1] != 3: + raise ValueError("candidate_bins must have shape (count, 3)") + if len(bins) < peak_count: + raise ValueError(f"only {len(bins)} candidate bins for {peak_count} peaks") + if ( + np.any(bins[:, 0] < 0) + or np.any(bins[:, 0] >= height) + or np.any(bins[:, 1] <= 0) + or np.any(bins[:, 1] > width // 2) + or np.any(bins[:, 2] < 0) + or np.any(bins[:, 2] > 2) + ): + raise ValueError("candidate_bins contain out-of-range coordinates") + + image_values = np.empty((len(paths), len(bins)), dtype=np.complex128) + for index, path in enumerate(paths): + rgb = _load_rgb(path, height=height, width=width) + image_values[index] = _extract_values(transform_color_space(rgb, color_space), bins) + magnitudes = np.abs(image_values) + units = np.divide(image_values, magnitudes, out=np.zeros_like(image_values), where=magnitudes != 0.0) + unit_sum = np.sum(units, axis=0) + count = float(len(paths)) + minimum_loo = np.ones(len(bins), dtype=np.float64) + for unit in units: + minimum_loo = np.minimum(minimum_loo, _leave_one_out_coherence(unit_sum, unit, count)) + expected_magnitudes = np.mean(magnitudes, axis=0) + selection = np.power(minimum_loo, 4.0) * np.log1p(expected_magnitudes) + chosen = np.argpartition(selection, -peak_count)[-peak_count:] + chosen = chosen[np.argsort(selection[chosen])[::-1]] + raw_weights = selection[chosen] + if np.sum(raw_weights) <= 0.0: + raise ValueError("candidate bins have no usable phase consensus") + selected = bins[chosen] + return ColorPhaseModel( + color_space=color_space, + height=height, + width=width, + rows=selected[:, 0].astype(np.int32), + columns=selected[:, 1].astype(np.int32), + channels=selected[:, 2].astype(np.int8), + phases=np.angle(unit_sum[chosen] / count).astype(np.float64), + weights=(raw_weights / np.sum(raw_weights)).astype(np.float64), + expected_magnitudes=expected_magnitudes[chosen].astype(np.float64), + ) + + +def score_image(path: Path, model: ColorPhaseModel) -> ColorPhaseScore: + """Score PATH against MODEL and expose additive channel evidence.""" + rgb = _load_rgb(path, height=model.height, width=model.width) + bins = np.column_stack((model.rows, model.columns, model.channels)) + values = _extract_values(transform_color_space(rgb, model.color_space), bins) + magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0) + contributions = model.weights * magnitude_gate * np.cos(np.angle(values) - model.phases) + active_weights = model.weights * magnitude_gate + active_weight = float(np.sum(active_weights)) + evidence = float(np.sum(contributions)) + phase_score = 0.0 if active_weight == 0.0 else evidence / active_weight + channel_evidence = tuple(float(np.sum(contributions[model.channels == channel])) for channel in range(3)) + selected_peak_counts = tuple(int(np.sum(model.channels == channel)) for channel in range(3)) + return ColorPhaseScore( + path=str(path), + color_space=model.color_space, + phase_score=phase_score, + active_weight_fraction=active_weight, + evidence_score=evidence, + channel_evidence=channel_evidence, + selected_peak_counts=selected_peak_counts, + peak_count=len(model.rows), + ) + + +def save_model(path: Path, model: ColorPhaseModel) -> None: + """Save MODEL as a pickle-free numeric NPZ.""" + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + path, + format_version=np.asarray(1, dtype=np.int32), + color_space=np.asarray(model.color_space), + height=np.asarray(model.height, dtype=np.int32), + width=np.asarray(model.width, dtype=np.int32), + rows=model.rows.astype(np.int32), + columns=model.columns.astype(np.int32), + channels=model.channels.astype(np.int8), + phases=model.phases.astype(np.float32), + weights=model.weights.astype(np.float32), + expected_magnitudes=model.expected_magnitudes.astype(np.float64), + ) + + +def load_model(path: Path) -> ColorPhaseModel: + """Load and validate one color-space phase-carrier artifact.""" + with np.load(path, allow_pickle=False) as artifact: + if int(artifact["format_version"]) != 1: + raise ValueError("unsupported color-phase model format version") + model = ColorPhaseModel( + color_space=str(artifact["color_space"]), + height=int(artifact["height"]), + width=int(artifact["width"]), + rows=np.asarray(artifact["rows"], dtype=np.int32), + columns=np.asarray(artifact["columns"], dtype=np.int32), + channels=np.asarray(artifact["channels"], dtype=np.int8), + phases=np.asarray(artifact["phases"], dtype=np.float64), + weights=np.asarray(artifact["weights"], dtype=np.float64), + expected_magnitudes=np.asarray(artifact["expected_magnitudes"], dtype=np.float64), + ) + count = len(model.rows) + arrays = (model.columns, model.channels, model.phases, model.weights, model.expected_magnitudes) + if model.color_space not in COLOR_SPACES: + raise ValueError("invalid model color space") + if model.height < 64 or model.width < 64 or count == 0 or any(array.shape != (count,) for array in arrays): + raise ValueError("invalid color-phase model shapes") + if np.any(model.rows < 0) or np.any(model.rows >= model.height): + raise ValueError("invalid color-phase row indices") + if np.any(model.columns <= 0) or np.any(model.columns > model.width // 2): + raise ValueError("invalid color-phase column indices") + if np.any(model.channels < 0) or np.any(model.channels > 2): + raise ValueError("invalid color-phase channel indices") + if not np.isclose(np.sum(model.weights), 1.0, atol=1e-5) or np.any(model.weights < 0.0): + raise ValueError("invalid color-phase weights") + if np.any(model.expected_magnitudes < 0.0): + raise ValueError("invalid expected magnitudes") + return model + + +@click.group() +def main() -> None: + """Discover and score phase carriers in multiple color spaces.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + + +@main.command() +@click.argument("codebook", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("positives", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--color-space", type=click.Choice(COLOR_SPACES), required=True) +@click.option("--source-peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--model-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def discover( + codebook: Path, + positives: tuple[Path, ...], + color_space: str, + source_peak_count: int, + peak_count: int, + model_out: Path, +) -> None: + """Learn a color-space carrier from exact-geometry POSITIVES.""" + with Image.open(positives[0]) as first: + width, height = first.size + candidates = candidate_bins_from_codebook( + codebook, + height=height, + width=width, + source_peak_count=source_peak_count, + ) + model = discover_model( + list(positives), + color_space=color_space, + candidate_bins=candidates, + peak_count=peak_count, + ) + save_model(model_out, model) + log.info("Wrote %s color-phase model with %s candidates: %s", color_space, len(candidates), model_out) + + +@main.command() +@click.argument("model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def score(model_path: Path, images: tuple[Path, ...], report_out: Path) -> None: + """Score exact-geometry IMAGES with MODEL_PATH.""" + model = load_model(model_path) + payload = { + "model": str(model_path), + "color_space": model.color_space, + "channel_names": CHANNEL_NAMES[model.color_space], + "height": model.height, + "width": model.width, + "peak_count": len(model.rows), + "scores": [asdict(score_image(image, model)) for image in images], + } + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + log.info("Wrote %s color-phase score report: %s", model.color_space, report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_confound_probe.py b/scripts/synthid_confound_probe.py new file mode 100644 index 0000000..8701f15 --- /dev/null +++ b/scripts/synthid_confound_probe.py @@ -0,0 +1,478 @@ +"""Measure how well non-watermark confounds predict provider SynthID labels. + +The experiment deliberately trains three provider-specific baselines: + +``container`` + File size, decoded geometry, aspect ratio, and format. +``thumbnail`` + Container features plus a small RGB thumbnail that can learn generator and + content style. +``canonical`` + Decoded, orientation-normalized RGB at a fixed geometry, with no container, + original-resolution, metadata, filename, or path features. + +These are challenge baselines, not SynthID detectors. A candidate detector must +beat the canonical baseline on same-provider hard negatives and a temporal +holdout before its result can be attributed to a watermark-specific signal. + +Usage: + uv run --extra pixels python scripts/synthid_confound_probe.py \ + .local-eval/synthid/manifest.csv --target-provider google \ + --report-out .local-eval/synthid/google-d1-confounds.json +""" + +from __future__ import annotations + +import csv +import json +import logging +from collections import Counter +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import numpy as np +from PIL import Image, ImageOps +from synthid_research_manifest import artifact_sha256, audit_manifest, resolve_artifact_path + +log = logging.getLogger(__name__) + +FEATURE_FAMILIES = ("container", "thumbnail", "canonical") +FINAL_SPLITS = ("train", "validation", "test", "temporal") +THUMBNAIL_SIZE = 8 +FORMAT_NAMES = ("png", "jpeg", "webp") +LOGISTIC_ITERATIONS = 2_000 +LOGISTIC_LEARNING_RATE = 0.2 +LOGISTIC_L2 = 0.01 + + +@dataclass(frozen=True) +class Example: + """One ordinary, provider-targeted manifest row eligible for D1.""" + + artifact_path: Path + artifact_sha256: str + group_id: str + split: str + label: int + negative_cohort: str | None + + +@dataclass(frozen=True) +class LogisticModel: + """Standardization and regularized logistic-regression parameters.""" + + mean: np.ndarray + scale: np.ndarray + weights: np.ndarray + bias: float + + +@dataclass(frozen=True) +class Metrics: + """Binary metrics at one validation-frozen threshold.""" + + count: int + positives: int + negatives: int + true_positives: int + false_positives: int + true_negatives: int + false_negatives: int + tpr: float | None + fpr: float | None + auc: float | None + + +def _safe_artifact_path(root: Path, value: str) -> Path: + """Resolve a manifest-relative path without accepting traversal.""" + candidate = resolve_artifact_path(root, value) + if candidate is None: + raise ValueError(f"unsafe artifact_path {value!r}") + return candidate + + +def _read_manifest(path: Path) -> list[dict[str, str]]: + """Read manifest rows after the caller has run the canonical auditor.""" + with path.open(newline="", encoding="utf-8") as stream: + return list(csv.DictReader(stream)) + + +def load_examples(manifest: Path, target_provider: str) -> list[Example]: + """Load ordinary final-label examples for one provider target. + + Candidate, sham, and source-control rows are deliberately excluded. A + remover-generated negative must not certify the detector that created it, + and repeated controls must not receive extra sample weight. + """ + root = manifest.parent + examples: list[Example] = [] + for row in _read_manifest(manifest): + if row.get("target_provider") != target_provider: + continue + if row.get("split") not in FINAL_SPLITS or row.get("oracle_role") != "ordinary": + continue + outcome = row.get("synthid_outcome") + if outcome not in {"detected", "not_detected"}: + continue + source_provider = row.get("source_provider", "") + if outcome == "detected" and source_provider != target_provider: + raise ValueError( + f"positive group {row.get('group_id')!r} targets {target_provider!r} " + f"but declares source_provider {source_provider!r}" + ) + negative_cohort: str | None = None + if outcome == "not_detected": + if source_provider == target_provider: + negative_cohort = "same_provider" + elif source_provider in {"openai", "google", "other_ai"}: + negative_cohort = "other_ai" + else: + negative_cohort = "external" + examples.append( + Example( + artifact_path=_safe_artifact_path(root, row.get("artifact_path", "")), + artifact_sha256=row.get("artifact_sha256", ""), + group_id=row.get("group_id", ""), + split=row.get("split", ""), + label=1 if outcome == "detected" else 0, + negative_cohort=negative_cohort, + ) + ) + if not examples: + raise ValueError(f"manifest has no eligible ordinary rows for target_provider={target_provider!r}") + return examples + + +def _decoded_rgb(path: Path) -> Image.Image: + """Return rendered RGB pixels with EXIF orientation applied once.""" + with Image.open(path) as image: + return ImageOps.exif_transpose(image).convert("RGB") + + +def _thumbnail_features(image: Image.Image) -> np.ndarray: + """Return a fixed-size RGB content fingerprint with no source geometry.""" + thumbnail = image.resize((THUMBNAIL_SIZE, THUMBNAIL_SIZE), Image.Resampling.LANCZOS) + return np.asarray(thumbnail, dtype=np.float64).reshape(-1) / 255.0 + + +def extract_feature_families(example: Example) -> dict[str, np.ndarray]: + """Extract every confounded feature family with one image decode.""" + with _decoded_rgb(example.artifact_path) as image: + width, height = image.size + thumbnail = _thumbnail_features(image) + file_size = example.artifact_path.stat().st_size + pixels = width * height + image_format = example.artifact_path.suffix.lower().lstrip(".") + if image_format == "jpg": + image_format = "jpeg" + container = np.asarray( + [ + np.log1p(width), + np.log1p(height), + np.log1p(pixels), + np.log1p(file_size), + width / height, + file_size / pixels, + *(1.0 if image_format == name else 0.0 for name in FORMAT_NAMES), + ], + dtype=np.float64, + ) + return { + "container": container, + "thumbnail": np.concatenate((container, thumbnail)), + "canonical": thumbnail, + } + + +def extract_features(example: Example, family: str) -> np.ndarray: + """Extract one deliberately confounded feature family.""" + if family not in FEATURE_FAMILIES: + raise ValueError(f"unsupported feature family {family!r}") + return extract_feature_families(example)[family] + + +def feature_matrix(examples: list[Example], family: str) -> np.ndarray: + """Extract a dense matrix in manifest order.""" + return np.stack([extract_features(example, family) for example in examples]) + + +def feature_matrices(examples: list[Example]) -> dict[str, np.ndarray]: + """Extract all dense feature matrices while decoding each artifact once.""" + rows = [extract_feature_families(example) for example in examples] + return {family: np.stack([row[family] for row in rows]) for family in FEATURE_FAMILIES} + + +def _balanced_sample_weights(labels: np.ndarray) -> np.ndarray: + """Give each class equal total weight regardless of corpus imbalance.""" + positives = int(np.sum(labels == 1)) + negatives = int(np.sum(labels == 0)) + if positives == 0 or negatives == 0: + raise ValueError("training requires at least one positive and one negative") + return np.where(labels == 1, 0.5 / positives, 0.5 / negatives) + + +def _sigmoid(values: np.ndarray) -> np.ndarray: + """Evaluate a numerically stable logistic sigmoid.""" + result = np.empty_like(values, dtype=np.float64) + positive = values >= 0 + result[positive] = 1.0 / (1.0 + np.exp(-values[positive])) + exponent = np.exp(values[~positive]) + result[~positive] = exponent / (1.0 + exponent) + return result + + +def fit_logistic( + features: np.ndarray, + labels: np.ndarray, + *, + iterations: int = LOGISTIC_ITERATIONS, + learning_rate: float = LOGISTIC_LEARNING_RATE, + l2: float = LOGISTIC_L2, +) -> LogisticModel: + """Fit deterministic class-balanced L2 logistic regression.""" + if features.ndim != 2 or labels.shape != (features.shape[0],): + raise ValueError("feature and label shapes are inconsistent") + if iterations < 1 or learning_rate <= 0.0 or l2 < 0.0: + raise ValueError("iterations and learning_rate must be positive; l2 must be nonnegative") + mean = np.mean(features, axis=0) + scale = np.std(features, axis=0) + scale = np.where(scale > 1e-12, scale, 1.0) + standardized = (features - mean) / scale + sample_weights = _balanced_sample_weights(labels) + weights = np.zeros(features.shape[1], dtype=np.float64) + bias = 0.0 + for _ in range(iterations): + probabilities = _sigmoid(standardized @ weights + bias) + error = (probabilities - labels) * sample_weights + gradient = standardized.T @ error + l2 * weights + weights -= learning_rate * gradient + bias -= learning_rate * float(np.sum(error)) + return LogisticModel(mean=mean, scale=scale, weights=weights, bias=bias) + + +def predict_scores(model: LogisticModel, features: np.ndarray) -> np.ndarray: + """Return positive-class probabilities for FEATURES.""" + standardized = (features - model.mean) / model.scale + return _sigmoid(standardized @ model.weights + model.bias) + + +def select_threshold(labels: np.ndarray, scores: np.ndarray, *, max_fpr: float) -> float: + """Choose the validation threshold with maximum TPR under MAX_FPR.""" + if labels.shape != scores.shape or labels.ndim != 1: + raise ValueError("validation labels and scores must be one-dimensional and aligned") + if not 0.0 <= max_fpr <= 1.0: + raise ValueError("max_fpr must be between zero and one") + if not np.any(labels == 1) or not np.any(labels == 0): + raise ValueError("threshold selection requires positive and negative validation examples") + candidates = [float(np.nextafter(np.max(scores), np.inf)), *sorted(set(map(float, scores)), reverse=True)] + best: tuple[float, float, float] | None = None + for threshold in candidates: + predicted = scores >= threshold + tpr = float(np.mean(predicted[labels == 1])) + fpr = float(np.mean(predicted[labels == 0])) + if fpr > max_fpr: + continue + candidate = (tpr, -fpr, threshold) + if best is None or candidate > best: + best = candidate + if best is None: + raise RuntimeError("threshold search found no feasible operating point") + return best[2] + + +def _auc(labels: np.ndarray, scores: np.ndarray) -> float | None: + """Return tie-aware ROC AUC, or None when one class is absent.""" + positive_count = int(np.sum(labels == 1)) + negative_count = int(np.sum(labels == 0)) + if positive_count == 0 or negative_count == 0: + return None + order = np.argsort(scores, kind="stable") + sorted_scores = scores[order] + ranks = np.empty(len(scores), dtype=np.float64) + start = 0 + while start < len(scores): + end = start + 1 + while end < len(scores) and sorted_scores[end] == sorted_scores[start]: + end += 1 + ranks[order[start:end]] = (start + 1 + end) / 2.0 + start = end + positive_rank_sum = float(np.sum(ranks[labels == 1])) + return (positive_rank_sum - positive_count * (positive_count + 1) / 2.0) / (positive_count * negative_count) + + +def calculate_metrics(labels: np.ndarray, scores: np.ndarray, threshold: float) -> Metrics: + """Calculate confusion counts, rates, and AUC at THRESHOLD.""" + predicted = scores >= threshold + positives = labels == 1 + negatives = ~positives + true_positives = int(np.sum(predicted & positives)) + false_positives = int(np.sum(predicted & negatives)) + true_negatives = int(np.sum(~predicted & negatives)) + false_negatives = int(np.sum(~predicted & positives)) + positive_count = int(np.sum(positives)) + negative_count = int(np.sum(negatives)) + return Metrics( + count=len(labels), + positives=positive_count, + negatives=negative_count, + true_positives=true_positives, + false_positives=false_positives, + true_negatives=true_negatives, + false_negatives=false_negatives, + tpr=true_positives / positive_count if positive_count else None, + fpr=false_positives / negative_count if negative_count else None, + auc=_auc(labels, scores), + ) + + +def _split_indices(examples: list[Example], split: str) -> np.ndarray: + """Return integer indices for SPLIT.""" + return np.asarray([index for index, example in enumerate(examples) if example.split == split], dtype=np.int64) + + +def _cohort_metrics( + examples: list[Example], scores: np.ndarray, threshold: float, split: str +) -> dict[str, dict[str, int | float | None]]: + """Report negative-only false-positive rates by provenance cohort.""" + result: dict[str, dict[str, int | float | None]] = {} + for cohort in ("same_provider", "other_ai", "external"): + indices = np.asarray( + [ + index + for index, example in enumerate(examples) + if example.split == split and example.negative_cohort == cohort + ], + dtype=np.int64, + ) + if not len(indices): + result[cohort] = {"count": 0, "false_positives": 0, "fpr": None} + continue + cohort_scores = scores[indices] + false_positives = int(np.sum(cohort_scores >= threshold)) + result[cohort] = { + "count": len(indices), + "false_positives": false_positives, + "fpr": false_positives / len(indices), + } + return result + + +def _require_split_classes(examples: list[Example], split: str) -> None: + """Require both labels in the train, validation, and locked test splits.""" + labels = {example.label for example in examples if example.split == split} + if labels != {0, 1}: + raise ValueError(f"split {split!r} must contain at least one ordinary positive and negative") + + +def run_experiment( + manifest: Path, + target_provider: str, + *, + max_fpr: float = 0.001, + verify_files: bool = True, +) -> dict[str, object]: + """Audit MANIFEST, train all confound baselines, and return a JSON-safe report.""" + errors = audit_manifest(manifest, verify_files=verify_files) + if errors: + preview = "; ".join(errors[:5]) + raise ValueError(f"manifest audit failed with {len(errors)} error(s): {preview}") + examples = load_examples(manifest, target_provider) + for split in ("train", "validation", "test"): + _require_split_classes(examples, split) + + labels = np.asarray([example.label for example in examples], dtype=np.int64) + split_counts = Counter(example.split for example in examples) + negative_counts = Counter(example.negative_cohort for example in examples if example.negative_cohort is not None) + train_indices = _split_indices(examples, "train") + validation_indices = _split_indices(examples, "validation") + report_families: dict[str, object] = {} + matrices = feature_matrices(examples) + + for family in FEATURE_FAMILIES: + features = matrices[family] + model = fit_logistic(features[train_indices], labels[train_indices]) + scores = predict_scores(model, features) + threshold = select_threshold(labels[validation_indices], scores[validation_indices], max_fpr=max_fpr) + split_metrics: dict[str, object] = {} + cohort_metrics: dict[str, object] = {} + for split in FINAL_SPLITS: + indices = _split_indices(examples, split) + if not len(indices): + split_metrics[split] = None + cohort_metrics[split] = { + cohort: {"count": 0, "false_positives": 0, "fpr": None} + for cohort in ("same_provider", "other_ai", "external") + } + continue + split_metrics[split] = asdict(calculate_metrics(labels[indices], scores[indices], threshold)) + cohort_metrics[split] = _cohort_metrics(examples, scores, threshold, split) + report_families[family] = { + "feature_count": features.shape[1], + "threshold": threshold, + "metrics": split_metrics, + "negative_cohorts": cohort_metrics, + } + + manifest_digest = artifact_sha256(manifest) + has_same_provider_test = any( + example.split == "test" and example.negative_cohort == "same_provider" for example in examples + ) + has_temporal_both_classes = {example.label for example in examples if example.split == "temporal"} == {0, 1} + return { + "schema_version": 1, + "experiment": "synthid-d1-confounds", + "target_provider": target_provider, + "manifest_sha256": manifest_digest, + "max_validation_fpr": max_fpr, + "eligible_examples": len(examples), + "split_counts": dict(sorted(split_counts.items())), + "negative_cohort_counts": dict(sorted(negative_counts.items())), + "evidence_ready": has_same_provider_test and has_temporal_both_classes, + "evidence_missing": [ + reason + for missing, reason in ( + (not has_same_provider_test, "locked test has no same-provider hard negative"), + (not has_temporal_both_classes, "temporal split does not contain both labels"), + ) + if missing + ], + "feature_schema": { + "families": list(FEATURE_FAMILIES), + "thumbnail_size": THUMBNAIL_SIZE, + "format_names": list(FORMAT_NAMES), + }, + "logistic_regression": { + "iterations": LOGISTIC_ITERATIONS, + "learning_rate": LOGISTIC_LEARNING_RATE, + "l2": LOGISTIC_L2, + "class_balancing": "equal total weight per class", + }, + "families": report_families, + } + + +@click.command() +@click.argument("manifest", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--target-provider", required=True, type=click.Choice(["openai", "google"])) +@click.option("--report-out", required=True, type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--max-fpr", type=click.FloatRange(0.0, 1.0), default=0.001, show_default=True) +@click.option("--verify-files/--no-verify-files", default=True, show_default=True) +def main(manifest: Path, target_provider: str, report_out: Path, max_fpr: float, verify_files: bool) -> None: + """Run D1 confound baselines from a provider-specific research MANIFEST.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + try: + report = run_experiment(manifest, target_provider, max_fpr=max_fpr, verify_files=verify_files) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + log.info("Wrote D1 confound report: %s", report_out) + if report["evidence_ready"]: + log.info("D1 report contains same-provider locked negatives and a two-class temporal holdout") + else: + log.warning("D1 report lacks required evidence: %s", "; ".join(report["evidence_missing"])) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_consensus_probe.py b/scripts/synthid_consensus_probe.py new file mode 100644 index 0000000..7d03fcf --- /dev/null +++ b/scripts/synthid_consensus_probe.py @@ -0,0 +1,259 @@ +"""Discover a polarity-invariant spectral carrier from low-texture groups. + +This research harness is intentionally separate from the shipping detector. It +uses repeated, independently generated low-texture images to find Fourier bins +whose phase is stable within a content group and whose phase axis is stable +across groups. Treat its output as a carrier hypothesis until it passes the +provider-oracle and hard-negative gates in the SynthID research plan. + +Usage: + uv run --extra pixels python scripts/synthid_consensus_probe.py discover \ + refs/black refs/white refs/red --limit 5 \ + --model-out .local-eval/synthid/consensus.npz + + uv run --extra pixels python scripts/synthid_consensus_probe.py score \ + .local-eval/synthid/consensus.npz images/*.png +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import numpy as np +from PIL import Image, ImageFilter + +log = logging.getLogger(__name__) + +IMAGE_SUFFIXES = {".jpeg", ".jpg", ".png", ".webp"} + + +@dataclass(frozen=True) +class ConsensusScore: + """One image's alignment with a frozen carrier hypothesis.""" + + path: str + score: float + active_weight_fraction: float + peak_count: int + + +@dataclass(frozen=True) +class ConsensusModel: + """A compact, pickle-free carrier hypothesis.""" + + size: int + peaks: np.ndarray + axial_phase: np.ndarray + weights: np.ndarray + expected_magnitude: np.ndarray + + +def _image_paths(directory: Path, limit: int | None) -> list[Path]: + """Return a deterministic list of supported images in DIRECTORY.""" + paths = sorted(path for path in directory.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + if limit is not None: + paths = paths[:limit] + if not paths: + raise ValueError(f"no supported images in {directory}") + return paths + + +def _high_pass_rgb(path: Path, size: int, blur_radius: float) -> np.ndarray: + """Decode PATH, canonicalize its geometry, and remove local image content.""" + with Image.open(path) as source: + image = source.convert("RGB").resize((size, size), Image.Resampling.LANCZOS) + pixels = np.asarray(image, dtype=np.float64) + blurred = np.asarray(image.filter(ImageFilter.GaussianBlur(radius=blur_radius)), dtype=np.float64) + return pixels - blurred + + +def _spectrum(path: Path, size: int, blur_radius: float) -> np.ndarray: + """Return a centered channel-wise spectrum for PATH.""" + residual = _high_pass_rgb(path, size, blur_radius) + return np.fft.fftshift(np.fft.fft2(residual, axes=(0, 1)), axes=(0, 1)) + + +def _group_statistics(paths: list[Path], size: int, blur_radius: float) -> tuple[np.ndarray, np.ndarray]: + """Return phase coherence and mean magnitude for one reference group.""" + spectra = [_spectrum(path, size, blur_radius) for path in paths] + units = [spectrum / (np.abs(spectrum) + 1e-12) for spectrum in spectra] + mean_unit = np.mean(units, axis=0) + coherence = np.abs(mean_unit) + mean_magnitude = np.mean([np.abs(spectrum) for spectrum in spectra], axis=0) + phase = np.angle(mean_unit) + return coherence * np.exp(1j * phase), mean_magnitude + + +def _valid_half_plane(size: int, min_radius: float, max_radius_fraction: float) -> np.ndarray: + """Return the nonredundant Fourier region allowed for carrier selection.""" + center = size // 2 + yy, xx = np.ogrid[:size, :size] + dy = yy - center + dx = xx - center + radius = np.sqrt(np.square(dy) + np.square(dx)) + half_plane = (dy > 0) | ((dy == 0) & (dx > 0)) + off_axis = (dy != 0) & (dx != 0) + return half_plane & off_axis & (radius >= min_radius) & (radius <= size * max_radius_fraction) + + +def discover_model( + groups: list[list[Path]], + *, + size: int = 512, + blur_radius: float = 2.0, + peak_count: int = 256, + min_radius: float = 8.0, + max_radius_fraction: float = 0.4, +) -> ConsensusModel: + """Discover a polarity-invariant carrier from independent image GROUPS.""" + if len(groups) < 2: + raise ValueError("at least two reference groups are required") + if any(len(group) < 2 for group in groups): + raise ValueError("each reference group requires at least two images") + + group_units: list[np.ndarray] = [] + group_magnitudes: list[np.ndarray] = [] + for paths in groups: + unit, magnitude = _group_statistics(paths, size, blur_radius) + group_units.append(unit) + group_magnitudes.append(magnitude) + + stacked = np.stack(group_units, axis=0) + within_coherence = np.abs(stacked) + group_phase = np.angle(stacked) + axial_mean = np.mean(np.exp(2j * group_phase) * within_coherence, axis=0) + axial_coherence = np.abs(axial_mean) / (np.mean(within_coherence, axis=0) + 1e-12) + mean_within = np.mean(within_coherence, axis=0) + expected_magnitude = np.mean(group_magnitudes, axis=0) + + magnitude_scale = np.median(expected_magnitude, axis=(0, 1), keepdims=True) + 1e-12 + magnitude_score = np.log1p(expected_magnitude / magnitude_scale) + selection_score = np.square(mean_within) * np.square(axial_coherence) * magnitude_score + selection_score *= _valid_half_plane(size, min_radius, max_radius_fraction)[:, :, None] + + candidate_count = int(np.count_nonzero(selection_score)) + if candidate_count < peak_count: + raise ValueError(f"only {candidate_count} valid carrier candidates for {peak_count} peaks") + flat = selection_score.ravel() + indices = np.argpartition(flat, -peak_count)[-peak_count:] + indices = indices[np.argsort(flat[indices])[::-1]] + rows, columns, channels = np.unravel_index(indices, selection_score.shape) + peaks = np.column_stack((rows - size // 2, columns - size // 2, channels)).astype(np.int32) + + axial_phase = 0.5 * np.angle(axial_mean[rows, columns, channels]) + weights = selection_score[rows, columns, channels] + weights /= np.sum(weights) + magnitudes = expected_magnitude[rows, columns, channels] + return ConsensusModel( + size=size, + peaks=peaks, + axial_phase=axial_phase.astype(np.float64), + weights=weights.astype(np.float64), + expected_magnitude=magnitudes.astype(np.float64), + ) + + +def score_image(path: Path, model: ConsensusModel, *, blur_radius: float = 2.0) -> ConsensusScore: + """Score PATH against a frozen polarity-invariant carrier model.""" + spectrum = _spectrum(path, model.size, blur_radius) + center = model.size // 2 + rows = center + model.peaks[:, 0] + columns = center + model.peaks[:, 1] + channels = model.peaks[:, 2] + values = spectrum[rows, columns, channels] + phase_alignment = np.cos(2.0 * (np.angle(values) - model.axial_phase)) + magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitude + 1e-12), 1.0) + active_weights = model.weights * magnitude_gate + active_weight = float(np.sum(active_weights)) + score = 0.0 if active_weight == 0.0 else float(np.sum(active_weights * phase_alignment) / active_weight) + return ConsensusScore( + path=str(path), + score=score, + active_weight_fraction=active_weight, + peak_count=len(model.peaks), + ) + + +def save_model(path: Path, model: ConsensusModel) -> None: + """Save MODEL as a pickle-free NPZ artifact.""" + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + path, + size=np.asarray(model.size, dtype=np.int32), + peaks=model.peaks.astype(np.int32), + axial_phase=model.axial_phase.astype(np.float32), + weights=model.weights.astype(np.float32), + expected_magnitude=model.expected_magnitude.astype(np.float32), + ) + + +def load_model(path: Path) -> ConsensusModel: + """Load and validate a pickle-free carrier model.""" + with np.load(path, allow_pickle=False) as artifact: + model = ConsensusModel( + size=int(artifact["size"]), + peaks=np.asarray(artifact["peaks"], dtype=np.int32), + axial_phase=np.asarray(artifact["axial_phase"], dtype=np.float64), + weights=np.asarray(artifact["weights"], dtype=np.float64), + expected_magnitude=np.asarray(artifact["expected_magnitude"], dtype=np.float64), + ) + count = len(model.peaks) + if model.peaks.ndim != 2 or model.peaks.shape[1] != 3: + raise ValueError("invalid peak shape") + if any(array.shape != (count,) for array in (model.axial_phase, model.weights, model.expected_magnitude)): + raise ValueError("model arrays do not match peak count") + if model.size < 64 or np.any(np.abs(model.peaks[:, :2]) >= model.size // 2): + raise ValueError("invalid canonical size or peak coordinates") + if np.any(model.peaks[:, 2] < 0) or np.any(model.peaks[:, 2] > 2): + raise ValueError("invalid channel index") + if not np.isclose(np.sum(model.weights), 1.0, atol=1e-5): + raise ValueError("model weights must sum to one") + return model + + +@click.group() +def main() -> None: + """Run low-texture carrier discovery and scoring experiments.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + + +@main.command() +@click.argument("group_dirs", nargs=-1, required=True, type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.option("--limit", type=click.IntRange(min=2)) +@click.option("--size", type=click.IntRange(min=64), default=512, show_default=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--model-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def discover(group_dirs: tuple[Path, ...], limit: int | None, size: int, peak_count: int, model_out: Path) -> None: + """Discover a carrier from the images in each GROUP_DIRS directory.""" + groups = [_image_paths(directory, limit) for directory in group_dirs] + model = discover_model(groups, size=size, peak_count=peak_count) + save_model(model_out, model) + log.info("Wrote consensus model: %s", model_out) + + +@main.command() +@click.argument("model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path)) +def score(model_path: Path, images: tuple[Path, ...], report_out: Path | None) -> None: + """Score IMAGES against MODEL_PATH.""" + model = load_model(model_path) + payload = { + "model": str(model_path), + "scores": [asdict(score_image(image, model)) for image in images], + } + rendered = json.dumps(payload, indent=2) + "\n" + if report_out is None: + log.info("%s", rendered.rstrip()) + return + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(rendered, encoding="utf-8") + log.info("Wrote score report: %s", report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_ensemble_attack.py b/scripts/synthid_ensemble_attack.py new file mode 100644 index 0000000..5a46f58 --- /dev/null +++ b/scripts/synthid_ensemble_attack.py @@ -0,0 +1,223 @@ +"""Build pixel-only alternating-projection candidates for the ensemble detector. + +The attack removes only the positive complex-spectrum projection onto the +learned RGB phases and HSV saturation/value phases. It preserves geometry and +does not invoke a generative model. Clearing the local research detector is not +proof that a provider oracle will clear SynthID. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict +from pathlib import Path +from typing import TYPE_CHECKING + +import click +import cv2 +import numpy as np +from PIL import Image +from synthid_ensemble_detector import EnsembleConfig, detect_image, load_config, load_models +from synthid_pixel_attack import load_rgb, measure, norm_matched_noise + +if TYPE_CHECKING: + from synthid_color_space_probe import ColorPhaseModel + +log = logging.getLogger(__name__) + + +def _remove_positive_projection( + channel: np.ndarray, + *, + rows: np.ndarray, + columns: np.ndarray, + phases: np.ndarray, + strength: float, +) -> np.ndarray: + """Remove STRENGTH of each positive phase projection from CHANNEL.""" + if strength < 0.0 or strength > 1.0: + raise ValueError("strength must be between zero and one") + height, width = channel.shape + spectrum = np.fft.fft2(channel.astype(np.float64)) + for row, column, phase in zip(rows, columns, phases, strict=True): + row_index = int(row) + column_index = int(column) + direction = np.exp(1j * float(phase)) + value = spectrum[row_index, column_index] + projection = max(0.0, float(np.real(value * np.conj(direction)))) + delta = strength * projection * direction + conjugate_row = (-row_index) % height + conjugate_column = (-column_index) % width + spectrum[row_index, column_index] -= delta + if (conjugate_row, conjugate_column) == (row_index, column_index): + spectrum[row_index, column_index] = complex(spectrum[row_index, column_index].real, 0.0) + else: + spectrum[conjugate_row, conjugate_column] -= np.conj(delta) + return np.fft.ifft2(spectrum).real + + +def _project_model_channels( + pixels: np.ndarray, + model: ColorPhaseModel, + *, + included_channels: frozenset[int], + strength: float, +) -> np.ndarray: + """Apply positive-projection removal to selected MODEL channels.""" + result = pixels.astype(np.float64, copy=True) + for channel in included_channels: + positions = np.flatnonzero(model.channels == channel) + if len(positions) == 0: + continue + result[:, :, channel] = _remove_positive_projection( + result[:, :, channel], + rows=model.rows[positions], + columns=model.columns[positions], + phases=model.phases[positions], + strength=strength, + ) + return result + + +def alternating_projection( + pixels: np.ndarray, + rgb_model: ColorPhaseModel, + hsv_model: ColorPhaseModel, + *, + strength: float, + iterations: int, +) -> np.ndarray: + """Alternate RGB and HSV S/V phase projections without regeneration.""" + if iterations < 1: + raise ValueError("iterations must be positive") + expected_shape = (rgb_model.height, rgb_model.width, 3) + if pixels.shape != expected_shape or pixels.shape != (hsv_model.height, hsv_model.width, 3): + raise ValueError("pixel and model geometries do not match") + result = pixels.astype(np.float64) + for _ in range(iterations): + result = _project_model_channels( + result, + rgb_model, + included_channels=frozenset({0, 1, 2}), + strength=strength, + ) + rgb_unit = np.clip(result / 255.0, 0.0, 1.0).astype(np.float32) + hsv = cv2.cvtColor(rgb_unit, cv2.COLOR_RGB2HSV).astype(np.float64) + hsv = _project_model_channels( + hsv, + hsv_model, + included_channels=frozenset({1, 2}), + strength=strength, + ) + hsv[:, :, 0] = np.mod(hsv[:, :, 0], 360.0) + hsv[:, :, 1:] = np.clip(hsv[:, :, 1:], 0.0, 1.0) + result = cv2.cvtColor(hsv.astype(np.float32), cv2.COLOR_HSV2RGB).astype(np.float64) * 255.0 + return np.clip(np.rint(result), 0, 255).astype(np.uint8) + + +def parse_positive_floats(value: str) -> tuple[float, ...]: + """Parse strictly increasing strengths in the interval (0, 1].""" + try: + values = tuple(float(item.strip()) for item in value.split(",")) + except ValueError as error: + raise click.BadParameter("strengths must be comma-separated numbers") from error + if not values or any(not np.isfinite(item) or item <= 0.0 or item > 1.0 for item in values): + raise click.BadParameter("strengths must be finite and in the interval (0, 1]") + if tuple(sorted(set(values))) != values: + raise click.BadParameter("strengths must be unique and strictly increasing") + return values + + +def parse_positive_integers(value: str) -> tuple[int, ...]: + """Parse strictly increasing positive iteration counts.""" + try: + values = tuple(int(item.strip()) for item in value.split(",")) + except ValueError as error: + raise click.BadParameter("iterations must be comma-separated integers") from error + if not values or any(item < 1 for item in values): + raise click.BadParameter("iterations must be positive") + if tuple(sorted(set(values))) != values: + raise click.BadParameter("iterations must be unique and strictly increasing") + return values + + +def _load_source(path: Path, config: EnsembleConfig) -> np.ndarray: + """Load an exact-geometry RGB source.""" + pixels = load_rgb(path) + if pixels.shape != (config.height, config.width, 3): + raise ValueError("source geometry does not match detector config") + return pixels + + +@click.command() +@click.argument("config_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +@click.option("--strengths", default="0.25,0.5,0.75,1", show_default=True) +@click.option("--iterations", default="1,2,4", show_default=True) +def main(config_path: Path, source: Path, output_dir: Path, strengths: str, iterations: str) -> None: + """Write a frozen pixel-only alternating-projection batch for SOURCE.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + config = load_config(config_path) + rgb_model, hsv_model = load_models(config) + reference = _load_source(source, config) + strength_values = parse_positive_floats(strengths) + iteration_values = parse_positive_integers(iterations) + output_dir.mkdir(parents=True, exist_ok=True) + + variants: list[dict[str, object]] = [] + strongest = reference + for iteration_count in iteration_values: + for strength in strength_values: + pixels = alternating_projection( + reference, + rgb_model, + hsv_model, + strength=strength, + iterations=iteration_count, + ) + strength_name = f"{strength:g}".replace(".", "p") + name = f"project-s{strength_name}-i{iteration_count}" + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + variants.append( + { + **asdict(measure(reference, pixels, name=name, path=path)), + **asdict(detect_image(path, config, rgb_model, hsv_model)), + "strength": strength, + "iterations": iteration_count, + } + ) + strongest = pixels + + sham = norm_matched_noise(reference, strongest, seed=20260823) + sham_path = output_dir / "sham-strongest-rms.png" + Image.fromarray(sham, mode="RGB").save(sham_path) + variants.append( + { + **asdict(measure(reference, sham, name="sham-strongest-rms", path=sham_path)), + **asdict(detect_image(sham_path, config, rgb_model, hsv_model)), + "strength": None, + "iterations": None, + } + ) + + report_path = output_dir / "report.json" + report_path.write_text( + json.dumps( + { + "source": str(source), + "config": str(config_path), + "variants": variants, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d alternating-projection candidates: %s", len(variants), report_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_ensemble_detector.py b/scripts/synthid_ensemble_detector.py new file mode 100644 index 0000000..26c436d --- /dev/null +++ b/scripts/synthid_ensemble_detector.py @@ -0,0 +1,188 @@ +"""Run a positive-only exact-geometry SynthID research detector. + +The detector requires independently frozen RGB and HSV phase models. It emits +``positive`` only when both branches and both support gates pass; every other +case is ``abstain``. It never claims that SynthID is absent. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +from PIL import Image +from synthid_color_space_probe import ColorPhaseModel, ColorPhaseScore, load_model, score_image +from synthid_research_manifest import artifact_sha256 + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class EnsembleConfig: + """Frozen paths, hashes, geometry, and positive thresholds.""" + + width: int + height: int + rgb_model_path: Path + rgb_model_sha256: str + rgb_evidence_threshold: float + rgb_active_threshold: float + hsv_model_path: Path + hsv_model_sha256: str + hsv_sv_evidence_threshold: float + hsv_active_threshold: float + + +@dataclass(frozen=True) +class EnsembleVerdict: + """Positive-only verdict with the evidence needed to audit it.""" + + path: str + verdict: str + reason: str + rgb_evidence: float | None + rgb_active_support: float | None + hsv_sv_evidence: float | None + hsv_active_support: float | None + + +def load_config(path: Path) -> EnsembleConfig: + """Load a frozen epoch manifest and verify both model artifacts.""" + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("verdict_scope") != "positive-only exact-geometry research detector": + raise ValueError("config is not a positive-only exact-geometry detector") + rgb = payload["rgb_model"] + hsv = payload["hsv_model"] + geometry = payload["geometry"] + config = EnsembleConfig( + width=int(geometry["width"]), + height=int(geometry["height"]), + rgb_model_path=Path(rgb["path"]), + rgb_model_sha256=str(rgb["sha256"]), + rgb_evidence_threshold=float(rgb["evidence_threshold"]), + rgb_active_threshold=float(rgb["active_support_threshold"]), + hsv_model_path=Path(hsv["path"]), + hsv_model_sha256=str(hsv["sha256"]), + hsv_sv_evidence_threshold=float(hsv["sv_evidence_threshold"]), + hsv_active_threshold=float(hsv["active_support_threshold"]), + ) + if config.width < 64 or config.height < 64: + raise ValueError("invalid detector geometry") + for model_path, expected_hash in ( + (config.rgb_model_path, config.rgb_model_sha256), + (config.hsv_model_path, config.hsv_model_sha256), + ): + if not model_path.is_file(): + raise ValueError(f"model artifact does not exist: {model_path}") + if artifact_sha256(model_path) != expected_hash: + raise ValueError(f"model artifact hash mismatch: {model_path}") + return config + + +def load_models(config: EnsembleConfig) -> tuple[ColorPhaseModel, ColorPhaseModel]: + """Load and cross-check the RGB and HSV models in CONFIG.""" + rgb_model = load_model(config.rgb_model_path) + hsv_model = load_model(config.hsv_model_path) + if rgb_model.color_space != "rgb" or hsv_model.color_space != "hsv": + raise ValueError("detector requires one RGB model and one HSV model") + expected_geometry = (config.height, config.width) + if (rgb_model.height, rgb_model.width) != expected_geometry: + raise ValueError("RGB model geometry does not match config") + if (hsv_model.height, hsv_model.width) != expected_geometry: + raise ValueError("HSV model geometry does not match config") + return rgb_model, hsv_model + + +def classify_scores( + path: Path, + rgb_score: ColorPhaseScore, + hsv_score: ColorPhaseScore, + config: EnsembleConfig, +) -> EnsembleVerdict: + """Apply CONFIG's positive-only rule to precomputed branch scores.""" + hsv_sv_evidence = float(sum(hsv_score.channel_evidence[1:])) + rgb_support = rgb_score.active_weight_fraction >= config.rgb_active_threshold + hsv_support = hsv_score.active_weight_fraction >= config.hsv_active_threshold + rgb_pass = rgb_score.evidence_score >= config.rgb_evidence_threshold + hsv_pass = hsv_sv_evidence >= config.hsv_sv_evidence_threshold + if rgb_support and hsv_support and rgb_pass and hsv_pass: + verdict = "positive" + reason = "ensemble_pass" + elif not rgb_support or not hsv_support: + verdict = "abstain" + reason = "insufficient_support" + elif rgb_pass != hsv_pass: + verdict = "abstain" + reason = "branch_disagreement" + else: + verdict = "abstain" + reason = "below_positive_threshold" + return EnsembleVerdict( + path=str(path), + verdict=verdict, + reason=reason, + rgb_evidence=rgb_score.evidence_score, + rgb_active_support=rgb_score.active_weight_fraction, + hsv_sv_evidence=hsv_sv_evidence, + hsv_active_support=hsv_score.active_weight_fraction, + ) + + +def detect_image( + path: Path, + config: EnsembleConfig, + rgb_model: ColorPhaseModel, + hsv_model: ColorPhaseModel, +) -> EnsembleVerdict: + """Evaluate PATH or abstain when its geometry is unsupported.""" + with Image.open(path) as image: + if image.size != (config.width, config.height): + return EnsembleVerdict( + path=str(path), + verdict="abstain", + reason="unsupported_geometry", + rgb_evidence=None, + rgb_active_support=None, + hsv_sv_evidence=None, + hsv_active_support=None, + ) + return classify_scores( + path, + score_image(path, rgb_model), + score_image(path, hsv_model), + config, + ) + + +@click.command() +@click.argument("config_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def main(config_path: Path, images: tuple[Path, ...], report_out: Path) -> None: + """Score IMAGES with the frozen positive-only detector CONFIG_PATH.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + config = load_config(config_path) + rgb_model, hsv_model = load_models(config) + verdicts = [detect_image(image, config, rgb_model, hsv_model) for image in images] + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text( + json.dumps( + { + "config": str(config_path), + "positive_count": sum(verdict.verdict == "positive" for verdict in verdicts), + "abstain_count": sum(verdict.verdict == "abstain" for verdict in verdicts), + "verdicts": [asdict(verdict) for verdict in verdicts], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d positive-only detector verdicts: %s", len(verdicts), report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_fragment_attack.py b/scripts/synthid_fragment_attack.py new file mode 100644 index 0000000..dca9a39 --- /dev/null +++ b/scripts/synthid_fragment_attack.py @@ -0,0 +1,243 @@ +"""Build non-generative spatial-fragmentation SynthID attack candidates. + +The candidates combine deterministic smooth local warps with mild global +resampling, color changes, and codec round-trips. These are pixel transforms, +not semantic reconstruction or generative inpainting. Provider-oracle results +must be evaluated in a frozen batch with a source-positive control. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict +from pathlib import Path + +import click +import cv2 +import numpy as np +from PIL import Image +from synthid_pixel_attack import ( + jpeg_round_trip, + load_rgb, + measure, + norm_matched_noise, + resize_squeeze, + smooth_warp, +) +from synthid_v3_codebook_probe import load_v3_model, score_image + +log = logging.getLogger(__name__) + + +def bounded_smooth_warp( + pixels: np.ndarray, + *, + max_displacement: float, + sigma: float, + seed: int, +) -> np.ndarray: + """Apply a smooth warp whose per-axis displacement is absolutely bounded.""" + if max_displacement < 0.0 or sigma <= 0.0: + raise ValueError("max_displacement must be nonnegative and sigma positive") + height, width = pixels.shape[:2] + rng = np.random.default_rng(seed) + fields: list[np.ndarray] = [] + for _ in range(2): + noise = rng.normal(size=(height, width)).astype(np.float32) + field = cv2.GaussianBlur(noise, (0, 0), sigmaX=sigma, sigmaY=sigma) + maximum = float(np.max(np.abs(field))) + fields.append(np.zeros_like(field) if maximum == 0.0 else field * (max_displacement / maximum)) + yy, xx = np.mgrid[:height, :width].astype(np.float32) + return cv2.remap( + pixels, + xx + fields[0], + yy + fields[1], + interpolation=cv2.INTER_LANCZOS4, + borderMode=cv2.BORDER_REFLECT_101, + ) + + +def affine_combo(pixels: np.ndarray, *, rotation_degrees: float, zoom: float) -> np.ndarray: + """Apply one centered rotation-and-zoom resampling operation.""" + if zoom < 0.0: + raise ValueError("zoom must be nonnegative") + height, width = pixels.shape[:2] + matrix = cv2.getRotationMatrix2D( + center=((width - 1) / 2.0, (height - 1) / 2.0), + angle=rotation_degrees, + scale=1.0 + zoom, + ) + return cv2.warpAffine( + pixels, + matrix, + (width, height), + flags=cv2.INTER_LANCZOS4, + borderMode=cv2.BORDER_REFLECT_101, + ) + + +def color_nudge( + pixels: np.ndarray, + *, + brightness: float, + contrast: float, + saturation: float, + hue_degrees: float, +) -> np.ndarray: + """Apply bounded global RGB contrast and HSV saturation/hue changes.""" + rgb = pixels.astype(np.float32) / 255.0 + rgb = np.clip((rgb - 0.5) * (1.0 + contrast) + 0.5 + brightness, 0.0, 1.0) + hsv = cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV) + hsv[:, :, 0] = np.mod(hsv[:, :, 0] + hue_degrees, 360.0) + hsv[:, :, 1] = np.clip(hsv[:, :, 1] * (1.0 + saturation), 0.0, 1.0) + result = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB) + return np.clip(np.rint(result * 255.0), 0, 255).astype(np.uint8) + + +def jpeg_chain(pixels: np.ndarray, qualities: tuple[int, ...]) -> np.ndarray: + """Apply sequential JPEG round-trips at QUALITIES.""" + result = pixels + for quality in qualities: + result = jpeg_round_trip(result, quality) + return result + + +def build_candidates(source: np.ndarray) -> dict[str, np.ndarray]: + """Build the frozen spatial-fragmentation ladder for SOURCE.""" + candidates: dict[str, np.ndarray] = { + "control": source.copy(), + "elastic-075": smooth_warp(source, amplitude=0.75, sigma=56.0, seed=20260812), + "elastic-125": smooth_warp(source, amplitude=1.25, sigma=52.0, seed=20260813), + "bounded-100": bounded_smooth_warp( + source, + max_displacement=1.0, + sigma=56.0, + seed=20260817, + ), + "bounded-180": bounded_smooth_warp( + source, + max_displacement=1.8, + sigma=56.0, + seed=20260818, + ), + "bounded-280": bounded_smooth_warp( + source, + max_displacement=2.8, + sigma=44.0, + seed=20260819, + ), + } + + balanced = smooth_warp(source, amplitude=0.75, sigma=56.0, seed=20260814) + balanced = affine_combo(balanced, rotation_degrees=0.2, zoom=0.004) + balanced = resize_squeeze(balanced, 0.94) + balanced = color_nudge( + balanced, + brightness=0.004, + contrast=0.006, + saturation=-0.005, + hue_degrees=0.15, + ) + balanced = jpeg_chain(balanced, (94, 90)) + candidates["fragment-balanced"] = balanced + + strong = smooth_warp(source, amplitude=1.5, sigma=48.0, seed=20260815) + strong = affine_combo(strong, rotation_degrees=0.4, zoom=0.01) + strong = resize_squeeze(strong, 0.88) + strong = color_nudge( + strong, + brightness=0.008, + contrast=0.012, + saturation=-0.01, + hue_degrees=0.3, + ) + strong = jpeg_chain(strong, (92, 88)) + candidates["fragment-strong"] = strong + candidates["sham-strong-rms"] = norm_matched_noise(source, strong, seed=20260816) + + bounded_balanced = bounded_smooth_warp( + source, + max_displacement=1.8, + sigma=56.0, + seed=20260820, + ) + bounded_balanced = resize_squeeze(bounded_balanced, 0.98) + bounded_balanced = color_nudge( + bounded_balanced, + brightness=0.002, + contrast=0.003, + saturation=-0.003, + hue_degrees=0.1, + ) + bounded_balanced = jpeg_chain(bounded_balanced, (96,)) + candidates["bounded-fragment-balanced"] = bounded_balanced + + bounded_strong = bounded_smooth_warp( + source, + max_displacement=2.8, + sigma=44.0, + seed=20260821, + ) + bounded_strong = affine_combo(bounded_strong, rotation_degrees=0.2, zoom=0.004) + bounded_strong = resize_squeeze(bounded_strong, 0.94) + bounded_strong = color_nudge( + bounded_strong, + brightness=0.004, + contrast=0.006, + saturation=-0.005, + hue_degrees=0.15, + ) + bounded_strong = jpeg_chain(bounded_strong, (94, 90)) + candidates["bounded-fragment-strong"] = bounded_strong + candidates["sham-bounded-strong-rms"] = norm_matched_noise(source, bounded_strong, seed=20260822) + return candidates + + +@click.command() +@click.argument("codebook", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +@click.option("--height", type=click.IntRange(min=64), required=True) +@click.option("--width", type=click.IntRange(min=64), required=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +def main(codebook: Path, source: Path, output_dir: Path, height: int, width: int, peak_count: int) -> None: + """Write a frozen spatial-fragmentation batch for SOURCE.""" + reference = load_rgb(source) + if reference.shape != (height, width, 3): + raise click.BadParameter("source geometry does not match --height and --width") + model = load_v3_model(codebook, height=height, width=width, peak_count=peak_count) + output_dir.mkdir(parents=True, exist_ok=True) + + rows: list[dict[str, object]] = [] + for name, pixels in build_candidates(reference).items(): + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + rows.append( + { + **asdict(measure(reference, pixels, name=name, path=path)), + **asdict(score_image(path, model)), + } + ) + report_path = output_dir / "report.json" + report_path.write_text( + json.dumps( + { + "source": str(source), + "codebook": str(codebook), + "height": height, + "width": width, + "peak_count": peak_count, + "variants": rows, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d frozen fragmentation candidates: %s", len(rows), report_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main() diff --git a/scripts/synthid_hybrid_attack.py b/scripts/synthid_hybrid_attack.py new file mode 100644 index 0000000..1bd649a --- /dev/null +++ b/scripts/synthid_hybrid_attack.py @@ -0,0 +1,140 @@ +"""Build non-generative hybrid phase-projection and fragmentation candidates. + +The matrix combines two independently measured mechanisms: sparse RGB/HSV +phase projection and spatially varying subpixel displacement. It is intended +for frozen provider-oracle batches with a visible-mark-removed source control. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict +from pathlib import Path +from typing import TYPE_CHECKING + +import click +from PIL import Image +from synthid_ensemble_attack import alternating_projection +from synthid_ensemble_detector import detect_image, load_config, load_models +from synthid_fragment_attack import bounded_smooth_warp, color_nudge, jpeg_chain +from synthid_pixel_attack import load_rgb, measure, norm_matched_noise, resize_squeeze, smooth_warp + +if TYPE_CHECKING: + import numpy as np + from synthid_color_space_probe import ColorPhaseModel + +log = logging.getLogger(__name__) + + +def build_candidates( + source: np.ndarray, + rgb_model: ColorPhaseModel, + hsv_model: ColorPhaseModel, +) -> dict[str, np.ndarray]: + """Return a frozen mechanism matrix derived from SOURCE.""" + projected_075 = alternating_projection( + source, + rgb_model, + hsv_model, + strength=0.75, + iterations=1, + ) + projected_100 = alternating_projection( + source, + rgb_model, + hsv_model, + strength=1.0, + iterations=1, + ) + bounded_100 = bounded_smooth_warp( + source, + max_displacement=1.0, + sigma=56.0, + seed=20260824, + ) + projected_bounded_100 = bounded_smooth_warp( + projected_075, + max_displacement=1.0, + sigma=56.0, + seed=20260824, + ) + + bounded_polish = resize_squeeze(projected_bounded_100, 0.98) + bounded_polish = color_nudge( + bounded_polish, + brightness=0.002, + contrast=0.003, + saturation=-0.003, + hue_degrees=0.1, + ) + bounded_polish = jpeg_chain(bounded_polish, (96,)) + + elastic_combo = smooth_warp(projected_100, amplitude=0.75, sigma=56.0, seed=20260825) + elastic_combo = resize_squeeze(elastic_combo, 0.98) + elastic_combo = jpeg_chain(elastic_combo, (96,)) + + return { + "projection-075": projected_075, + "bounded-100": bounded_100, + "projection-075-bounded-100": projected_bounded_100, + "projection-075-bounded-polish": bounded_polish, + "projection-100-elastic-075": elastic_combo, + } + + +@click.command() +@click.argument("config_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +def main(config_path: Path, source: Path, output_dir: Path) -> None: + """Write a frozen non-generative hybrid attack matrix for SOURCE.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + config = load_config(config_path) + rgb_model, hsv_model = load_models(config) + reference = load_rgb(source) + if reference.shape != (config.height, config.width, 3): + raise click.BadParameter("source geometry does not match detector config") + output_dir.mkdir(parents=True, exist_ok=True) + + variants: list[dict[str, object]] = [] + candidates = build_candidates(reference, rgb_model, hsv_model) + for name, pixels in candidates.items(): + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + variants.append( + { + **asdict(measure(reference, pixels, name=name, path=path)), + **asdict(detect_image(path, config, rgb_model, hsv_model)), + } + ) + + selected = candidates["projection-075-bounded-100"] + sham = norm_matched_noise(reference, selected, seed=20260826) + sham_path = output_dir / "sham-projection-bounded-rms.png" + Image.fromarray(sham, mode="RGB").save(sham_path) + variants.append( + { + **asdict(measure(reference, sham, name="sham-projection-bounded-rms", path=sham_path)), + **asdict(detect_image(sham_path, config, rgb_model, hsv_model)), + } + ) + + report_path = output_dir / "report.json" + report_path.write_text( + json.dumps( + { + "source": str(source), + "config": str(config_path), + "variants": variants, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d frozen hybrid candidates: %s", len(variants), report_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_phase_carrier.py b/scripts/synthid_phase_carrier.py new file mode 100644 index 0000000..8c4fce4 --- /dev/null +++ b/scripts/synthid_phase_carrier.py @@ -0,0 +1,312 @@ +"""Discover and evaluate an exact-geometry phase-carrier hypothesis. + +The model is learned only from supplied images and stored as numeric arrays in +a pickle-free NPZ. It is a research baseline, not a proprietary SynthID +decoder. A valid detector claim still requires provider labels, same-provider +hard negatives, group-aware splits, and a locked operating point. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import numpy as np +from PIL import Image + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PhaseCarrierModel: + """Sparse exact-geometry phase carrier learned from positive images.""" + + height: int + width: int + rows: np.ndarray + columns: np.ndarray + channels: np.ndarray + phases: np.ndarray + weights: np.ndarray + expected_magnitudes: np.ndarray + + +@dataclass(frozen=True) +class PhaseCarrierScore: + """Alignment of one exact-geometry image with a phase-carrier model.""" + + path: str + score: float + active_weight_fraction: float + peak_count: int + + +def _load_rgb(path: Path, *, height: int, width: int, canonicalize_geometry: bool = False) -> np.ndarray: + """Load PATH as float64 RGB, optionally resizing to model geometry.""" + with Image.open(path) as image: + rgb = image.convert("RGB") + if rgb.size != (width, height): + if not canonicalize_geometry: + raise ValueError(f"{path}: geometry {rgb.width}x{rgb.height} does not match {width}x{height}") + rgb = rgb.resize((width, height), Image.Resampling.LANCZOS) + return np.asarray(rgb, dtype=np.float64) + + +def _valid_frequency_mask(height: int, width: int, min_radius: float) -> np.ndarray: + """Return eligible non-DC bins in an rFFT half-plane.""" + rows = np.arange(height) + signed_rows = np.where(rows > height // 2, rows - height, rows) + columns = np.arange(width // 2 + 1) + radius = np.sqrt(np.square(signed_rows[:, None]) + np.square(columns[None, :])) + return (radius >= min_radius) & (columns[None, :] > 0) + + +def _leave_one_out_coherence(unit_sum: np.ndarray, held_out_unit: np.ndarray, count: float) -> np.ndarray: + """Return phase coherence after removing HELD_OUT_UNIT from UNIT_SUM.""" + if count <= 1.0: + raise ValueError("leave-one-out coherence requires at least two samples") + return np.abs((unit_sum - held_out_unit) / (count - 1.0)) + + +def discover_model( + paths: list[Path], + *, + peak_count: int = 256, + min_radius: float = 15.0, + candidate_bins: np.ndarray | None = None, +) -> PhaseCarrierModel: + """Learn a sparse phase-consensus model from exact-geometry PATHS.""" + if len(paths) < 3: + raise ValueError("at least three positive images are required") + with Image.open(paths[0]) as first: + width, height = first.size + if min(height, width) < 64: + raise ValueError("images must be at least 64 pixels per side") + + half_width = width // 2 + 1 + unit_sum = np.zeros((height, half_width, 3), dtype=np.complex64) + magnitude_sum = np.zeros((height, half_width, 3), dtype=np.float64) + for path in paths: + pixels = _load_rgb(path, height=height, width=width) + for channel in range(3): + spectrum = np.fft.rfft2(pixels[:, :, channel]) + magnitude = np.abs(spectrum) + unit_sum[:, :, channel] += np.divide( + spectrum, + magnitude, + out=np.zeros_like(spectrum), + where=magnitude != 0.0, + ).astype(np.complex64) + magnitude_sum[:, :, channel] += magnitude + + count = float(len(paths)) + mean_unit = unit_sum / count + minimum_loo_coherence = np.ones((height, half_width, 3), dtype=np.float32) + for path in paths: + pixels = _load_rgb(path, height=height, width=width) + for channel in range(3): + spectrum = np.fft.rfft2(pixels[:, :, channel]) + magnitude = np.abs(spectrum) + unit = np.divide( + spectrum, + magnitude, + out=np.zeros_like(spectrum), + where=magnitude != 0.0, + ) + loo_coherence = _leave_one_out_coherence(unit_sum[:, :, channel], unit, count) + np.minimum(minimum_loo_coherence[:, :, channel], loo_coherence, out=minimum_loo_coherence[:, :, channel]) + expected_magnitude = magnitude_sum / count + selection = np.power(minimum_loo_coherence.astype(np.float64), 4.0) * np.log1p(expected_magnitude) + selection *= _valid_frequency_mask(height, width, min_radius)[:, :, None] + if candidate_bins is None: + candidate_indices = np.flatnonzero(selection) + else: + bins = np.asarray(candidate_bins, dtype=np.int64) + if bins.ndim != 2 or bins.shape[1] != 3: + raise ValueError("candidate_bins must have shape (count, 3)") + if ( + np.any(bins[:, 0] < 0) + or np.any(bins[:, 0] >= height) + or np.any(bins[:, 1] <= 0) + or np.any(bins[:, 1] > width // 2) + or np.any(bins[:, 2] < 0) + or np.any(bins[:, 2] > 2) + ): + raise ValueError("candidate_bins contain out-of-range coordinates") + candidate_indices = np.unique(np.ravel_multi_index(bins.T, selection.shape)) + candidate_indices = candidate_indices[selection.ravel()[candidate_indices] > 0.0] + candidate_count = len(candidate_indices) + if candidate_count < peak_count: + raise ValueError(f"only {candidate_count} eligible bins for {peak_count} peaks") + + flat = selection.ravel() + candidate_scores = flat[candidate_indices] + chosen = np.argpartition(candidate_scores, -peak_count)[-peak_count:] + indices = candidate_indices[chosen] + indices = indices[np.argsort(flat[indices])[::-1]] + rows, columns, channels = np.unravel_index(indices, selection.shape) + raw_weights = selection[rows, columns, channels] + return PhaseCarrierModel( + height=height, + width=width, + rows=rows.astype(np.int32), + columns=columns.astype(np.int32), + channels=channels.astype(np.int8), + phases=np.angle(mean_unit[rows, columns, channels]).astype(np.float64), + weights=(raw_weights / np.sum(raw_weights)).astype(np.float64), + expected_magnitudes=expected_magnitude[rows, columns, channels].astype(np.float64), + ) + + +def score_image( + path: Path, + model: PhaseCarrierModel, + *, + canonicalize_geometry: bool = False, +) -> PhaseCarrierScore: + """Score PATH against MODEL, with optional geometry canonicalization.""" + pixels = _load_rgb( + path, + height=model.height, + width=model.width, + canonicalize_geometry=canonicalize_geometry, + ) + values = np.empty(len(model.rows), dtype=np.complex128) + for channel in range(3): + positions = np.flatnonzero(model.channels == channel) + if len(positions) == 0: + continue + spectrum = np.fft.rfft2(pixels[:, :, channel]) + values[positions] = spectrum[model.rows[positions], model.columns[positions]] + magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0) + active_weights = model.weights * magnitude_gate + active_weight = float(np.sum(active_weights)) + score = ( + 0.0 + if active_weight == 0.0 + else float(np.sum(active_weights * np.cos(np.angle(values) - model.phases)) / active_weight) + ) + return PhaseCarrierScore( + path=str(path), + score=score, + active_weight_fraction=active_weight, + peak_count=len(model.rows), + ) + + +def save_model(path: Path, model: PhaseCarrierModel) -> None: + """Save MODEL as a validated numeric NPZ artifact.""" + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + path, + format_version=np.asarray(1, dtype=np.int32), + height=np.asarray(model.height, dtype=np.int32), + width=np.asarray(model.width, dtype=np.int32), + rows=model.rows.astype(np.int32), + columns=model.columns.astype(np.int32), + channels=model.channels.astype(np.int8), + phases=model.phases.astype(np.float32), + weights=model.weights.astype(np.float32), + expected_magnitudes=model.expected_magnitudes.astype(np.float64), + ) + + +def load_model(path: Path) -> PhaseCarrierModel: + """Load and validate a numeric phase-carrier artifact.""" + with np.load(path, allow_pickle=False) as artifact: + if int(artifact["format_version"]) != 1: + raise ValueError("unsupported phase-carrier format version") + model = PhaseCarrierModel( + height=int(artifact["height"]), + width=int(artifact["width"]), + rows=np.asarray(artifact["rows"], dtype=np.int32), + columns=np.asarray(artifact["columns"], dtype=np.int32), + channels=np.asarray(artifact["channels"], dtype=np.int8), + phases=np.asarray(artifact["phases"], dtype=np.float64), + weights=np.asarray(artifact["weights"], dtype=np.float64), + expected_magnitudes=np.asarray(artifact["expected_magnitudes"], dtype=np.float64), + ) + count = len(model.rows) + arrays = (model.columns, model.channels, model.phases, model.weights, model.expected_magnitudes) + if model.height < 64 or model.width < 64 or any(array.shape != (count,) for array in arrays): + raise ValueError("invalid phase-carrier model shapes") + if count == 0 or np.any(model.rows < 0) or np.any(model.rows >= model.height): + raise ValueError("invalid phase-carrier row indices") + if np.any(model.columns <= 0) or np.any(model.columns > model.width // 2): + raise ValueError("invalid phase-carrier column indices") + if np.any(model.channels < 0) or np.any(model.channels > 2): + raise ValueError("invalid phase-carrier channel indices") + if not np.isclose(np.sum(model.weights), 1.0, atol=1e-5) or np.any(model.weights < 0.0): + raise ValueError("invalid phase-carrier weights") + return model + + +@click.group() +def main() -> None: + """Discover and evaluate an exact-geometry phase carrier.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + + +@main.command() +@click.argument("positives", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--candidate-codebook", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--candidate-count", type=click.IntRange(min=1), default=16384, show_default=True) +@click.option("--model-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def discover( + positives: tuple[Path, ...], + peak_count: int, + candidate_codebook: Path | None, + candidate_count: int, + model_out: Path, +) -> None: + """Learn a phase carrier from exact-geometry POSITIVES.""" + candidate_bins: np.ndarray | None = None + if candidate_codebook is not None: + from synthid_v3_codebook_probe import load_v3_model + + with Image.open(positives[0]) as first: + width, height = first.size + prior = load_v3_model( + candidate_codebook, + height=height, + width=width, + peak_count=candidate_count, + ) + candidate_bins = np.column_stack((prior.rows, prior.columns, prior.channels)) + model = discover_model(list(positives), peak_count=peak_count, candidate_bins=candidate_bins) + save_model(model_out, model) + log.info("Wrote phase-carrier model: %s", model_out) + + +@main.command() +@click.argument("model_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +@click.option("--canonicalize-geometry", is_flag=True, help="Resize inputs to the model geometry before scoring.") +def score( + model_path: Path, + images: tuple[Path, ...], + report_out: Path, + canonicalize_geometry: bool, +) -> None: + """Score IMAGES with MODEL_PATH.""" + model = load_model(model_path) + payload = { + "model": str(model_path), + "height": model.height, + "width": model.width, + "peak_count": len(model.rows), + "canonicalize_geometry": canonicalize_geometry, + "scores": [asdict(score_image(image, model, canonicalize_geometry=canonicalize_geometry)) for image in images], + } + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + log.info("Wrote phase-carrier score report: %s", report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_pixel_attack.py b/scripts/synthid_pixel_attack.py new file mode 100644 index 0000000..b604043 --- /dev/null +++ b/scripts/synthid_pixel_attack.py @@ -0,0 +1,199 @@ +"""Build deterministic, pixel-only SynthID attack candidates and controls. + +The generated variants use quantization, resampling, and a smooth sub-pixel +warp. No generative model or image synthesis stage is involved. The command +also emits a norm-matched random-noise control so an oracle change cannot be +attributed to pixel distance alone. + +This is a research harness. A candidate is successful only when the matching +provider oracle changes from detected to not detected while the crop-only +control remains detected. +""" + +from __future__ import annotations + +import json +import logging +import math +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import cv2 +import numpy as np +from invisible_quality_audit import _ssim +from PIL import Image + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FidelityMeasurement: + """Paired pixel metrics for one attack candidate.""" + + name: str + path: str + width: int + height: int + psnr_db: float + ssim: float + changed_pixel_fraction: float + residual_rms: float + residual_max: float + + +def load_rgb(path: Path) -> np.ndarray: + """Load PATH as uint8 RGB pixels.""" + with Image.open(path) as image: + return np.asarray(image.convert("RGB"), dtype=np.uint8) + + +def crop_visible_badge(pixels: np.ndarray, margin: int) -> np.ndarray: + """Remove the bottom and right margins that contain the visible badge.""" + height, width = pixels.shape[:2] + if margin < 0 or margin >= min(height, width): + raise ValueError("crop margin must be nonnegative and smaller than the image") + if margin == 0: + return pixels.copy() + return pixels[: height - margin, : width - margin].copy() + + +def quantize(pixels: np.ndarray, step: int) -> np.ndarray: + """Round RGB samples to the nearest multiple of STEP.""" + if step < 2 or step > 64: + raise ValueError("quantization step must be between 2 and 64") + values = np.rint(pixels.astype(np.float64) / step) * step + return np.clip(values, 0, 255).astype(np.uint8) + + +def smooth_warp(pixels: np.ndarray, *, amplitude: float, sigma: float, seed: int) -> np.ndarray: + """Apply a deterministic smooth sub-pixel displacement field.""" + if amplitude < 0.0 or sigma <= 0.0: + raise ValueError("warp amplitude must be nonnegative and sigma positive") + height, width = pixels.shape[:2] + rng = np.random.default_rng(seed) + fields: list[np.ndarray] = [] + for _ in range(2): + noise = rng.normal(size=(height, width)).astype(np.float32) + field = cv2.GaussianBlur(noise, (0, 0), sigmaX=sigma, sigmaY=sigma) + field_std = float(np.std(field)) + fields.append(np.zeros_like(field) if field_std == 0.0 else field * (amplitude / field_std)) + yy, xx = np.mgrid[:height, :width].astype(np.float32) + return cv2.remap( + pixels, + xx + fields[0], + yy + fields[1], + interpolation=cv2.INTER_LANCZOS4, + borderMode=cv2.BORDER_REFLECT_101, + ) + + +def resize_squeeze(pixels: np.ndarray, factor: float) -> np.ndarray: + """Downsample and restore the original geometry without synthesis.""" + if not 0.5 <= factor < 1.0: + raise ValueError("resize factor must be in [0.5, 1.0)") + height, width = pixels.shape[:2] + reduced = cv2.resize( + pixels, + (max(1, round(width * factor)), max(1, round(height * factor))), + interpolation=cv2.INTER_AREA, + ) + return cv2.resize(reduced, (width, height), interpolation=cv2.INTER_LANCZOS4) + + +def jpeg_round_trip(pixels: np.ndarray, quality: int) -> np.ndarray: + """Apply one in-memory JPEG encode/decode while returning RGB pixels.""" + if quality < 1 or quality > 100: + raise ValueError("JPEG quality must be between 1 and 100") + success, encoded = cv2.imencode( + ".jpg", + cv2.cvtColor(pixels, cv2.COLOR_RGB2BGR), + [cv2.IMWRITE_JPEG_QUALITY, quality], + ) + if not success: + raise RuntimeError("JPEG encoding failed") + decoded = cv2.imdecode(encoded, cv2.IMREAD_COLOR) + if decoded is None: + raise RuntimeError("JPEG decoding failed") + return cv2.cvtColor(decoded, cv2.COLOR_BGR2RGB) + + +def norm_matched_noise(reference: np.ndarray, target: np.ndarray, *, seed: int) -> np.ndarray: + """Return random RGB noise with approximately TARGET's residual RMS.""" + target_residual = target.astype(np.float64) - reference.astype(np.float64) + target_rms = float(np.sqrt(np.mean(np.square(target_residual)))) + rng = np.random.default_rng(seed) + noise = rng.normal(size=reference.shape) + noise *= target_rms / (float(np.sqrt(np.mean(np.square(noise)))) + 1e-12) + return np.clip(np.rint(reference.astype(np.float64) + noise), 0, 255).astype(np.uint8) + + +def measure(reference: np.ndarray, candidate: np.ndarray, *, name: str, path: Path) -> FidelityMeasurement: + """Measure paired fidelity between equal-shaped RGB arrays.""" + if reference.shape != candidate.shape: + raise ValueError("reference and candidate shapes differ") + residual = candidate.astype(np.float64) - reference.astype(np.float64) + mse = float(np.mean(np.square(residual))) + psnr = math.inf if mse == 0.0 else 20.0 * math.log10(255.0 / math.sqrt(mse)) + reference_gray = cv2.cvtColor(reference, cv2.COLOR_RGB2GRAY) + candidate_gray = cv2.cvtColor(candidate, cv2.COLOR_RGB2GRAY) + return FidelityMeasurement( + name=name, + path=str(path), + width=int(reference.shape[1]), + height=int(reference.shape[0]), + psnr_db=psnr, + ssim=float(_ssim(reference_gray, candidate_gray)), + changed_pixel_fraction=float(np.mean(np.any(residual != 0.0, axis=2))), + residual_rms=float(math.sqrt(mse)), + residual_max=float(np.max(np.abs(residual))), + ) + + +def build_candidates(source: np.ndarray) -> dict[str, np.ndarray]: + """Build the preregistered attack batch from cropped SOURCE pixels.""" + candidates: dict[str, np.ndarray] = { + "control-crop": source.copy(), + "quantize-2": quantize(source, 2), + "quantize-4": quantize(source, 4), + "quantize-8": quantize(source, 8), + "warp-035": smooth_warp(source, amplitude=0.35, sigma=48.0, seed=20260809), + } + combo = smooth_warp(source, amplitude=0.55, sigma=48.0, seed=20260810) + combo = resize_squeeze(combo, 0.96) + combo = quantize(combo, 4) + combo = jpeg_round_trip(combo, 96) + candidates["combo-mild"] = combo + candidates["sham-combo-rms"] = norm_matched_noise(source, combo, seed=20260811) + return candidates + + +@click.command() +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +@click.option("--crop-margin", type=click.IntRange(min=0), default=160, show_default=True) +def main(source: Path, output_dir: Path, crop_margin: int) -> None: + """Write a frozen pixel-only attack batch for SOURCE into OUTPUT_DIR.""" + reference = crop_visible_badge(load_rgb(source), crop_margin) + output_dir.mkdir(parents=True, exist_ok=True) + measurements: list[FidelityMeasurement] = [] + for name, pixels in build_candidates(reference).items(): + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + measurements.append(measure(reference, pixels, name=name, path=path)) + report_path = output_dir / "fidelity.json" + payload = { + "source": str(source), + "crop_margin": crop_margin, + "variants": [asdict(row) for row in measurements], + } + report_path.write_text( + json.dumps(payload, indent=2) + "\n", + encoding="utf-8", + ) + log.info("Wrote %d candidates and fidelity report: %s", len(measurements), report_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main() diff --git a/scripts/synthid_research_inventory.py b/scripts/synthid_research_inventory.py new file mode 100644 index 0000000..fd4cf71 --- /dev/null +++ b/scripts/synthid_research_inventory.py @@ -0,0 +1,170 @@ +"""Inventory local SynthID research images without assigning evidence labels. + +The inventory is deliberately weaker than the research manifest. It records +artifact hashes, decoded RGB hashes, geometry, format, and exact duplicates, +but contains no provider, SynthID outcome, oracle, or split fields. Promotion +from inventory to manifest therefore remains an explicit evidence decision. + +Usage: + uv run python scripts/synthid_research_inventory.py \ + --root .local-eval/synthid negatives google openai \ + --inventory-out .local-eval/synthid/inventory.csv +""" + +from __future__ import annotations + +import csv +import logging +from collections import Counter +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +from synthid_research_manifest import artifact_sha256, decoded_image_fingerprint + +log = logging.getLogger(__name__) + +IMAGE_SUFFIXES = {".jpeg", ".jpg", ".png", ".webp"} +FIELDNAMES = ( + "artifact_sha256", + "pixel_sha256", + "artifact_path", + "width", + "height", + "format", + "exact_pixel_group", + "artifact_duplicate_of", + "pixel_duplicate_of", +) + + +@dataclass(frozen=True) +class InventoryRow: + """One decoded local image with exact-duplicate provenance.""" + + artifact_sha256: str + pixel_sha256: str + artifact_path: str + width: int + height: int + format: str + exact_pixel_group: str + artifact_duplicate_of: str + pixel_duplicate_of: str + + +def _inside_root(root: Path, path: Path) -> Path: + """Resolve PATH and reject anything outside ROOT.""" + resolved_root = root.resolve() + resolved = path.resolve() + try: + resolved.relative_to(resolved_root) + except ValueError as exc: + raise ValueError(f"source is outside inventory root: {path}") from exc + return resolved + + +def discover_images(root: Path, sources: tuple[Path, ...]) -> list[Path]: + """Return supported images below explicit in-root sources in stable order.""" + resolved_root = root.resolve() + discovered: set[Path] = set() + for source in sources: + candidate = source if source.is_absolute() else resolved_root / source + candidate = _inside_root(resolved_root, candidate) + if not candidate.exists(): + raise ValueError(f"inventory source does not exist: {source}") + if candidate.is_file(): + if candidate.suffix.lower() in IMAGE_SUFFIXES: + discovered.add(candidate) + continue + for path in candidate.rglob("*"): + if path.is_file() and path.suffix.lower() in IMAGE_SUFFIXES: + discovered.add(_inside_root(resolved_root, path)) + return sorted(discovered, key=lambda path: path.relative_to(resolved_root).as_posix()) + + +def build_inventory(root: Path, sources: tuple[Path, ...]) -> list[InventoryRow]: + """Hash and decode all selected images without inferring any labels.""" + resolved_root = root.resolve() + first_artifact: dict[str, str] = {} + first_pixels: dict[str, str] = {} + rows: list[InventoryRow] = [] + for path in discover_images(resolved_root, sources): + relative = path.relative_to(resolved_root).as_posix() + artifact_digest = artifact_sha256(path) + pixel_digest, width, height, image_format = decoded_image_fingerprint(path) + rows.append( + InventoryRow( + artifact_sha256=artifact_digest, + pixel_sha256=pixel_digest, + artifact_path=relative, + width=width, + height=height, + format=image_format, + exact_pixel_group=f"pixel-{pixel_digest[:16]}", + artifact_duplicate_of=first_artifact.get(artifact_digest, ""), + pixel_duplicate_of=first_pixels.get(pixel_digest, ""), + ) + ) + first_artifact.setdefault(artifact_digest, relative) + first_pixels.setdefault(pixel_digest, relative) + return rows + + +def write_inventory(path: Path, rows: list[InventoryRow], *, replace: bool = False) -> None: + """Write a complete inventory atomically enough to avoid partial decode results.""" + if path.exists() and not replace: + raise FileExistsError(f"inventory already exists: {path}; pass --replace to overwrite it") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + try: + with temporary.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(asdict(row) for row in rows) + temporary.replace(path) + finally: + if temporary.exists(): + temporary.unlink() + + +def inventory_summary(rows: list[InventoryRow]) -> dict[str, object]: + """Return aggregate coverage without exposing media paths.""" + formats = Counter(row.format for row in rows) + geometries = Counter(f"{row.width}x{row.height}" for row in rows) + return { + "images": len(rows), + "unique_artifacts": len({row.artifact_sha256 for row in rows}), + "unique_pixels": len({row.pixel_sha256 for row in rows}), + "formats": dict(sorted(formats.items())), + "geometries": dict(sorted(geometries.items())), + } + + +@click.command() +@click.option("--root", required=True, type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.argument("sources", nargs=-1, required=True, type=click.Path(path_type=Path)) +@click.option("--inventory-out", required=True, type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--replace", is_flag=True, help="Replace an existing generated inventory.") +def main(root: Path, sources: tuple[Path, ...], inventory_out: Path, replace: bool) -> None: + """Inventory image SOURCES below ROOT without assigning SynthID labels.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + try: + rows = build_inventory(root, sources) + if not rows: + raise ValueError("selected sources contain no supported images") + write_inventory(inventory_out, rows, replace=replace) + except (OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + summary = inventory_summary(rows) + log.info( + "Wrote inventory: %s images=%s unique_artifacts=%s unique_pixels=%s", + inventory_out, + summary["images"], + summary["unique_artifacts"], + summary["unique_pixels"], + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_research_manifest.py b/scripts/synthid_research_manifest.py new file mode 100644 index 0000000..c5b0779 --- /dev/null +++ b/scripts/synthid_research_manifest.py @@ -0,0 +1,347 @@ +"""Audit a private SynthID research manifest before training or evaluation. + +The research manifest is intentionally separate from ``data/synthid/manifest.csv``. +The latter records a small public regression corpus, while this schema tracks +private experiment lineage, provider-specific oracle evidence, and split groups. + +Usage: + uv run python scripts/synthid_research_manifest.py MANIFEST.csv + uv run python scripts/synthid_research_manifest.py MANIFEST.csv --verify-files +""" + +from __future__ import annotations + +import csv +import hashlib +import logging +import re +from collections import defaultdict +from datetime import datetime +from pathlib import Path + +import click +from PIL import Image + +log = logging.getLogger(__name__) + +FIELDNAMES = ( + "artifact_sha256", + "pixel_sha256", + "artifact_path", + "parent_sha256", + "group_id", + "target_provider", + "source_provider", + "surface", + "model_epoch", + "generation_session", + "content_stratum", + "width", + "height", + "format", + "transform", + "split", + "c2pa_outcome", + "synthid_outcome", + "verified_via", + "evidence_reference", + "oracle_session", + "oracle_role", + "captured_at", + "oracle_checked_at", + "notes", +) + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_TARGET_PROVIDERS = {"openai", "google"} +_SOURCE_PROVIDERS = {"openai", "google", "camera", "other_ai", "synthetic", "editor"} +_SPLITS = {"discovery", "train", "validation", "test", "temporal"} +_C2PA_OUTCOMES = {"detected", "not_detected", "invalid", "not_present", "not_checked"} +_SYNTHID_OUTCOMES = {"detected", "not_detected", "indeterminate", "refused", "not_checked"} +_VERIFIERS = {"openai-api", "openai-web", "gemini-app", "synthid-portal", "source-evidence", "none"} +_FORMATS = {"png", "jpeg", "webp"} +_ORACLE_ROLES = {"ordinary", "source_control", "candidate", "sham"} +_FINAL_SPLITS = {"train", "validation", "test", "temporal"} +_MATCHING_VERIFIERS = { + "openai": {"openai-api", "openai-web"}, + "google": {"gemini-app", "synthid-portal"}, +} + + +def _read_rows(path: Path) -> tuple[list[dict[str, str]], list[str]]: + """Read a CSV and return rows plus header errors.""" + with path.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + actual = tuple(reader.fieldnames or ()) + missing = [field for field in FIELDNAMES if field not in actual] + errors = [f"header: missing required field {field!r}" for field in missing] + return list(reader), errors + + +def _is_iso8601(value: str) -> bool: + """Return whether VALUE is a timezone-aware ISO-8601 timestamp.""" + if not value: + return False + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return parsed.tzinfo is not None + + +def _file_sha256(path: Path) -> str: + """Hash a file without loading it entirely into memory.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def decoded_image_fingerprint(path: Path) -> tuple[str, int, int, str]: + """Return the decoded-RGB digest, geometry, and source format.""" + with Image.open(path) as image: + image_format = (image.format or path.suffix.lstrip(".")).lower() + rgb = image.convert("RGB") + digest = hashlib.sha256(rgb.tobytes()).hexdigest() + return digest, rgb.width, rgb.height, "jpeg" if image_format == "jpg" else image_format + + +def _pixel_sha256(path: Path) -> tuple[str, int, int]: + """Hash canonical decoded RGB pixels and return hash, width, and height.""" + digest, width, height, _ = decoded_image_fingerprint(path) + return digest, width, height + + +def artifact_sha256(path: Path) -> str: + """Return the artifact digest used by research manifests and inventories.""" + return _file_sha256(path) + + +def pixel_fingerprint(path: Path) -> tuple[str, int, int]: + """Return the decoded-RGB digest and geometry used by manifest verification.""" + return _pixel_sha256(path) + + +def resolve_artifact_path(root: Path, value: str) -> Path | None: + """Resolve a manifest-relative artifact path without allowing traversal.""" + relative = Path(value) + if not value or relative.is_absolute() or ".." in relative.parts: + return None + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError: + return None + return candidate + + +def _row_errors(row: dict[str, str], index: int) -> list[str]: + """Validate one row without consulting other rows.""" + prefix = f"row {index}" + errors: list[str] = [] + + artifact_sha = row.get("artifact_sha256", "") + pixel_sha = row.get("pixel_sha256", "") + parent_sha = row.get("parent_sha256", "") + if not _SHA256.fullmatch(artifact_sha): + errors.append(f"{prefix}: invalid artifact_sha256") + if not _SHA256.fullmatch(pixel_sha): + errors.append(f"{prefix}: invalid pixel_sha256") + if parent_sha and not _SHA256.fullmatch(parent_sha): + errors.append(f"{prefix}: invalid parent_sha256") + + for field in ("group_id", "surface", "model_epoch", "generation_session", "content_stratum", "transform"): + if not row.get(field, "").strip(): + errors.append(f"{prefix}: {field} must not be empty") + + target = row.get("target_provider", "") + source = row.get("source_provider", "") + split = row.get("split", "") + c2pa = row.get("c2pa_outcome", "") + synthid = row.get("synthid_outcome", "") + verifier = row.get("verified_via", "") + oracle_role = row.get("oracle_role", "") + image_format = row.get("format", "").lower() + + if target not in _TARGET_PROVIDERS: + errors.append(f"{prefix}: unsupported target_provider {target!r}") + if source not in _SOURCE_PROVIDERS: + errors.append(f"{prefix}: unsupported source_provider {source!r}") + if split not in _SPLITS: + errors.append(f"{prefix}: unsupported split {split!r}") + if c2pa not in _C2PA_OUTCOMES: + errors.append(f"{prefix}: unsupported c2pa_outcome {c2pa!r}") + if synthid not in _SYNTHID_OUTCOMES: + errors.append(f"{prefix}: unsupported synthid_outcome {synthid!r}") + if verifier not in _VERIFIERS: + errors.append(f"{prefix}: unsupported verified_via {verifier!r}") + if image_format not in _FORMATS: + errors.append(f"{prefix}: unsupported format {image_format!r}") + if oracle_role not in _ORACLE_ROLES: + errors.append(f"{prefix}: unsupported oracle_role {oracle_role!r}") + + for dimension in ("width", "height"): + try: + if int(row.get(dimension, "")) <= 0: + raise ValueError + except ValueError: + errors.append(f"{prefix}: {dimension} must be a positive integer") + + if not _is_iso8601(row.get("captured_at", "")): + errors.append(f"{prefix}: captured_at must be timezone-aware ISO-8601") + + final_outcome = synthid in {"detected", "not_detected"} + if split in _FINAL_SPLITS and not final_outcome: + errors.append(f"{prefix}: split {split!r} requires a detected or not_detected SynthID outcome") + if final_outcome and not _is_iso8601(row.get("oracle_checked_at", "")): + errors.append(f"{prefix}: a final SynthID outcome requires oracle_checked_at") + + matching = _MATCHING_VERIFIERS.get(target, set()) + if synthid == "detected" and verifier not in matching: + errors.append(f"{prefix}: a detected {target!r} signal requires a matching provider verifier") + if synthid == "not_detected" and source == target and verifier not in matching: + errors.append(f"{prefix}: a same-provider negative requires a matching provider verifier") + if verifier in {"source-evidence", "none"} and synthid == "detected": + errors.append(f"{prefix}: {verifier!r} cannot establish a positive SynthID label") + if verifier == "source-evidence" and source == target: + errors.append(f"{prefix}: source-evidence cannot establish a same-provider negative") + if verifier == "source-evidence" and not row.get("evidence_reference", "").strip(): + errors.append(f"{prefix}: source-evidence requires evidence_reference") + if verifier in matching and not row.get("oracle_session", "").strip(): + errors.append(f"{prefix}: provider verification requires oracle_session") + if oracle_role == "source_control" and synthid != "detected": + errors.append(f"{prefix}: a source_control must have a detected SynthID outcome") + + transform = row.get("transform", "") + if transform == "original" and parent_sha: + errors.append(f"{prefix}: an original must not have parent_sha256") + if transform != "original" and not parent_sha: + errors.append(f"{prefix}: a derivative requires parent_sha256") + + return errors + + +def _lineage_errors(rows: list[dict[str, str]]) -> list[str]: + """Validate uniqueness, parent links, group splits, and lineage cycles.""" + errors: list[str] = [] + by_sha: dict[str, dict[str, str]] = {} + group_splits: defaultdict[str, set[str]] = defaultdict(set) + pixel_groups: defaultdict[str, set[str]] = defaultdict(set) + + for index, row in enumerate(rows, start=2): + artifact_sha = row.get("artifact_sha256", "") + if artifact_sha in by_sha: + errors.append(f"row {index}: duplicate artifact_sha256 {artifact_sha}") + else: + by_sha[artifact_sha] = row + group_splits[row.get("group_id", "")].add(row.get("split", "")) + pixel_groups[row.get("pixel_sha256", "")].add(row.get("group_id", "")) + + for index, row in enumerate(rows, start=2): + parent_sha = row.get("parent_sha256", "") + if not parent_sha: + continue + parent = by_sha.get(parent_sha) + if parent is None: + errors.append(f"row {index}: parent_sha256 is not present in the manifest") + continue + if parent.get("group_id") != row.get("group_id"): + errors.append(f"row {index}: derivative and parent must share group_id") + if parent.get("target_provider") != row.get("target_provider"): + errors.append(f"row {index}: derivative and parent must share target_provider") + + for group_id, splits in sorted(group_splits.items()): + if group_id and len(splits) > 1: + errors.append(f"group {group_id!r}: leaks across splits {sorted(splits)}") + for pixel_sha, groups in sorted(pixel_groups.items()): + if pixel_sha and len(groups) > 1: + errors.append(f"pixel_sha256 {pixel_sha}: appears in multiple groups {sorted(groups)}") + + for artifact_sha in by_sha: + seen: set[str] = set() + current_sha = artifact_sha + while current_sha: + if current_sha in seen: + errors.append(f"artifact_sha256 {artifact_sha}: lineage cycle detected") + break + seen.add(current_sha) + current = by_sha.get(current_sha) + if current is None: + break + current_sha = current.get("parent_sha256", "") + + return errors + + +def _oracle_session_errors(rows: list[dict[str, str]]) -> list[str]: + """Require a healthy source control before accepting removal outcomes.""" + errors: list[str] = [] + positive_controls = { + (row.get("oracle_session", ""), row.get("group_id", ""), row.get("target_provider", "")) + for row in rows + if row.get("oracle_role") == "source_control" and row.get("synthid_outcome") == "detected" + } + for index, row in enumerate(rows, start=2): + if row.get("oracle_role") not in {"candidate", "sham"} or row.get("synthid_outcome") != "not_detected": + continue + key = (row.get("oracle_session", ""), row.get("group_id", ""), row.get("target_provider", "")) + if key not in positive_controls: + errors.append( + f"row {index}: a not_detected {row.get('oracle_role')} requires a detected " + "source_control in the same oracle session, group, and provider" + ) + return errors + + +def audit_manifest(path: Path, *, verify_files: bool = False) -> list[str]: + """Return all manifest errors, including optional on-disk hash checks.""" + rows, errors = _read_rows(path) + if errors: + return errors + for index, row in enumerate(rows, start=2): + errors.extend(_row_errors(row, index)) + errors.extend(_lineage_errors(rows)) + errors.extend(_oracle_session_errors(rows)) + + if verify_files: + root = path.parent + for index, row in enumerate(rows, start=2): + artifact = resolve_artifact_path(root, row.get("artifact_path", "")) + if artifact is None: + errors.append(f"row {index}: artifact_path must be a safe manifest-relative path") + continue + if not artifact.is_file(): + errors.append(f"row {index}: artifact_path does not exist: {row.get('artifact_path', '')}") + continue + if _file_sha256(artifact) != row.get("artifact_sha256"): + errors.append(f"row {index}: artifact_sha256 does not match the file") + try: + pixel_sha, width, height = _pixel_sha256(artifact) + except Exception as exc: # Pillow intentionally accepts many user-controlled formats. + errors.append(f"row {index}: could not decode artifact: {exc}") + continue + if pixel_sha != row.get("pixel_sha256"): + errors.append(f"row {index}: pixel_sha256 does not match decoded RGB pixels") + if str(width) != row.get("width") or str(height) != row.get("height"): + errors.append(f"row {index}: dimensions do not match decoded pixels") + + return errors + + +@click.command() +@click.argument("manifest", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--verify-files", is_flag=True, help="Verify artifact bytes, decoded pixels, and dimensions.") +def main(manifest: Path, verify_files: bool) -> None: + """Audit MANIFEST for evidence, lineage, and split integrity.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + errors = audit_manifest(manifest, verify_files=verify_files) + if errors: + for error in errors: + log.error("ERROR: %s", error) + raise click.ClickException(f"manifest audit failed with {len(errors)} error(s)") + log.info("Manifest audit passed: %s", manifest) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_spectral_probe.py b/scripts/synthid_spectral_probe.py new file mode 100644 index 0000000..5c082a1 --- /dev/null +++ b/scripts/synthid_spectral_probe.py @@ -0,0 +1,312 @@ +"""Discover and score a shared spectral carrier from exact image pairs. + +This is a research harness, not a production SynthID detector. Pair provenance +and oracle labels remain external evidence. The harness deliberately separates +template discovery from single-image scoring and stores arrays in NPZ without +pickle. + +Usage: + uv run python scripts/synthid_spectral_probe.py discover \ + --pair clean.png marked.png --pair clean2.png marked2.png \ + --template-out .local-eval/synthid/template.npz \ + --report-out .local-eval/synthid/pair-report.json + + uv run python scripts/synthid_spectral_probe.py score \ + .local-eval/synthid/template.npz image.png other.png +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from itertools import combinations +from pathlib import Path + +import click +import numpy as np +from PIL import Image, ImageFilter + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PairMeasurement: + """Pixel-domain measurements for one exact clean/marked pair.""" + + clean: str + marked: str + width: int + height: int + psnr_db: float + changed_pixel_fraction: float + difference_min: float + difference_max: float + channel_mean: tuple[float, float, float] + channel_std: tuple[float, float, float] + + +@dataclass(frozen=True) +class ImageScore: + """Single-image phase alignment against a discovered template.""" + + path: str + phase_mean: float + phase_weighted: float + channel_phase_weighted: tuple[float, float, float] + top_two_channel_phase_12: float + peak_count: int + + +def load_rgb(path: Path) -> np.ndarray: + """Load PATH as float64 RGB pixels.""" + with Image.open(path) as image: + return np.asarray(image.convert("RGB"), dtype=np.float64) + + +def _resize_float(channel: np.ndarray, size: int) -> np.ndarray: + """Resize one floating-point channel without quantizing the residual.""" + image = Image.fromarray(channel.astype(np.float32), mode="F") + return np.asarray(image.resize((size, size), Image.Resampling.BILINEAR), dtype=np.float64) + + +def pair_residual(clean: Path, marked: Path, *, size: int = 512) -> tuple[np.ndarray, PairMeasurement]: + """Return a canonical RGB residual and measurements for an exact pair.""" + clean_rgb = load_rgb(clean) + marked_rgb = load_rgb(marked) + if clean_rgb.shape != marked_rgb.shape: + raise ValueError(f"pair shapes differ: {clean_rgb.shape} != {marked_rgb.shape}") + + difference = marked_rgb - clean_rgb + mse = float(np.mean(np.square(difference))) + psnr = float("inf") if mse == 0.0 else float(20.0 * np.log10(255.0 / np.sqrt(mse))) + residual = np.stack([_resize_float(difference[:, :, channel], size) for channel in range(3)], axis=2) + measurement = PairMeasurement( + clean=str(clean), + marked=str(marked), + width=int(clean_rgb.shape[1]), + height=int(clean_rgb.shape[0]), + psnr_db=psnr, + changed_pixel_fraction=float(np.mean(np.any(difference != 0.0, axis=2))), + difference_min=float(np.min(difference)), + difference_max=float(np.max(difference)), + channel_mean=tuple(float(value) for value in np.mean(difference, axis=(0, 1))), + channel_std=tuple(float(value) for value in np.std(difference, axis=(0, 1))), + ) + return residual, measurement + + +def normalized_channels(residual: np.ndarray) -> np.ndarray: + """Zero-center and unit-normalize each residual channel.""" + centered = residual - np.mean(residual, axis=(0, 1), keepdims=True) + norms = np.linalg.norm(centered, axis=(0, 1), keepdims=True) + return np.divide(centered, norms, out=np.zeros_like(centered), where=norms != 0.0) + + +def channel_ncc(first: np.ndarray, second: np.ndarray) -> tuple[float, float, float]: + """Return per-channel normalized cross-correlation.""" + first_norm = normalized_channels(first) + second_norm = normalized_channels(second) + values = np.sum(first_norm * second_norm, axis=(0, 1)) + return tuple(float(value) for value in values) + + +def build_template(residuals: list[np.ndarray]) -> np.ndarray: + """Average canonical residuals after per-channel normalization.""" + if not residuals: + raise ValueError("at least one residual is required") + shape = residuals[0].shape + if any(residual.shape != shape for residual in residuals): + raise ValueError("all canonical residuals must have the same shape") + return np.mean([normalized_channels(residual) for residual in residuals], axis=0) + + +def _template_fft(template: np.ndarray) -> np.ndarray: + """Return a centered two-dimensional FFT for each RGB channel.""" + return np.fft.fftshift(np.fft.fft2(template, axes=(0, 1)), axes=(0, 1)) + + +def select_peaks( + template: np.ndarray, + *, + count: int = 64, + min_radius: float = 8.0, + max_radius_fraction: float = 0.35, + min_distance: float = 3.0, +) -> np.ndarray: + """Select separated high-energy carrier bins from one Fourier half-plane.""" + if count <= 0: + raise ValueError("count must be positive") + height, width, channels = template.shape + if height != width or channels != 3: + raise ValueError("template must be a square RGB array") + center = height // 2 + spectrum = _template_fft(template) + magnitude = np.linalg.norm(spectrum, axis=2) + yy, xx = np.ogrid[:height, :width] + radius = np.sqrt(np.square(yy - center) + np.square(xx - center)) + valid = (radius >= min_radius) & (radius <= height * max_radius_fraction) + candidates = np.flatnonzero(valid) + order = candidates[np.argsort(magnitude.ravel()[candidates])[::-1]] + + selected: list[tuple[int, int]] = [] + for flat_index in order: + row, column = np.unravel_index(flat_index, magnitude.shape) + dy, dx = int(row - center), int(column - center) + if dy < 0 or (dy == 0 and dx < 0): + continue + if any((dy - old_dy) ** 2 + (dx - old_dx) ** 2 < min_distance**2 for old_dy, old_dx in selected): + continue + selected.append((dy, dx)) + if len(selected) == count: + break + if len(selected) != count: + raise ValueError(f"could select only {len(selected)} of {count} peaks") + return np.asarray(selected, dtype=np.int32) + + +def _high_pass_rgb(path: Path, size: int, blur_radius: float) -> np.ndarray: + """Decode, resize, and subtract a small Gaussian blur from RGB pixels.""" + with Image.open(path) as source: + image = source.convert("RGB").resize((size, size), Image.Resampling.LANCZOS) + pixels = np.asarray(image, dtype=np.float64) + blurred = np.asarray(image.filter(ImageFilter.GaussianBlur(radius=blur_radius)), dtype=np.float64) + return pixels - blurred + + +def score_image(path: Path, template: np.ndarray, peaks: np.ndarray, *, blur_radius: float = 2.0) -> ImageScore: + """Score one image by phase alignment at discovered carrier bins.""" + size = int(template.shape[0]) + image_fft = np.fft.fftshift(np.fft.fft2(_high_pass_rgb(path, size, blur_radius), axes=(0, 1)), axes=(0, 1)) + template_fft = _template_fft(template) + center = size // 2 + phase_values: list[np.ndarray] = [] + weights: list[np.ndarray] = [] + for dy, dx in peaks: + image_value = image_fft[center + int(dy), center + int(dx)] + template_value = template_fft[center + int(dy), center + int(dx)] + phase = np.real(image_value * np.conj(template_value)) / (np.abs(image_value) * np.abs(template_value) + 1e-12) + phase_values.append(phase) + weights.append(np.abs(template_value)) + phases = np.asarray(phase_values) + carrier_weights = np.asarray(weights) + channel_weighted = np.sum(phases * carrier_weights, axis=0) / np.sum(carrier_weights, axis=0) + consensus_count = min(12, len(peaks)) + consensus_channels = np.sum(phases[:consensus_count] * carrier_weights[:consensus_count], axis=0) / np.sum( + carrier_weights[:consensus_count], axis=0 + ) + top_two_channel_phase_12 = float(np.mean(np.sort(consensus_channels)[-2:])) + return ImageScore( + path=str(path), + phase_mean=float(np.mean(phases)), + phase_weighted=float(np.sum(phases * carrier_weights) / np.sum(carrier_weights)), + channel_phase_weighted=tuple(float(value) for value in channel_weighted), + top_two_channel_phase_12=top_two_channel_phase_12, + peak_count=len(peaks), + ) + + +def save_template(path: Path, template: np.ndarray, peaks: np.ndarray) -> None: + """Store a template in a pickle-free compressed NPZ artifact.""" + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(path, template=template.astype(np.float32), peaks=peaks.astype(np.int32)) + + +def load_template(path: Path) -> tuple[np.ndarray, np.ndarray]: + """Load a template artifact without enabling pickle.""" + with np.load(path, allow_pickle=False) as artifact: + template = np.asarray(artifact["template"], dtype=np.float64) + peaks = np.asarray(artifact["peaks"], dtype=np.int32) + if template.ndim != 3 or template.shape[2] != 3 or template.shape[0] != template.shape[1]: + raise ValueError("invalid template shape") + if peaks.ndim != 2 or peaks.shape[1] != 2: + raise ValueError("invalid peak shape") + return template, peaks + + +def discovery_report( + residuals: list[np.ndarray], measurements: list[PairMeasurement], peaks: np.ndarray +) -> dict[str, object]: + """Build a JSON-safe report with pair statistics and cross-pair NCC.""" + pairwise = [ + { + "first": measurements[first].marked, + "second": measurements[second].marked, + "channel_ncc": channel_ncc(residuals[first], residuals[second]), + } + for first, second in combinations(range(len(residuals)), 2) + ] + return { + "pair_count": len(measurements), + "pairs": [asdict(measurement) for measurement in measurements], + "pairwise": pairwise, + "peaks": peaks.tolist(), + } + + +@click.group() +def main() -> None: + """Discover and score an experimental shared spectral carrier.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + + +@main.command() +@click.option( + "--pair", + "pairs", + type=( + click.Path(exists=True, dir_okay=False, path_type=Path), + click.Path(exists=True, dir_okay=False, path_type=Path), + ), + multiple=True, + required=True, + help="Exact CLEAN MARKED pair; repeat for multiple pairs.", +) +@click.option("--size", type=click.IntRange(min=64), default=512, show_default=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=64, show_default=True) +@click.option("--template-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def discover( + pairs: tuple[tuple[Path, Path], ...], + size: int, + peak_count: int, + template_out: Path, + report_out: Path, +) -> None: + """Build a template and report from exact CLEAN MARKED pairs.""" + residuals: list[np.ndarray] = [] + measurements: list[PairMeasurement] = [] + for clean, marked in pairs: + residual, measurement = pair_residual(clean, marked, size=size) + residuals.append(residual) + measurements.append(measurement) + template = build_template(residuals) + peaks = select_peaks(template, count=peak_count) + save_template(template_out, template, peaks) + report = discovery_report(residuals, measurements, peaks) + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + log.info("Wrote template: %s", template_out) + log.info("Wrote report: %s", report_out) + + +@main.command() +@click.argument("template_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path)) +def score(template_path: Path, images: tuple[Path, ...], report_out: Path | None) -> None: + """Score IMAGES against TEMPLATE_PATH.""" + template, peaks = load_template(template_path) + scores = [asdict(score_image(image, template, peaks)) for image in images] + payload = {"template": str(template_path), "scores": scores} + rendered = json.dumps(payload, indent=2) + "\n" + if report_out is None: + log.info("%s", rendered.rstrip()) + return + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(rendered, encoding="utf-8") + log.info("Wrote score report: %s", report_out) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_tile_attack.py b/scripts/synthid_tile_attack.py new file mode 100644 index 0000000..57110ec --- /dev/null +++ b/scripts/synthid_tile_attack.py @@ -0,0 +1,166 @@ +"""Estimate and subtract a periodic SynthID residual tile without regeneration. + +At 1536x2816, the dominant carrier bins lie on an FFT lattice spaced by 96 +rows and 88 columns, corresponding to a 16x32 spatial tile. Folding a +high-pass residual modulo that tile averages over 8448 repetitions and +suppresses non-periodic image content. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict +from pathlib import Path + +import click +import cv2 +import numpy as np +from PIL import Image +from synthid_ensemble_detector import detect_image, load_config, load_models +from synthid_pixel_attack import load_rgb, measure + +log = logging.getLogger(__name__) + + +def fold_residual_template( + pixels: np.ndarray, + *, + tile_height: int, + tile_width: int, + denoise_sigma: float, +) -> np.ndarray: + """Estimate a zero-mean periodic residual template by modulo folding.""" + if pixels.ndim != 3 or pixels.shape[2] != 3: + raise ValueError("pixels must have shape (height, width, 3)") + if tile_height < 1 or tile_width < 1 or denoise_sigma <= 0.0: + raise ValueError("tile dimensions and denoise sigma must be positive") + height, width = pixels.shape[:2] + if height % tile_height != 0 or width % tile_width != 0: + raise ValueError("image geometry must be divisible by the tile geometry") + source = pixels.astype(np.float64) + denoised = cv2.GaussianBlur( + source, + (0, 0), + sigmaX=denoise_sigma, + sigmaY=denoise_sigma, + borderType=cv2.BORDER_REFLECT_101, + ) + residual = source - denoised + repeats_y = height // tile_height + repeats_x = width // tile_width + folded = residual.reshape(repeats_y, tile_height, repeats_x, tile_width, 3).mean(axis=(0, 2)) + return folded - np.mean(folded, axis=(0, 1), keepdims=True) + + +def subtract_tiled_template(pixels: np.ndarray, template: np.ndarray, *, strength: float) -> np.ndarray: + """Subtract STRENGTH times TEMPLATE repeated over PIXELS.""" + if strength < 0.0: + raise ValueError("strength must be nonnegative") + height, width = pixels.shape[:2] + tile_height, tile_width = template.shape[:2] + if template.shape[2:] != (3,) or height % tile_height != 0 or width % tile_width != 0: + raise ValueError("template does not tile the pixel geometry") + repeated = np.tile(template, (height // tile_height, width // tile_width, 1)) + result = pixels.astype(np.float64) - strength * repeated + return np.clip(np.rint(result), 0, 255).astype(np.uint8) + + +def parse_positive_floats(value: str, *, option_name: str) -> tuple[float, ...]: + """Parse a strictly increasing comma-separated positive-float sweep.""" + try: + values = tuple(float(item.strip()) for item in value.split(",")) + except ValueError as error: + raise click.BadParameter(f"{option_name} must be comma-separated numbers") from error + if not values or any(not np.isfinite(item) or item <= 0.0 for item in values): + raise click.BadParameter(f"{option_name} must be finite and positive") + if tuple(sorted(set(values))) != values: + raise click.BadParameter(f"{option_name} must be unique and strictly increasing") + return values + + +@click.command() +@click.argument("config_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +@click.option("--tile-height", type=click.IntRange(min=1), default=16, show_default=True) +@click.option("--tile-width", type=click.IntRange(min=1), default=32, show_default=True) +@click.option("--denoise-sigmas", default="0.6,1,1.5", show_default=True) +@click.option("--strengths", default="0.5,1,1.5,2", show_default=True) +def main( + config_path: Path, + source: Path, + output_dir: Path, + tile_height: int, + tile_width: int, + denoise_sigmas: str, + strengths: str, +) -> None: + """Write a frozen periodic-tile subtraction sweep for SOURCE.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + config = load_config(config_path) + rgb_model, hsv_model = load_models(config) + reference = load_rgb(source) + if reference.shape != (config.height, config.width, 3): + raise click.BadParameter("source geometry does not match detector config") + sigma_values = parse_positive_floats(denoise_sigmas, option_name="denoise sigmas") + strength_values = parse_positive_floats(strengths, option_name="strengths") + output_dir.mkdir(parents=True, exist_ok=True) + + variants: list[dict[str, object]] = [] + templates: list[dict[str, object]] = [] + for sigma in sigma_values: + template = fold_residual_template( + reference, + tile_height=tile_height, + tile_width=tile_width, + denoise_sigma=sigma, + ) + sigma_name = f"{sigma:g}".replace(".", "p") + templates.append( + { + "denoise_sigma": sigma, + "template_rms": float(np.sqrt(np.mean(np.square(template)))), + "template_max_abs": float(np.max(np.abs(template))), + } + ) + shifted = np.roll(template, shift=(1, 1), axis=(0, 1)) + for strength in strength_values: + strength_name = f"{strength:g}".replace(".", "p") + for control_name, selected_template in (("aligned", template), ("shifted", shifted)): + name = f"tile-{control_name}-sigma{sigma_name}-s{strength_name}" + pixels = subtract_tiled_template(reference, selected_template, strength=strength) + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + variants.append( + { + **asdict(measure(reference, pixels, name=name, path=path)), + **asdict(detect_image(path, config, rgb_model, hsv_model)), + "denoise_sigma": sigma, + "strength": strength, + "template_alignment": control_name, + } + ) + + report_path = output_dir / "report.json" + report_path.write_text( + json.dumps( + { + "source": str(source), + "config": str(config_path), + "tile_height": tile_height, + "tile_width": tile_width, + "repeat_count": (config.height // tile_height) * (config.width // tile_width), + "templates": templates, + "variants": variants, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d periodic-tile candidates: %s", len(variants), report_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/synthid_v3_carrier_subtract.py b/scripts/synthid_v3_carrier_subtract.py new file mode 100644 index 0000000..335b541 --- /dev/null +++ b/scripts/synthid_v3_carrier_subtract.py @@ -0,0 +1,148 @@ +"""Build pixel-only V3 carrier-subtraction candidates and matched controls. + +The command uses a frozen numeric frequency profile as a local research +surrogate. It subtracts a sparse Hermitian spectrum, preserves image geometry, +and never invokes a generative model. A lower local score is not evidence that +the provider's SynthID verifier will change its decision. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict +from pathlib import Path + +import click +import numpy as np +from PIL import Image +from synthid_pixel_attack import load_rgb, measure, norm_matched_noise +from synthid_v3_codebook_probe import V3CarrierModel, load_v3_model, score_image + +log = logging.getLogger(__name__) + + +def load_exact_rgb(path: Path, model: V3CarrierModel) -> np.ndarray: + """Load PATH as RGB and reject geometry that differs from MODEL.""" + pixels = load_rgb(path) + if pixels.shape != (model.height, model.width, 3): + height, width = pixels.shape[:2] + raise ValueError(f"image geometry {width}x{height} does not match profile {model.width}x{model.height}") + return pixels + + +def subtract_carrier(pixels: np.ndarray, model: V3CarrierModel, *, strength: float) -> np.ndarray: + """Subtract STRENGTH times MODEL's sparse complex carrier from PIXELS.""" + if strength < 0.0: + raise ValueError("strength must be nonnegative") + expected_shape = (model.height, model.width, 3) + if pixels.shape != expected_shape: + raise ValueError(f"pixel shape {pixels.shape} does not match {expected_shape}") + + result = np.empty_like(pixels, dtype=np.float64) + for channel in range(3): + spectrum = np.fft.fft2(pixels[:, :, channel].astype(np.float64)) + positions = np.flatnonzero(model.channels == channel) + deltas: dict[tuple[int, int], complex] = {} + for position in positions: + row = int(model.rows[position]) + column = int(model.columns[position]) + delta = strength * model.expected_magnitudes[position] * np.exp(1j * model.phases[position]) + key = (row, column) + conjugate_key = ((-row) % model.height, (-column) % model.width) + deltas[key] = deltas.get(key, 0.0j) + delta + if conjugate_key == key: + deltas[key] = complex(deltas[key].real, 0.0) + else: + deltas[conjugate_key] = deltas.get(conjugate_key, 0.0j) + np.conj(delta) + for (row, column), delta in deltas.items(): + spectrum[row, column] -= delta + result[:, :, channel] = np.fft.ifft2(spectrum).real + return np.clip(np.rint(result), 0, 255).astype(np.uint8) + + +def parse_strengths(value: str) -> tuple[float, ...]: + """Parse a comma-separated, strictly increasing nonnegative sweep.""" + try: + strengths = tuple(float(item.strip()) for item in value.split(",")) + except ValueError as error: + raise click.BadParameter("strengths must be comma-separated numbers") from error + if not strengths or any(not np.isfinite(item) or item < 0.0 for item in strengths): + raise click.BadParameter("strengths must be finite and nonnegative") + if tuple(sorted(set(strengths))) != strengths: + raise click.BadParameter("strengths must be unique and strictly increasing") + return strengths + + +@click.command() +@click.argument("codebook", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("source", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("output_dir", type=click.Path(file_okay=False, path_type=Path)) +@click.option("--height", type=click.IntRange(min=64), required=True) +@click.option("--width", type=click.IntRange(min=64), required=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--strengths", default="0.25,0.5,1,1.5,2,4", show_default=True) +def main( + codebook: Path, + source: Path, + output_dir: Path, + height: int, + width: int, + peak_count: int, + strengths: str, +) -> None: + """Write a frozen analytical carrier-subtraction batch for SOURCE.""" + model = load_v3_model(codebook, height=height, width=width, peak_count=peak_count) + reference = load_exact_rgb(source, model) + sweep = parse_strengths(strengths) + output_dir.mkdir(parents=True, exist_ok=True) + + variants: list[dict[str, object]] = [] + strongest = reference + for strength in sweep: + pixels = subtract_carrier(reference, model, strength=strength) + name = f"subtract-{strength:g}".replace(".", "p") + path = output_dir / f"{name}.png" + Image.fromarray(pixels, mode="RGB").save(path) + variants.append( + { + **asdict(measure(reference, pixels, name=name, path=path)), + **asdict(score_image(path, model)), + "strength": strength, + } + ) + strongest = pixels + + sham = norm_matched_noise(reference, strongest, seed=20260809) + sham_path = output_dir / "sham-strongest-rms.png" + Image.fromarray(sham, mode="RGB").save(sham_path) + variants.append( + { + **asdict(measure(reference, sham, name="sham-strongest-rms", path=sham_path)), + **asdict(score_image(sham_path, model)), + "strength": None, + } + ) + + report_path = output_dir / "report.json" + report_path.write_text( + json.dumps( + { + "source": str(source), + "codebook": str(codebook), + "height": height, + "width": width, + "peak_count": peak_count, + "variants": variants, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + log.info("Wrote %d frozen carrier candidates: %s", len(variants), report_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main() diff --git a/scripts/synthid_v3_codebook_probe.py b/scripts/synthid_v3_codebook_probe.py new file mode 100644 index 0000000..fcf8828 --- /dev/null +++ b/scripts/synthid_v3_codebook_probe.py @@ -0,0 +1,172 @@ +"""Independently evaluate a numeric reverse-SynthID V3 NPZ codebook. + +The loader accepts only the documented numeric format-v2 arrays and disables +pickle. It does not import or execute third-party code. Scores are exploratory: +the external reference provenance and labels still require independent oracle +validation before this can support a SynthID detector claim. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import asdict, dataclass +from pathlib import Path + +import click +import numpy as np +from PIL import Image + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class V3CarrierModel: + """Selected numeric bins from one exact-resolution V3 profile.""" + + height: int + width: int + rows: np.ndarray + columns: np.ndarray + channels: np.ndarray + phases: np.ndarray + weights: np.ndarray + expected_magnitudes: np.ndarray + + +@dataclass(frozen=True) +class V3Score: + """Phase-alignment scores for one image.""" + + path: str + phase_score: float + axial_phase_score: float + active_weight_fraction: float + peak_count: int + + +def _load_sparse_channel(artifact: np.lib.npyio.NpzFile, prefix: str, channel: int) -> tuple[np.ndarray, ...]: + """Load one sparse channel without reconstructing full image-sized arrays.""" + indices = np.asarray(artifact[f"{prefix}idx_{channel}"], dtype=np.uint32) + magnitudes = np.exp2(np.asarray(artifact[f"{prefix}mag_{channel}"], dtype=np.float64)) - 1.0 + phases = np.asarray(artifact[f"{prefix}phase_{channel}"], dtype=np.float64) + coherence = np.asarray(artifact[f"{prefix}cons_{channel}"], dtype=np.float64) / 255.0 + if not (indices.shape == magnitudes.shape == phases.shape == coherence.shape): + raise ValueError("sparse profile arrays have inconsistent shapes") + return indices, magnitudes, phases, coherence + + +def load_v3_model( + path: Path, + *, + height: int, + width: int, + peak_count: int = 256, + min_radius: float = 15.0, +) -> V3CarrierModel: + """Load top phase-consistent bins from a numeric V3 codebook profile.""" + prefix = f"{height}x{width}/" + half_width = width // 2 + 1 + candidates: list[tuple[float, int, int, int, float, float]] = [] + with np.load(path, allow_pickle=False) as artifact: + if int(artifact["format_version"]) != 2: + raise ValueError("only numeric V3 format version 2 is supported") + if not bool(int(artifact[f"{prefix}sparse"])): + raise ValueError("only sparse profiles are supported by this audit loader") + for channel in range(3): + indices, magnitudes, phases, coherence = _load_sparse_channel(artifact, prefix, channel) + rows, columns = np.unravel_index(indices, (height, half_width)) + signed_rows = np.where(rows > height // 2, rows - height, rows) + radius = np.sqrt(np.square(signed_rows) + np.square(columns)) + valid = (radius >= min_radius) & (columns > 0) + selection = np.square(coherence) * np.log1p(magnitudes) + for index in np.flatnonzero(valid): + candidates.append( + ( + float(selection[index]), + int(rows[index]), + int(columns[index]), + channel, + float(phases[index]), + float(magnitudes[index]), + ) + ) + if len(candidates) < peak_count: + raise ValueError(f"profile exposes only {len(candidates)} eligible bins") + selected = sorted(candidates, reverse=True)[:peak_count] + raw_weights = np.asarray([item[0] for item in selected], dtype=np.float64) + return V3CarrierModel( + height=height, + width=width, + rows=np.asarray([item[1] for item in selected], dtype=np.int32), + columns=np.asarray([item[2] for item in selected], dtype=np.int32), + channels=np.asarray([item[3] for item in selected], dtype=np.int8), + phases=np.asarray([item[4] for item in selected], dtype=np.float64), + weights=raw_weights / np.sum(raw_weights), + expected_magnitudes=np.asarray([item[5] for item in selected], dtype=np.float64), + ) + + +def _load_profile_rgb(path: Path, model: V3CarrierModel) -> np.ndarray: + """Load PATH and resize only when it does not match the profile geometry.""" + with Image.open(path) as source: + image = source.convert("RGB") + if image.size != (model.width, model.height): + image = image.resize((model.width, model.height), Image.Resampling.LANCZOS) + return np.asarray(image, dtype=np.float64) + + +def score_image(path: Path, model: V3CarrierModel) -> V3Score: + """Score PATH against selected V3 phase bins.""" + pixels = _load_profile_rgb(path, model) + values = np.empty(len(model.rows), dtype=np.complex128) + for channel in range(3): + positions = np.flatnonzero(model.channels == channel) + if len(positions) == 0: + continue + spectrum = np.fft.fft2(pixels[:, :, channel]) + values[positions] = spectrum[model.rows[positions], model.columns[positions]] + phase_difference = np.angle(values) - model.phases + magnitude_gate = np.minimum(np.abs(values) / (model.expected_magnitudes + 1e-12), 1.0) + active_weights = model.weights * magnitude_gate + active_weight = float(np.sum(active_weights)) + if active_weight == 0.0: + phase_score = 0.0 + axial_score = 0.0 + else: + phase_score = float(np.sum(active_weights * np.cos(phase_difference)) / active_weight) + axial_score = float(np.sum(active_weights * np.cos(2.0 * phase_difference)) / active_weight) + return V3Score( + path=str(path), + phase_score=phase_score, + axial_phase_score=axial_score, + active_weight_fraction=active_weight, + peak_count=len(model.rows), + ) + + +@click.command() +@click.argument("codebook", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.argument("images", nargs=-1, required=True, type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option("--height", type=click.IntRange(min=64), required=True) +@click.option("--width", type=click.IntRange(min=64), required=True) +@click.option("--peak-count", type=click.IntRange(min=1), default=256, show_default=True) +@click.option("--report-out", type=click.Path(dir_okay=False, path_type=Path), required=True) +def main(codebook: Path, images: tuple[Path, ...], height: int, width: int, peak_count: int, report_out: Path) -> None: + """Score IMAGES against one exact-resolution profile from CODEBOOK.""" + model = load_v3_model(codebook, height=height, width=width, peak_count=peak_count) + payload = { + "codebook": str(codebook), + "height": height, + "width": width, + "peak_count": peak_count, + "scores": [asdict(score_image(image, model)) for image in images], + } + report_out.parent.mkdir(parents=True, exist_ok=True) + report_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + log.info("Wrote V3 score report: %s", report_out) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main() diff --git a/tests/test_synthid_color_space_probe.py b/tests/test_synthid_color_space_probe.py new file mode 100644 index 0000000..1f4c697 --- /dev/null +++ b/tests/test_synthid_color_space_probe.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_color_space_probe as probe + + +def _write_image(path: Path, *, phase: float | None, seed: int) -> None: + height = width = 64 + rng = np.random.default_rng(seed) + pixels = 100.0 + rng.normal(0.0, 3.0, size=(height, width, 3)) + if phase is not None: + yy, xx = np.mgrid[:height, :width] + pixels[:, :, 0] += 12.0 * np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + phase) + pixels[:, :, 1] -= 8.0 * np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + phase) + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +@pytest.mark.parametrize("color_space", probe.COLOR_SPACES) +def test_color_transforms_are_finite_and_preserve_shape(color_space: str) -> None: + rgb = np.asarray([[[0.0, 128.0, 255.0], [64.0, 64.0, 64.0]]]) + + transformed = probe.transform_color_space(rgb, color_space) + + assert transformed.shape == rgb.shape + assert np.all(np.isfinite(transformed)) + + +def test_neutral_gray_has_neutral_chroma() -> None: + gray = np.full((1, 1, 3), 96.0) + + ycbcr = probe.transform_color_space(gray, "ycbcr") + ycocg = probe.transform_color_space(gray, "ycocg") + opponent = probe.transform_color_space(gray, "opponent") + lab = probe.transform_color_space(gray, "lab") + hsv = probe.transform_color_space(gray, "hsv") + + assert ycbcr[0, 0, 1:] == pytest.approx((128.0, 128.0), abs=1e-4) + assert ycocg[0, 0, 1:] == pytest.approx((0.0, 0.0), abs=1e-8) + assert opponent[0, 0, 1:] == pytest.approx((0.0, 0.0), abs=1e-8) + assert lab[0, 0, 1:] == pytest.approx((0.0, 0.0), abs=0.1) + assert hsv[0, 0, 1] == pytest.approx(0.0, abs=1e-8) + + +@pytest.mark.parametrize("color_space", ["rgb", "opponent"]) +def test_discovered_model_separates_shared_phase_from_noise(tmp_path: Path, color_space: str) -> None: + positives: list[Path] = [] + for index in range(4): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + heldout = tmp_path / "heldout.png" + negative = tmp_path / "negative.png" + _write_image(heldout, phase=0.4, seed=10) + _write_image(negative, phase=None, seed=11) + bins = np.asarray([(7, 5, channel) for channel in range(3)], dtype=np.int32) + + model = probe.discover_model( + positives, + color_space=color_space, + candidate_bins=bins, + peak_count=3, + ) + + assert probe.score_image(heldout, model).evidence_score > probe.score_image(negative, model).evidence_score + + +def test_model_round_trip_is_pickle_free(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + bins = np.asarray([(7, 5, channel) for channel in range(3)], dtype=np.int32) + model = probe.discover_model(positives, color_space="ycocg", candidate_bins=bins, peak_count=3) + artifact = tmp_path / "model.npz" + + probe.save_model(artifact, model) + loaded = probe.load_model(artifact) + + assert loaded.color_space == "ycocg" + assert np.array_equal(loaded.channels, model.channels) + assert np.isclose(np.sum(loaded.weights), 1.0) + + +def test_channel_evidence_adds_to_total(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + bins = np.asarray([(7, 5, channel) for channel in range(3)], dtype=np.int32) + model = probe.discover_model(positives, color_space="rgb", candidate_bins=bins, peak_count=3) + + score = probe.score_image(positives[0], model) + + assert sum(score.channel_evidence) == pytest.approx(score.evidence_score) + assert sum(score.selected_peak_counts) == score.peak_count diff --git a/tests/test_synthid_confound_probe.py b/tests/test_synthid_confound_probe.py new file mode 100644 index 0000000..2ea5be3 --- /dev/null +++ b/tests/test_synthid_confound_probe.py @@ -0,0 +1,243 @@ +"""Tests for the manifest-driven SynthID D1 confound challenge.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image +from PIL.PngImagePlugin import PngInfo + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_confound_probe as probe +import synthid_research_manifest as manifest + + +def _write_image(path: Path, color: tuple[int, int, int], *, note: str | None = None) -> tuple[str, str]: + image = Image.new("RGB", (12, 10), color) + pnginfo = None + if note is not None: + pnginfo = PngInfo() + pnginfo.add_text("note", note) + image.save(path, pnginfo=pnginfo) + return hashlib.sha256(path.read_bytes()).hexdigest(), hashlib.sha256(image.tobytes()).hexdigest() + + +def _row( + artifact_sha: str, + pixel_sha: str, + artifact_path: str, + *, + group_id: str, + split: str, + outcome: str, + source_provider: str, + oracle_role: str = "ordinary", +) -> dict[str, str]: + matching_oracle = source_provider == "openai" + return { + "artifact_sha256": artifact_sha, + "pixel_sha256": pixel_sha, + "artifact_path": artifact_path, + "parent_sha256": "", + "group_id": group_id, + "target_provider": "openai", + "source_provider": source_provider, + "surface": "synthetic-test", + "model_epoch": "test-epoch", + "generation_session": f"session-{group_id}", + "content_stratum": "flat-graphic", + "width": "12", + "height": "10", + "format": "png", + "transform": "original", + "split": split, + "c2pa_outcome": "detected" if outcome == "detected" else "not_present", + "synthid_outcome": outcome, + "verified_via": "openai-api" if matching_oracle else "source-evidence", + "evidence_reference": "" if matching_oracle else f"https://example.test/{group_id}", + "oracle_session": f"oracle-{group_id}" if matching_oracle else "", + "oracle_role": oracle_role, + "captured_at": "2026-08-09T10:00:00Z", + "oracle_checked_at": "2026-08-09T10:05:00Z", + "notes": "synthetic confound fixture", + } + + +def _write_manifest(root: Path, rows: list[dict[str, str]]) -> Path: + path = root / "manifest.csv" + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=manifest.FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + return path + + +def _corpus(root: Path, *, include_temporal: bool = True) -> Path: + rows: list[dict[str, str]] = [] + index = 0 + split_sizes = {"train": 4, "validation": 2, "test": 2} + if include_temporal: + split_sizes["temporal"] = 2 + for split, per_class in split_sizes.items(): + for label in (1, 0): + for offset in range(per_class): + index += 1 + color = (230, 10 + index, 20 + offset) if label else (20 + offset, 10 + index, 230) + path = root / f"image-{index}.png" + artifact_sha, pixel_sha = _write_image(path, color) + source_provider = "openai" if label or offset == 0 else "camera" + rows.append( + _row( + artifact_sha, + pixel_sha, + path.name, + group_id=f"group-{index}", + split=split, + outcome="detected" if label else "not_detected", + source_provider=source_provider, + ) + ) + return _write_manifest(root, rows) + + +def test_canonical_features_ignore_container_metadata(tmp_path: Path): + first = tmp_path / "first.png" + second = tmp_path / "second.png" + first_sha, _ = _write_image(first, (10, 20, 30), note="short") + second_sha, _ = _write_image(second, (10, 20, 30), note="a much longer metadata value") + first_example = probe.Example(first, first_sha, "one", "train", 1, None) + second_example = probe.Example(second, second_sha, "two", "train", 1, None) + + assert not np.array_equal( + probe.extract_features(first_example, "container"), + probe.extract_features(second_example, "container"), + ) + np.testing.assert_array_equal( + probe.extract_features(first_example, "canonical"), + probe.extract_features(second_example, "canonical"), + ) + + +def test_feature_matrices_decode_each_artifact_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + first = tmp_path / "first.png" + second = tmp_path / "second.png" + first_sha, _ = _write_image(first, (10, 20, 30)) + second_sha, _ = _write_image(second, (30, 20, 10)) + examples = [ + probe.Example(first, first_sha, "one", "train", 1, None), + probe.Example(second, second_sha, "two", "train", 0, "external"), + ] + original = probe._decoded_rgb + decode_count = 0 + + def counting_decode(path: Path): + nonlocal decode_count + decode_count += 1 + return original(path) + + monkeypatch.setattr(probe, "_decoded_rgb", counting_decode) + + matrices = probe.feature_matrices(examples) + + assert decode_count == len(examples) + assert set(matrices) == set(probe.FEATURE_FAMILIES) + assert all(matrix.shape[0] == len(examples) for matrix in matrices.values()) + + +def test_logistic_model_separates_simple_content_confound(): + features = np.asarray([[0.0], [0.1], [0.9], [1.0]], dtype=np.float64) + labels = np.asarray([0, 0, 1, 1], dtype=np.int64) + + model = probe.fit_logistic(features, labels) + scores = probe.predict_scores(model, features) + + assert max(scores[:2]) < min(scores[2:]) + + +def test_threshold_respects_validation_false_positive_limit(): + labels = np.asarray([1, 1, 0, 0], dtype=np.int64) + scores = np.asarray([0.9, 0.8, 0.7, 0.1], dtype=np.float64) + + threshold = probe.select_threshold(labels, scores, max_fpr=0.0) + metrics = probe.calculate_metrics(labels, scores, threshold) + + assert threshold == pytest.approx(0.8) + assert metrics.tpr == 1.0 + assert metrics.fpr == 0.0 + + +def test_run_experiment_reports_same_provider_and_temporal_gate(tmp_path: Path): + path = _corpus(tmp_path) + + report = probe.run_experiment(path, "openai") + + assert report["evidence_ready"] is True + assert report["evidence_missing"] == [] + canonical = report["families"]["canonical"] + assert canonical["metrics"]["test"]["auc"] == 1.0 + assert canonical["negative_cohorts"]["test"]["same_provider"]["count"] == 1 + assert canonical["metrics"]["temporal"]["positives"] == 2 + assert canonical["metrics"]["temporal"]["negatives"] == 2 + assert str(tmp_path) not in json.dumps(report) + + +def test_report_stays_explicit_when_temporal_holdout_is_missing(tmp_path: Path): + path = _corpus(tmp_path, include_temporal=False) + + report = probe.run_experiment(path, "openai") + + assert report["evidence_ready"] is False + assert report["evidence_missing"] == ["temporal split does not contain both labels"] + assert report["families"]["container"]["metrics"]["temporal"] is None + + +def test_candidate_rows_are_not_detector_examples(tmp_path: Path): + path = _corpus(tmp_path) + rows = probe._read_manifest(path) + candidate_path = tmp_path / "candidate.png" + artifact_sha, pixel_sha = _write_image(candidate_path, (90, 90, 90)) + rows.append( + _row( + artifact_sha, + pixel_sha, + candidate_path.name, + group_id="candidate-group", + split="train", + outcome="detected", + source_provider="openai", + oracle_role="candidate", + ) + ) + _write_manifest(tmp_path, rows) + + examples = probe.load_examples(path, "openai") + + assert all(example.artifact_path.name != candidate_path.name for example in examples) + + +def test_manifest_audit_blocks_group_leakage_before_training(tmp_path: Path): + path = _corpus(tmp_path) + rows = probe._read_manifest(path) + rows[-1]["group_id"] = rows[0]["group_id"] + _write_manifest(tmp_path, rows) + + with pytest.raises(ValueError, match="manifest audit failed"): + probe.run_experiment(path, "openai") + + +def test_positive_source_provider_must_match_target(tmp_path: Path): + path = _corpus(tmp_path) + rows = probe._read_manifest(path) + positive = next(row for row in rows if row["synthid_outcome"] == "detected") + positive["source_provider"] = "google" + _write_manifest(tmp_path, rows) + + with pytest.raises(ValueError, match="declares source_provider"): + probe.load_examples(path, "openai") diff --git a/tests/test_synthid_consensus_probe.py b/tests/test_synthid_consensus_probe.py new file mode 100644 index 0000000..b763339 --- /dev/null +++ b/tests/test_synthid_consensus_probe.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_consensus_probe as probe + + +def _write_pattern(path: Path, base: tuple[int, int, int], *, sign: float, marked: bool) -> None: + size = 128 + yy, xx = np.mgrid[:size, :size] + carrier = 2.0 * np.sin(2.0 * np.pi * (11.0 * yy + 7.0 * xx) / size) + carrier += 1.5 * np.sin(2.0 * np.pi * (17.0 * yy - 5.0 * xx) / size + 0.4) + pixels = np.broadcast_to(np.asarray(base, dtype=np.float64), (size, size, 3)).copy() + if marked: + pixels += sign * carrier[:, :, None] + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def _reference_groups(tmp_path: Path) -> tuple[list[list[Path]], Path, Path]: + groups: list[list[Path]] = [] + for group_index, base in enumerate(((40, 50, 60), (170, 180, 190))): + directory = tmp_path / f"group-{group_index}" + directory.mkdir() + paths: list[Path] = [] + for image_index in range(3): + path = directory / f"marked-{image_index}.png" + sign = -1.0 if group_index == 1 else 1.0 + _write_pattern(path, base, sign=sign, marked=True) + paths.append(path) + groups.append(paths) + positive = tmp_path / "positive.png" + negative = tmp_path / "negative.png" + _write_pattern(positive, (100, 110, 120), sign=-1.0, marked=True) + _write_pattern(negative, (100, 110, 120), sign=1.0, marked=False) + return groups, positive, negative + + +def test_discovers_polarity_invariant_carrier(tmp_path: Path) -> None: + groups, positive, negative = _reference_groups(tmp_path) + + model = probe.discover_model(groups, size=128, peak_count=16, min_radius=3.0) + positive_score = probe.score_image(positive, model) + negative_score = probe.score_image(negative, model) + + assert positive_score.score > 0.8 + assert positive_score.active_weight_fraction > 0.5 + assert negative_score.active_weight_fraction < 0.01 + + +def test_model_round_trip_disables_pickle(tmp_path: Path) -> None: + groups, positive, _ = _reference_groups(tmp_path) + model = probe.discover_model(groups, size=128, peak_count=8, min_radius=3.0) + artifact = tmp_path / "model.npz" + + probe.save_model(artifact, model) + loaded = probe.load_model(artifact) + + assert probe.score_image(positive, loaded).score == pytest.approx(probe.score_image(positive, model).score) + assert loaded.peaks.dtype == np.int32 + + +def test_requires_independent_groups(tmp_path: Path) -> None: + groups, _, _ = _reference_groups(tmp_path) + + with pytest.raises(ValueError, match="at least two"): + probe.discover_model(groups[:1], size=128, peak_count=8) diff --git a/tests/test_synthid_ensemble_attack.py b/tests/test_synthid_ensemble_attack.py new file mode 100644 index 0000000..aec1f4b --- /dev/null +++ b/tests/test_synthid_ensemble_attack.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_color_space_probe as probe +import synthid_ensemble_attack as attack + + +def _write_image(path: Path, *, phase: float, seed: int) -> None: + height = width = 64 + rng = np.random.default_rng(seed) + pixels = 100.0 + rng.normal(0.0, 2.0, size=(height, width, 3)) + yy, xx = np.mgrid[:height, :width] + wave = np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + phase) + pixels[:, :, 0] += 14.0 * wave + pixels[:, :, 1] -= 10.0 * wave + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def test_projection_removes_only_positive_phase_component() -> None: + height = width = 64 + yy, xx = np.mgrid[:height, :width] + channel = np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + 0.4) + before = np.fft.fft2(channel)[7, 5] + phase = np.angle(before) + + projected = attack._remove_positive_projection( + channel, + rows=np.asarray([7]), + columns=np.asarray([5]), + phases=np.asarray([phase]), + strength=1.0, + ) + after = np.fft.fft2(projected)[7, 5] + + assert np.real(after * np.exp(-1j * phase)) == pytest.approx(0.0, abs=1e-10) + assert np.max(np.abs(np.imag(np.fft.ifft2(np.fft.fft2(projected))))) < 1e-12 + + +def test_alternating_projection_reduces_rgb_and_sv_evidence(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + source = tmp_path / "source.png" + _write_image(source, phase=0.4, seed=10) + bins = np.asarray([(7, 5, channel) for channel in range(3)], dtype=np.int32) + rgb_model = probe.discover_model(positives, color_space="rgb", candidate_bins=bins, peak_count=3) + hsv_model = probe.discover_model(positives, color_space="hsv", candidate_bins=bins, peak_count=3) + before_rgb = probe.score_image(source, rgb_model) + before_hsv = probe.score_image(source, hsv_model) + + with Image.open(source) as image: + source_pixels = np.asarray(image.convert("RGB"), dtype=np.uint8) + projected = attack.alternating_projection( + source_pixels, + rgb_model, + hsv_model, + strength=1.0, + iterations=2, + ) + output = tmp_path / "projected.png" + Image.fromarray(projected, mode="RGB").save(output) + after_rgb = probe.score_image(output, rgb_model) + after_hsv = probe.score_image(output, hsv_model) + + assert after_rgb.evidence_score < before_rgb.evidence_score + assert sum(after_hsv.channel_evidence[1:]) < sum(before_hsv.channel_evidence[1:]) + + +@pytest.mark.parametrize("strength", [-0.1, 1.1]) +def test_projection_rejects_out_of_range_strength(strength: float) -> None: + channel = np.zeros((64, 64)) + + with pytest.raises(ValueError, match="between zero and one"): + attack._remove_positive_projection( + channel, + rows=np.asarray([7]), + columns=np.asarray([5]), + phases=np.asarray([0.0]), + strength=strength, + ) diff --git a/tests/test_synthid_ensemble_detector.py b/tests/test_synthid_ensemble_detector.py new file mode 100644 index 0000000..a0fb333 --- /dev/null +++ b/tests/test_synthid_ensemble_detector.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_ensemble_detector as detector +from synthid_color_space_probe import ColorPhaseScore + + +def _config(tmp_path: Path) -> detector.EnsembleConfig: + return detector.EnsembleConfig( + width=64, + height=64, + rgb_model_path=tmp_path / "rgb.npz", + rgb_model_sha256="0" * 64, + rgb_evidence_threshold=0.3, + rgb_active_threshold=0.5, + hsv_model_path=tmp_path / "hsv.npz", + hsv_model_sha256="1" * 64, + hsv_sv_evidence_threshold=0.3, + hsv_active_threshold=0.5, + ) + + +def _score( + *, color_space: str, evidence: float, active: float, channels: tuple[float, float, float] +) -> ColorPhaseScore: + return ColorPhaseScore( + path="fixture.png", + color_space=color_space, + phase_score=0.8, + active_weight_fraction=active, + evidence_score=evidence, + channel_evidence=channels, + selected_peak_counts=(80, 88, 88), + peak_count=256, + ) + + +def test_positive_requires_both_branches_and_support(tmp_path: Path) -> None: + config = _config(tmp_path) + rgb = _score(color_space="rgb", evidence=0.4, active=0.7, channels=(0.1, 0.1, 0.2)) + hsv = _score(color_space="hsv", evidence=0.45, active=0.8, channels=(0.05, 0.2, 0.2)) + + verdict = detector.classify_scores(Path("fixture.png"), rgb, hsv, config) + + assert verdict.verdict == "positive" + assert verdict.reason == "ensemble_pass" + + +def test_low_support_abstains_even_when_scores_pass(tmp_path: Path) -> None: + config = _config(tmp_path) + rgb = _score(color_space="rgb", evidence=0.4, active=0.49, channels=(0.1, 0.1, 0.2)) + hsv = _score(color_space="hsv", evidence=0.45, active=0.8, channels=(0.05, 0.2, 0.2)) + + verdict = detector.classify_scores(Path("fixture.png"), rgb, hsv, config) + + assert verdict.verdict == "abstain" + assert verdict.reason == "insufficient_support" + + +def test_branch_disagreement_abstains(tmp_path: Path) -> None: + config = _config(tmp_path) + rgb = _score(color_space="rgb", evidence=0.4, active=0.8, channels=(0.1, 0.1, 0.2)) + hsv = _score(color_space="hsv", evidence=0.2, active=0.8, channels=(0.02, 0.1, 0.08)) + + verdict = detector.classify_scores(Path("fixture.png"), rgb, hsv, config) + + assert verdict.verdict == "abstain" + assert verdict.reason == "branch_disagreement" + + +def test_unsupported_geometry_abstains_without_scoring(tmp_path: Path) -> None: + config = _config(tmp_path) + image_path = tmp_path / "small.png" + Image.new("RGB", (32, 32)).save(image_path) + + verdict = detector.detect_image(image_path, config, None, None) # type: ignore[arg-type] + + assert verdict.verdict == "abstain" + assert verdict.reason == "unsupported_geometry" + assert verdict.rgb_evidence is None diff --git a/tests/test_synthid_fragment_attack.py b/tests/test_synthid_fragment_attack.py new file mode 100644 index 0000000..dacaa6e --- /dev/null +++ b/tests/test_synthid_fragment_attack.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_fragment_attack as attack + + +def _fixture() -> np.ndarray: + yy, xx = np.mgrid[:96, :128] + return ( + np.stack( + [ + 40.0 + 0.8 * xx, + 50.0 + 0.7 * yy, + 30.0 + 0.4 * xx + 0.3 * yy, + ], + axis=2, + ) + .clip(0, 255) + .astype(np.uint8) + ) + + +def test_affine_combo_preserves_geometry_and_is_deterministic() -> None: + source = _fixture() + + first = attack.affine_combo(source, rotation_degrees=0.2, zoom=0.004) + second = attack.affine_combo(source, rotation_degrees=0.2, zoom=0.004) + + assert first.shape == source.shape + assert first.dtype == np.uint8 + assert np.array_equal(first, second) + + +def test_bounded_smooth_warp_preserves_geometry_and_is_deterministic() -> None: + source = _fixture() + + first = attack.bounded_smooth_warp(source, max_displacement=1.8, sigma=8.0, seed=17) + second = attack.bounded_smooth_warp(source, max_displacement=1.8, sigma=8.0, seed=17) + + assert first.shape == source.shape + assert first.dtype == np.uint8 + assert np.array_equal(first, second) + assert not np.array_equal(first, source) + + +def test_zero_bounded_warp_is_pixel_identical() -> None: + source = _fixture() + + result = attack.bounded_smooth_warp(source, max_displacement=0.0, sigma=8.0, seed=17) + + assert np.array_equal(result, source) + + +def test_color_nudge_is_bounded_and_changes_pixels() -> None: + source = _fixture() + + result = attack.color_nudge( + source, + brightness=0.004, + contrast=0.006, + saturation=-0.005, + hue_degrees=0.15, + ) + + assert result.shape == source.shape + assert result.dtype == np.uint8 + assert not np.array_equal(result, source) + + +def test_jpeg_chain_rejects_bad_quality() -> None: + with pytest.raises(ValueError, match="quality"): + attack.jpeg_chain(_fixture(), (94, 101)) + + +def test_candidate_batch_has_control_target_and_sham() -> None: + candidates = attack.build_candidates(_fixture()) + + assert np.array_equal(candidates["control"], _fixture()) + assert "fragment-balanced" in candidates + assert "fragment-strong" in candidates + assert "sham-strong-rms" in candidates + assert "bounded-fragment-balanced" in candidates + assert "bounded-fragment-strong" in candidates + assert "sham-bounded-strong-rms" in candidates + assert all(pixels.shape == _fixture().shape for pixels in candidates.values()) diff --git a/tests/test_synthid_hybrid_attack.py b/tests/test_synthid_hybrid_attack.py new file mode 100644 index 0000000..d2db1e9 --- /dev/null +++ b/tests/test_synthid_hybrid_attack.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_color_space_probe as probe +import synthid_hybrid_attack as attack + + +def _write_image(path: Path, *, seed: int) -> None: + height = width = 64 + rng = np.random.default_rng(seed) + pixels = 100.0 + rng.normal(0.0, 2.0, size=(height, width, 3)) + yy, xx = np.mgrid[:height, :width] + wave = np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + 0.4) + pixels[:, :, 0] += 14.0 * wave + pixels[:, :, 1] -= 10.0 * wave + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def test_hybrid_matrix_preserves_geometry_and_has_controls(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, seed=index) + positives.append(path) + source_path = tmp_path / "source.png" + _write_image(source_path, seed=10) + with Image.open(source_path) as image: + source = np.asarray(image.convert("RGB"), dtype=np.uint8) + bins = np.asarray([(7, 5, channel) for channel in range(3)], dtype=np.int32) + rgb_model = probe.discover_model(positives, color_space="rgb", candidate_bins=bins, peak_count=3) + hsv_model = probe.discover_model(positives, color_space="hsv", candidate_bins=bins, peak_count=3) + + candidates = attack.build_candidates(source, rgb_model, hsv_model) + + assert set(candidates) == { + "projection-075", + "bounded-100", + "projection-075-bounded-100", + "projection-075-bounded-polish", + "projection-100-elastic-075", + } + assert all(candidate.shape == source.shape for candidate in candidates.values()) + assert np.array_equal(candidates["projection-075"], source) is False diff --git a/tests/test_synthid_phase_carrier.py b/tests/test_synthid_phase_carrier.py new file mode 100644 index 0000000..62ce0fe --- /dev/null +++ b/tests/test_synthid_phase_carrier.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_phase_carrier as carrier + + +def _write_image(path: Path, *, phase: float | None, seed: int) -> None: + height = width = 64 + rng = np.random.default_rng(seed) + pixels = 100.0 + rng.normal(0.0, 3.0, size=(height, width, 3)) + if phase is not None: + yy, xx = np.mgrid[:height, :width] + wave = 12.0 * np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + phase) + pixels += wave[:, :, None] + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def test_discovered_model_separates_shared_phase_from_noise(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(4): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + heldout = tmp_path / "heldout.png" + negative = tmp_path / "negative.png" + _write_image(heldout, phase=0.4, seed=10) + _write_image(negative, phase=None, seed=11) + + model = carrier.discover_model(positives, peak_count=8, min_radius=1.0) + + assert carrier.score_image(heldout, model).score > carrier.score_image(negative, model).score + + +def test_leave_one_out_coherence_exposes_phase_outlier() -> None: + units = np.asarray([1.0 + 0.0j, 1.0 + 0.0j, 1.0 + 0.0j, -1.0 + 0.0j]) + unit_sum = np.sum(units) + + coherences = [carrier._leave_one_out_coherence(unit_sum, unit, 4.0) for unit in units] + + assert min(coherences) == pytest.approx(1.0 / 3.0) + assert max(coherences) == pytest.approx(1.0) + + +def test_model_round_trip_is_pickle_free(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + model = carrier.discover_model(positives, peak_count=4, min_radius=1.0) + artifact = tmp_path / "model.npz" + + carrier.save_model(artifact, model) + loaded = carrier.load_model(artifact) + + assert loaded.height == model.height + assert loaded.width == model.width + assert np.array_equal(loaded.rows, model.rows) + assert np.isclose(np.sum(loaded.weights), 1.0) + + +def test_candidate_bins_restrict_discovery(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + bins = np.asarray([[7, 5, 0], [7, 5, 1], [7, 5, 2]], dtype=np.int32) + + model = carrier.discover_model(positives, peak_count=3, min_radius=1.0, candidate_bins=bins) + + actual = set(zip(model.rows.tolist(), model.columns.tolist(), model.channels.tolist(), strict=True)) + expected = set(map(tuple, bins.tolist())) + assert actual == expected + + +def test_discovery_requires_three_images(tmp_path: Path) -> None: + path = tmp_path / "positive.png" + _write_image(path, phase=0.4, seed=1) + + with pytest.raises(ValueError, match="at least three"): + carrier.discover_model([path, path], peak_count=4) + + +def test_scoring_rejects_geometry_mismatch(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + model = carrier.discover_model(positives, peak_count=4, min_radius=1.0) + mismatch = tmp_path / "mismatch.png" + Image.new("RGB", (80, 64)).save(mismatch) + + with pytest.raises(ValueError, match="does not match"): + carrier.score_image(mismatch, model) + + +def test_scoring_can_canonicalize_geometry(tmp_path: Path) -> None: + positives: list[Path] = [] + for index in range(3): + path = tmp_path / f"positive-{index}.png" + _write_image(path, phase=0.4, seed=index) + positives.append(path) + model = carrier.discover_model(positives, peak_count=4, min_radius=1.0) + mismatch = tmp_path / "mismatch.png" + Image.new("RGB", (80, 64), color=(100, 100, 100)).save(mismatch) + + score = carrier.score_image(mismatch, model, canonicalize_geometry=True) + + assert score.path == str(mismatch) + assert score.peak_count == 4 diff --git a/tests/test_synthid_pixel_attack.py b/tests/test_synthid_pixel_attack.py new file mode 100644 index 0000000..82c7676 --- /dev/null +++ b/tests/test_synthid_pixel_attack.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_pixel_attack as attack + + +def _fixture() -> np.ndarray: + yy, xx = np.mgrid[:128, :160] + channels = [ + 80 + xx * 0.4 + yy * 0.2, + 60 + xx * 0.3 + yy * 0.5, + 40 + xx * 0.6 + yy * 0.1, + ] + return np.clip(np.stack(channels, axis=2), 0, 255).astype(np.uint8) + + +def test_quantization_is_bounded_and_deterministic() -> None: + pixels = _fixture() + + first = attack.quantize(pixels, 4) + second = attack.quantize(pixels, 4) + + assert np.array_equal(first, second) + assert np.max(np.abs(first.astype(int) - pixels.astype(int))) <= 2 + + +def test_smooth_warp_preserves_geometry_and_is_deterministic() -> None: + pixels = _fixture() + + first = attack.smooth_warp(pixels, amplitude=0.35, sigma=8.0, seed=17) + second = attack.smooth_warp(pixels, amplitude=0.35, sigma=8.0, seed=17) + + assert first.shape == pixels.shape + assert first.dtype == np.uint8 + assert np.array_equal(first, second) + assert not np.array_equal(first, pixels) + + +def test_norm_matched_control_has_similar_rms(tmp_path: Path) -> None: + pixels = _fixture() + target = attack.quantize(pixels, 8) + + sham = attack.norm_matched_noise(pixels, target, seed=23) + target_metrics = attack.measure(pixels, target, name="target", path=tmp_path / "target.png") + sham_metrics = attack.measure(pixels, sham, name="sham", path=tmp_path / "sham.png") + + assert sham_metrics.residual_rms == pytest.approx(target_metrics.residual_rms, rel=0.1) + + +def test_crop_visible_badge() -> None: + pixels = _fixture() + + cropped = attack.crop_visible_badge(pixels, 16) + + assert cropped.shape == (112, 144, 3) diff --git a/tests/test_synthid_research_inventory.py b/tests/test_synthid_research_inventory.py new file mode 100644 index 0000000..86b1231 --- /dev/null +++ b/tests/test_synthid_research_inventory.py @@ -0,0 +1,107 @@ +"""Tests for label-free local SynthID research inventories.""" + +from __future__ import annotations + +import csv +import sys +from pathlib import Path + +import pytest +from PIL import Image +from PIL.PngImagePlugin import PngInfo + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_research_inventory as inventory + + +def _write_png(path: Path, color: tuple[int, int, int], *, note: str | None = None) -> None: + image = Image.new("RGB", (9, 7), color) + pnginfo = None + if note is not None: + pnginfo = PngInfo() + pnginfo.add_text("note", note) + image.save(path, format="PNG", pnginfo=pnginfo) + + +def test_inventory_is_stable_and_contains_no_evidence_labels(tmp_path: Path): + media = tmp_path / "media" + media.mkdir() + _write_png(media / "b.png", (20, 30, 40)) + _write_png(media / "a.png", (10, 20, 30)) + + rows = inventory.build_inventory(tmp_path, (Path("media"),)) + + assert [row.artifact_path for row in rows] == ["media/a.png", "media/b.png"] + assert set(inventory.FIELDNAMES).isdisjoint({"target_provider", "synthid_outcome", "verified_via", "split"}) + assert all(row.format == "png" and row.width == 9 and row.height == 7 for row in rows) + + +def test_inventory_marks_byte_and_pixel_duplicates(tmp_path: Path): + media = tmp_path / "media" + media.mkdir() + first = media / "first.png" + exact = media / "exact.png" + metadata_variant = media / "metadata.png" + _write_png(first, (10, 20, 30), note="one") + exact.write_bytes(first.read_bytes()) + _write_png(metadata_variant, (10, 20, 30), note="different metadata") + + rows = {row.artifact_path: row for row in inventory.build_inventory(tmp_path, (Path("media"),))} + + assert rows["media/exact.png"].artifact_duplicate_of == "" + assert rows["media/first.png"].artifact_duplicate_of == "media/exact.png" + assert rows["media/first.png"].pixel_duplicate_of == "media/exact.png" + assert rows["media/metadata.png"].artifact_duplicate_of == "" + assert rows["media/metadata.png"].pixel_duplicate_of == "media/exact.png" + assert len({row.exact_pixel_group for row in rows.values()}) == 1 + + +def test_discovery_rejects_source_outside_root(tmp_path: Path): + root = tmp_path / "root" + root.mkdir() + outside = tmp_path / "outside.png" + _write_png(outside, (10, 20, 30)) + + with pytest.raises(ValueError, match="outside inventory root"): + inventory.discover_images(root, (outside,)) + + +def test_inventory_uses_content_format_not_suffix(tmp_path: Path): + disguised = tmp_path / "disguised.jpg" + _write_png(disguised, (10, 20, 30)) + + row = inventory.build_inventory(tmp_path, (Path("disguised.jpg"),))[0] + + assert row.format == "png" + + +def test_write_refuses_to_replace_without_explicit_flag(tmp_path: Path): + media = tmp_path / "image.png" + _write_png(media, (10, 20, 30)) + rows = inventory.build_inventory(tmp_path, (Path("image.png"),)) + output = tmp_path / "inventory.csv" + inventory.write_inventory(output, rows) + + with pytest.raises(FileExistsError, match="--replace"): + inventory.write_inventory(output, rows) + + inventory.write_inventory(output, rows, replace=True) + with output.open(newline="", encoding="utf-8") as stream: + written = list(csv.DictReader(stream)) + assert written[0]["artifact_path"] == "image.png" + + +def test_summary_reports_only_aggregates(tmp_path: Path): + _write_png(tmp_path / "image.png", (10, 20, 30)) + rows = inventory.build_inventory(tmp_path, (Path("image.png"),)) + + summary = inventory.inventory_summary(rows) + + assert summary == { + "images": 1, + "unique_artifacts": 1, + "unique_pixels": 1, + "formats": {"png": 1}, + "geometries": {"9x7": 1}, + } diff --git a/tests/test_synthid_research_manifest.py b/tests/test_synthid_research_manifest.py new file mode 100644 index 0000000..706609d --- /dev/null +++ b/tests/test_synthid_research_manifest.py @@ -0,0 +1,252 @@ +"""Tests for the private SynthID research-manifest auditor.""" + +from __future__ import annotations + +import csv +import hashlib +import sys +from pathlib import Path + +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_research_manifest as manifest + + +def _write_image(path: Path, color: tuple[int, int, int]) -> tuple[str, str]: + image = Image.new("RGB", (8, 6), color) + image.save(path) + artifact_sha = hashlib.sha256(path.read_bytes()).hexdigest() + pixel_sha = hashlib.sha256(image.tobytes()).hexdigest() + return artifact_sha, pixel_sha + + +def _row(artifact_sha: str, pixel_sha: str, *, path: str = "image.png", **updates: str) -> dict[str, str]: + row = { + "artifact_sha256": artifact_sha, + "pixel_sha256": pixel_sha, + "artifact_path": path, + "parent_sha256": "", + "group_id": "group-1", + "target_provider": "openai", + "source_provider": "openai", + "surface": "api", + "model_epoch": "gpt-image-2026-08", + "generation_session": "session-1", + "content_stratum": "flat-graphic", + "width": "8", + "height": "6", + "format": "png", + "transform": "original", + "split": "train", + "c2pa_outcome": "detected", + "synthid_outcome": "detected", + "verified_via": "openai-api", + "evidence_reference": "", + "oracle_session": "oracle-1", + "oracle_role": "ordinary", + "captured_at": "2026-08-08T12:00:00Z", + "oracle_checked_at": "2026-08-08T12:05:00Z", + "notes": "synthetic test row", + } + row.update(updates) + return row + + +def _write_manifest(path: Path, rows: list[dict[str, str]]) -> None: + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=manifest.FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + +def test_valid_manifest_and_files_pass(tmp_path: Path): + artifact = tmp_path / "image.png" + artifact_sha, pixel_sha = _write_image(artifact, (10, 20, 30)) + path = tmp_path / "manifest.csv" + _write_manifest(path, [_row(artifact_sha, pixel_sha)]) + + assert manifest.audit_manifest(path, verify_files=True) == [] + + +def test_rejects_same_provider_negative_without_matching_oracle(tmp_path: Path): + artifact_sha = "a" * 64 + pixel_sha = "b" * 64 + path = tmp_path / "manifest.csv" + row = _row( + artifact_sha, + pixel_sha, + synthid_outcome="not_detected", + verified_via="source-evidence", + oracle_session="", + ) + _write_manifest(path, [row]) + + errors = manifest.audit_manifest(path) + + assert any("same-provider negative" in error for error in errors) + + +def test_rejects_external_source_evidence_without_reference(tmp_path: Path): + path = tmp_path / "manifest.csv" + row = _row( + "a" * 64, + "b" * 64, + source_provider="camera", + synthid_outcome="not_detected", + verified_via="source-evidence", + oracle_session="", + c2pa_outcome="not_present", + ) + _write_manifest(path, [row]) + + errors = manifest.audit_manifest(path) + + assert any("source-evidence requires evidence_reference" in error for error in errors) + + +def test_accepts_external_source_evidence_with_reference(tmp_path: Path): + path = tmp_path / "manifest.csv" + row = _row( + "a" * 64, + "b" * 64, + source_provider="camera", + synthid_outcome="not_detected", + verified_via="source-evidence", + evidence_reference="https://example.test/original-record", + oracle_session="", + c2pa_outcome="not_present", + ) + _write_manifest(path, [row]) + + assert manifest.audit_manifest(path) == [] + + +def test_rejects_indeterminate_training_label(tmp_path: Path): + path = tmp_path / "manifest.csv" + row = _row( + "a" * 64, + "b" * 64, + synthid_outcome="indeterminate", + verified_via="openai-api", + oracle_checked_at="", + ) + _write_manifest(path, [row]) + + errors = manifest.audit_manifest(path) + + assert any("requires a detected or not_detected" in error for error in errors) + + +def test_rejects_group_leakage_across_splits(tmp_path: Path): + first = _row("a" * 64, "b" * 64) + second = _row( + "c" * 64, + "d" * 64, + artifact_path="other.png", + split="test", + generation_session="session-2", + ) + path = tmp_path / "manifest.csv" + _write_manifest(path, [first, second]) + + errors = manifest.audit_manifest(path) + + assert any("leaks across splits" in error for error in errors) + + +def test_rejects_derivative_with_missing_parent(tmp_path: Path): + path = tmp_path / "manifest.csv" + row = _row("a" * 64, "b" * 64, transform="jpeg-q90", parent_sha256="c" * 64) + _write_manifest(path, [row]) + + errors = manifest.audit_manifest(path) + + assert any("parent_sha256 is not present" in error for error in errors) + + +def test_rejects_identical_pixels_in_different_groups(tmp_path: Path): + first = _row("a" * 64, "b" * 64) + second = _row( + "c" * 64, + "b" * 64, + artifact_path="other.png", + group_id="group-2", + generation_session="session-2", + ) + path = tmp_path / "manifest.csv" + _write_manifest(path, [first, second]) + + errors = manifest.audit_manifest(path) + + assert any("appears in multiple groups" in error for error in errors) + + +def test_verify_files_detects_changed_artifact(tmp_path: Path): + artifact = tmp_path / "image.png" + artifact_sha, pixel_sha = _write_image(artifact, (10, 20, 30)) + path = tmp_path / "manifest.csv" + _write_manifest(path, [_row(artifact_sha, pixel_sha)]) + _write_image(artifact, (11, 21, 31)) + + errors = manifest.audit_manifest(path, verify_files=True) + + assert any("artifact_sha256 does not match" in error for error in errors) + assert any("pixel_sha256 does not match" in error for error in errors) + + +def test_rejects_path_traversal_during_file_verification(tmp_path: Path): + path = tmp_path / "manifest.csv" + _write_manifest(path, [_row("a" * 64, "b" * 64, path="../outside.png")]) + + errors = manifest.audit_manifest(path, verify_files=True) + + assert any("safe manifest-relative path" in error for error in errors) + + +def test_accepts_candidate_negative_with_detected_session_control(tmp_path: Path): + control = _row("a" * 64, "b" * 64, oracle_role="source_control") + candidate = _row( + "c" * 64, + "d" * 64, + artifact_path="candidate.png", + parent_sha256="a" * 64, + transform="carrier-subtract", + synthid_outcome="not_detected", + oracle_role="candidate", + ) + path = tmp_path / "manifest.csv" + _write_manifest(path, [control, candidate]) + + assert manifest.audit_manifest(path) == [] + + +def test_rejects_candidate_negative_without_detected_session_control(tmp_path: Path): + candidate = _row( + "a" * 64, + "b" * 64, + synthid_outcome="not_detected", + oracle_role="candidate", + ) + path = tmp_path / "manifest.csv" + _write_manifest(path, [candidate]) + + errors = manifest.audit_manifest(path) + + assert any("requires a detected source_control" in error for error in errors) + + +def test_rejects_failed_source_control(tmp_path: Path): + control = _row( + "a" * 64, + "b" * 64, + synthid_outcome="not_detected", + oracle_role="source_control", + ) + path = tmp_path / "manifest.csv" + _write_manifest(path, [control]) + + errors = manifest.audit_manifest(path) + + assert any("source_control must have a detected" in error for error in errors) diff --git a/tests/test_synthid_spectral_probe.py b/tests/test_synthid_spectral_probe.py new file mode 100644 index 0000000..c9ce250 --- /dev/null +++ b/tests/test_synthid_spectral_probe.py @@ -0,0 +1,103 @@ +"""Synthetic tests for the paired SynthID spectral research harness.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_spectral_probe as probe + + +def _carrier(height: int, width: int) -> np.ndarray: + yy, xx = np.mgrid[:height, :width] + channels = [ + np.cos(2 * np.pi * (14 * xx / width + 14 * yy / height)), + 1.3 * np.cos(2 * np.pi * (14 * xx / width + 14 * yy / height) + 0.2), + 0.8 * np.cos(2 * np.pi * (14 * xx / width + 14 * yy / height) - 0.3), + ] + return np.stack(channels, axis=2) + + +def _write_pair(tmp_path: Path, name: str, shape: tuple[int, int], base: int) -> tuple[Path, Path]: + height, width = shape + yy, xx = np.mgrid[:height, :width] + scene = base + 8 * np.sin(2 * np.pi * xx / width) + 5 * np.cos(2 * np.pi * yy / height) + clean = np.stack([scene, scene + 3, scene - 3], axis=2) + marked = clean + _carrier(height, width) + clean_path = tmp_path / f"{name}-clean.png" + marked_path = tmp_path / f"{name}-marked.png" + Image.fromarray(np.clip(np.rint(clean), 0, 255).astype(np.uint8)).save(clean_path) + Image.fromarray(np.clip(np.rint(marked), 0, 255).astype(np.uint8)).save(marked_path) + return clean_path, marked_path + + +def test_pair_residual_preserves_float_signal_across_shapes(tmp_path: Path): + first = _write_pair(tmp_path, "first", (96, 96), 100) + second = _write_pair(tmp_path, "second", (128, 80), 140) + + first_residual, first_measurement = probe.pair_residual(*first, size=64) + second_residual, second_measurement = probe.pair_residual(*second, size=64) + + assert first_residual.shape == (64, 64, 3) + assert first_measurement.changed_pixel_fraction > 0.2 + assert second_measurement.width == 80 + assert min(probe.channel_ncc(first_residual, second_residual)) > 0.75 + + +def test_selected_peaks_find_injected_frequency(tmp_path: Path): + pair = _write_pair(tmp_path, "pair", (96, 96), 100) + residual, _ = probe.pair_residual(*pair, size=64) + template = probe.build_template([residual]) + + peaks = probe.select_peaks(template, count=4) + + assert any(abs(int(dy)) == 14 and abs(int(dx)) == 14 for dy, dx in peaks) + + +def test_marked_image_scores_above_clean_image(tmp_path: Path): + pair = _write_pair(tmp_path, "pair", (128, 128), 100) + residual, _ = probe.pair_residual(*pair, size=128) + template = probe.build_template([residual]) + peaks = probe.select_peaks(template, count=8) + + clean_score = probe.score_image(pair[0], template, peaks) + marked_score = probe.score_image(pair[1], template, peaks) + + assert marked_score.phase_weighted > clean_score.phase_weighted + assert marked_score.top_two_channel_phase_12 > clean_score.top_two_channel_phase_12 + + +def test_template_round_trip_does_not_use_pickle(tmp_path: Path): + template = np.zeros((64, 64, 3), dtype=np.float64) + template[1, 1] = (1.0, 2.0, 3.0) + peaks = np.asarray([[1, 1], [2, 2]], dtype=np.int32) + path = tmp_path / "template.npz" + + probe.save_template(path, template, peaks) + loaded_template, loaded_peaks = probe.load_template(path) + + assert np.array_equal(loaded_template, template) + assert np.array_equal(loaded_peaks, peaks) + + +def test_discovery_report_contains_cross_pair_ncc(tmp_path: Path): + first = _write_pair(tmp_path, "first", (96, 96), 100) + second = _write_pair(tmp_path, "second", (128, 80), 140) + first_residual, first_measurement = probe.pair_residual(*first, size=64) + second_residual, second_measurement = probe.pair_residual(*second, size=64) + template = probe.build_template([first_residual, second_residual]) + peaks = probe.select_peaks(template, count=4) + + report = probe.discovery_report( + [first_residual, second_residual], + [first_measurement, second_measurement], + peaks, + ) + + assert report["pair_count"] == 2 + assert len(report["pairwise"]) == 1 diff --git a/tests/test_synthid_tile_attack.py b/tests/test_synthid_tile_attack.py new file mode 100644 index 0000000..29380ad --- /dev/null +++ b/tests/test_synthid_tile_attack.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_tile_attack as attack + + +def test_modulo_folding_recovers_repeated_high_frequency_tile() -> None: + tile = np.fromfunction(lambda y, x, channel: ((x + y + channel) % 2) * 2.0 - 1.0, (8, 16, 3)) + pixels = 100.0 + np.tile(tile, (8, 4, 1)) + + estimated = attack.fold_residual_template( + pixels, + tile_height=8, + tile_width=16, + denoise_sigma=1.0, + ) + + correlation = np.corrcoef(tile.ravel(), estimated.ravel())[0, 1] + assert correlation > 0.99 + + +def test_subtraction_reduces_repeated_tile_energy() -> None: + tile = np.fromfunction(lambda y, x, channel: ((x + y + channel) % 2) * 2.0 - 1.0, (8, 16, 3)) + pixels = np.clip(np.rint(100.0 + 4.0 * np.tile(tile, (8, 4, 1))), 0, 255).astype(np.uint8) + template = attack.fold_residual_template( + pixels, + tile_height=8, + tile_width=16, + denoise_sigma=1.0, + ) + + result = attack.subtract_tiled_template(pixels, template, strength=1.0) + before = np.std(pixels.astype(np.float64) - np.mean(pixels, axis=(0, 1), keepdims=True)) + after = np.std(result.astype(np.float64) - np.mean(result, axis=(0, 1), keepdims=True)) + + assert after < before + + +def test_folding_rejects_nondivisible_geometry() -> None: + pixels = np.zeros((63, 64, 3), dtype=np.uint8) + + with pytest.raises(ValueError, match="divisible"): + attack.fold_residual_template( + pixels, + tile_height=8, + tile_width=16, + denoise_sigma=1.0, + ) diff --git a/tests/test_synthid_v3_carrier_subtract.py b/tests/test_synthid_v3_carrier_subtract.py new file mode 100644 index 0000000..f289191 --- /dev/null +++ b/tests/test_synthid_v3_carrier_subtract.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import click +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_v3_carrier_subtract as subtract +from synthid_v3_codebook_probe import V3CarrierModel + + +def _model() -> V3CarrierModel: + height = width = 64 + return V3CarrierModel( + height=height, + width=width, + rows=np.asarray([7, 7, 7], dtype=np.int32), + columns=np.asarray([5, 5, 5], dtype=np.int32), + channels=np.asarray([0, 1, 2], dtype=np.int8), + phases=np.asarray([0.4, 0.4, 0.4]), + weights=np.full(3, 1.0 / 3.0), + expected_magnitudes=np.full(3, 20.0 * height * width / 2.0), + ) + + +def _carrier(model: V3CarrierModel) -> np.ndarray: + yy, xx = np.mgrid[: model.height, : model.width] + values = 100.0 + 20.0 * np.cos(2.0 * np.pi * (7.0 * yy / model.height + 5.0 * xx / model.width) + 0.4) + return np.repeat(values[:, :, None], 3, axis=2).round().astype(np.uint8) + + +def test_subtract_carrier_preserves_shape_and_removes_known_component() -> None: + model = _model() + source = _carrier(model) + + result = subtract.subtract_carrier(source, model, strength=1.0) + + assert result.shape == source.shape + assert result.dtype == np.uint8 + assert np.std(result.astype(np.float64)) < 1.0 + + +def test_zero_strength_is_pixel_identical() -> None: + model = _model() + source = _carrier(model) + + result = subtract.subtract_carrier(source, model, strength=0.0) + + assert np.array_equal(result, source) + + +def test_rejects_wrong_geometry() -> None: + model = _model() + + with pytest.raises(ValueError, match="does not match"): + subtract.subtract_carrier(np.zeros((32, 32, 3), dtype=np.uint8), model, strength=1.0) + + +@pytest.mark.parametrize("value", ["1,0.5", "0.5,0.5", "-1,0.5", "x"]) +def test_rejects_invalid_strength_sweep(value: str) -> None: + with pytest.raises(click.BadParameter): + subtract.parse_strengths(value) diff --git a/tests/test_synthid_v3_codebook_probe.py b/tests/test_synthid_v3_codebook_probe.py new file mode 100644 index 0000000..6c5df84 --- /dev/null +++ b/tests/test_synthid_v3_codebook_probe.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import synthid_v3_codebook_probe as probe + + +def _write_codebook(path: Path, *, height: int, width: int, phase: float) -> None: + half_width = width // 2 + 1 + rows = np.asarray([7, 11, 13, 17], dtype=np.uint32) + columns = np.asarray([5, 9, 12, 15], dtype=np.uint32) + indices = np.ravel_multi_index((rows, columns), (height, half_width)).astype(np.uint32) + payload: dict[str, np.ndarray] = { + "format_version": np.asarray(2), + f"{height}x{width}/sparse": np.asarray(1), + } + for channel in range(3): + payload[f"{height}x{width}/idx_{channel}"] = indices + magnitudes = np.asarray([1000.0, 10.0, 10.0, 10.0]) + payload[f"{height}x{width}/mag_{channel}"] = np.log2(1.0 + magnitudes).astype(np.float16) + payload[f"{height}x{width}/phase_{channel}"] = np.full(4, phase, dtype=np.float16) + payload[f"{height}x{width}/cons_{channel}"] = np.full(4, 255, dtype=np.uint8) + np.savez(path, **payload) + + +def _write_carrier(path: Path, *, height: int, width: int, phase: float) -> None: + yy, xx = np.mgrid[:height, :width] + carrier = 80.0 + 20.0 * np.cos(2.0 * np.pi * (7.0 * yy / height + 5.0 * xx / width) + phase) + pixels = np.repeat(carrier[:, :, None], 3, axis=2) + Image.fromarray(np.clip(np.rint(pixels), 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def test_numeric_codebook_scores_matching_phase(tmp_path: Path) -> None: + height = width = 64 + codebook = tmp_path / "codebook.npz" + image = tmp_path / "image.png" + _write_codebook(codebook, height=height, width=width, phase=0.4) + _write_carrier(image, height=height, width=width, phase=0.4) + + model = probe.load_v3_model(codebook, height=height, width=width, peak_count=4, min_radius=1.0) + score = probe.score_image(image, model) + + assert score.peak_count == 4 + assert score.phase_score > 0.0 + + +def test_rejects_wrong_format(tmp_path: Path) -> None: + artifact = tmp_path / "bad.npz" + np.savez(artifact, format_version=np.asarray(1)) + + with pytest.raises(ValueError, match="version 2"): + probe.load_v3_model(artifact, height=64, width=64)