From c150180acf08a629211b13662dc493b778139a64 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Mon, 20 Jul 2026 08:14:50 -0700 Subject: [PATCH] Add full-surface verification harnesses and corpus sweep plan Co-Authored-By: Claude Opus 4.8 --- docs/verification-plan.md | 496 +++++++++++++++++++++++++++ scripts/fill_quality.py | 323 ++++++++++++++++++ scripts/pill_gate_audit.py | 224 ++++++++++++ scripts/sidecar_regression.py | 216 ++++++++++++ scripts/smoke_matrix.py | 621 ++++++++++++++++++++++++++++++++++ scripts/visible_positives.py | 141 ++++++++ 6 files changed, 2021 insertions(+) create mode 100644 docs/verification-plan.md create mode 100644 scripts/fill_quality.py create mode 100644 scripts/pill_gate_audit.py create mode 100644 scripts/sidecar_regression.py create mode 100644 scripts/smoke_matrix.py create mode 100644 scripts/visible_positives.py diff --git a/docs/verification-plan.md b/docs/verification-plan.md new file mode 100644 index 0000000..acab963 --- /dev/null +++ b/docs/verification-plan.md @@ -0,0 +1,496 @@ +# Full verification plan + +How we convince ourselves the library actually works, across its whole surface, on real data. + +This is the pre-release and periodic-audit plan. It is deliberately organized by **oracle +strength** rather than by module, because the hard part is never "call the function" -- it +is "know what the right answer was". A sweep with no oracle proves only that nothing threw. + +Measured throughput on the local corpus (39,430 images, M-series, 2026-07-19): + +| path | per image | full corpus, 8 procs | +|---|---|---| +| `identify` | 0.58 s | ~0.8 h | +| `detect_marks` | 0.28 s | ~0.4 h | +| visible remove (cv2) | 0.58 s | ~0.8 h | +| diffusion @512px (MPS) | ~50 s | ~23 days -- sample only | + +So every CPU path is affordable at FULL corpus scale; only the diffusion paths need +sampling. Plan accordingly: never sample where a full sweep costs an hour. + +## Data sources + +| source | size | committed | role | +|---|---|---|---| +| `data/spaces/originals/` | 39,430 imgs, 87.5 GB | no (gitignored) | the real-upload corpus | +| `data/spaces/identify/` | 39,314 JSON sidecars | no | **recorded `identify` verdict per image** | +| `data/spaces/_visible_datasets/` | 3,741 imgs, per vendor | no | mark-positive pools | +| `data/synthid_corpus/` | 39 imgs, labelled | yes | pos/neg/cleaned SynthID references | +| `data/samples/` | 11 fixtures | yes | deterministic fixtures | +| `data/*_capture/` | vendor captures | yes | detection silhouettes | +| synthesized | generated | no | constructed ground truth (tier B) | + +**Data safety.** The corpus is user uploads: local analysis only. No run may copy, promote, +or commit corpus images into a tracked path, and no report may embed them. All harness +output goes to gitignored paths under `data/spaces/`. + +## Tier A -- self-evident oracles (full corpus, unattended) + +Properties that are true or false without anyone labelling anything. These are the +backbone: they scale to 39k images and catch regressions with zero human cost. + +### A1. Sidecar regression -- the highest-value check we are not running + +`data/spaces/identify/` holds 39,314 recorded `identify` verdicts, keyed by the same uid as +the image. Re-running `identify` today and diffing against them turns the corpus into a +**39k-image behavioral regression suite for free**. Any drift in verdict, platform, +confidence or signal set shows up as a diff, bucketed by cause. + +Caveat that makes this honest: a diff is not automatically a bug -- the sidecars were +written by older versions, so intended improvements also show up. The output is therefore a +**classified diff** (new detections / lost detections / changed platform / changed +confidence), reviewed once, then re-baselined. Lost detections are the alarm. + +Implemented as `scripts/sidecar_regression.py` (resumable, ~1.5 h at 8 workers). + +#### First full run, 2026-07-19, all 39,314 sidecars + +| class | n | share | +|---|---|---| +| unchanged | 37,326 | 94.9% | +| lost_signal | 1,302 | 3.3% | +| platform changed | 922 | 2.3% | +| confidence changed | 899 | 2.3% | +| new_signal | 694 | 1.8% | +| lost_ai | 747 | 1.9% | +| new_ai | 152 | 0.4% | + +Two results worth keeping: + +**No metadata signal regressed anywhere.** Lost families were exclusively visual +(`visible_sparkle` 1,256, `visible_doubao` 45, `visible_jimeng` 4) -- zero c2pa, synthid, +aigc_tc260, iptc, exif_generator or xai_signature losses across the whole corpus. And +`identify` raised on **none** of the 39,314 real uploads. + +**The sparkle losses are mostly corrected false fires, but not entirely.** Sampling 400 of +the 1,256 and checking whether the file still carries Google provenance independently of +the sparkle: 12.5% (95% CI 9.6-16.1%) still do, i.e. ~120-200 corpus-wide are **genuine +misses**; the other ~1,050-1,135 had nothing backing them. Read the split as a trade the +FP-gate tightening made, not as a clean win. + +Caveat on that split: "no Google provenance" is not proof of a false positive -- a +metadata-stripped Gemini screenshot also has none while still carrying the pixel sparkle. +So 87.5% is an **upper bound** on false fires; only the 12.5% genuine-miss figure is solid. + +Doubao moved the other way (-45 / +628 net +583), which is the `scale_basis` landscape fix +showing up at corpus scale. `trustmark` +38 and `open_invisible` +9 are not behavior: those +extras were simply not installed when the sidecars were written. + +### A2. Parity: whatever we detect, we must be able to remove + +For every image where a signal fires: remove, re-scan with the same oracle, assert quiet. +- metadata: `scripts/metadata_removal_audit.py` (exists) -- run full corpus. +- visible: `scripts/visible_removal_audit.py` (exists) -- run **once per backend** + (cv2 / migan / lama). It is single-process, so a full-corpus sweep is ~10 h and three + backends ~30 h. Its expensive half is DETECTION, which does not depend on the backend, + so run `scripts/visible_positives.py` once (parallel, ~40 min) and feed the result to + the audit's `--paths-file` seam: a few thousand images per backend instead of 39k. + +#### Metadata parity, first full run, 2026-07-19 (20,153 carriers + 1,500 clean controls) + +Zero scan/strip/decode errors. Survival after strip: + +| signal | carriers | survived | +|---|---|---| +| c2pa_manifest / claim_generator | 15,410 | **3** | +| synthid_watermark | 14,985 | 0 | +| aigc_label | 4,414 | 0 | +| the other 12 signal types | - | 0 | + +The no-op control is clean: the strip **added** a signal to 0 of 1,500 clean images. Two +real defects fell out of the run. + +**Defect 1 -- the fail-safe reports success on a file it did not strip.** All 3 parity +failures are Samsung Galaxy S22 camera PNGs (`Galaxy S22 c2pa-rs/0.37.0`) whose `caBX` +chunk survives. Cause: PIL raises `UnidentifiedImageError` on them, so +`remove_ai_metadata`'s fail-safe copies the file through byte-identical -- correct in +intent (never crash a worker on a partial upload) but it returns an output path +indistinguishable from a real strip. User-visible: `metadata --remove` prints +"AI metadata stripped ->", exits 0, and `identify` on the output still reports C2PA. The +warning is logged but the success line contradicts it. Rare here (3 of 20,153) but the +mechanism fires on ANY file PIL cannot decode. The fail-safe should stay; what needs +fixing is that the caller cannot tell a no-op from a strip. + +**Defect 2 -- 16-bit PNGs are silently downconverted to 8-bit.** 5 of the 1,500 clean +controls failed the pixel-identity check; all are 16-bit PNGs, and the PIL re-save halves +their bit depth (one went 9.2 MB -> 2.5 MB). This is the known limitation recorded in +CLAUDE.md, now measured: a byte-level IHDR scan over every corpus PNG puts it at +**42 of 27,018 (0.16%)**. + +Method note worth keeping: the first attempt to reproduce Defect 2 said "pixels +identical" and nearly closed it as a harness bug. That check read both files through +`image_io.imread`, which returns 8-bit -- **the reader destroyed the very property under +test**. The audit was right because it reads via `read_bgr_and_alpha`, which preserves +uint16. When verifying a fidelity property, check that the verification path can still +represent it. + +### A3. Byte-level invariants + +- no-op `remove_visible` returns the ORIGINAL bytes (not a re-encode) +- pixels outside the fill mask are bit-identical to the input +- JPEG metadata strip is pixel-lossless on the DEFAULT path (`--remove-all` re-encodes by + design -- see `metadata.py`; assert the split, not losslessness everywhere) +- lossless source formats survive a misnamed extension + +### A4. Idempotence and order-independence + +- `remove_visible(remove_visible(x)) == remove_visible(x)` +- `strip(remove(x)) == remove(strip(x))` in signal terms +- a second `identify` on a cleaned output reports no metadata signals + +### A5. Contract sweep across every parameter choice + +`scripts/smoke_matrix.py` (exists, 68 rows, 0 skipped with `--diffusion`) covers every +choice-valued flag on fixtures. Extend from fixtures to a stratified corpus slice +(~500 images spanning format x provenance x aspect ratio), asserting exit-code semantics +rather than just absence of crash. + +Known trap to encode: **exit 2 is triply overloaded** (no-visible-mark, no-invisible-signal, +Click usage error). A wrapper cannot distinguish them without parsing stderr. Either the +sweep asserts on stderr, or the codes get split -- the latter is the better fix. + +#### Coverage before the extension (measured 2026-07-19, not estimated) + +Comparing the flags the matrix actually executed against the flags the CLI declares: +**18 of 38 had never been executed even once** -- `--pipeline`, `--strength`, `--steps`, +`--guidance-scale`, `--device`, `--model`, `--upscaler`, `--tile`/`--tile-size`/ +`--tile-overlap`, `--humanize`, `--unsharp`, `--adaptive-polish`, `--controlnet-scale`, +`--min-resolution`, `--hf-token`, `--auto`, `--verbose`. Plus uncovered VALUES of covered +flags: `--backend migan|lama` had never been driven through the CLI at all (only at +library level), `erase --backend` only ever ran `cv2`, `batch --mode all` never ran, and +`--pipeline` only ever ran its default. + +Whole subsystems had unit tests but no real-data run: tiling (27 unit tests, never +processed a real image through the CLI), the region-targeted composite, the ESRGAN +upscaler, and the ffmpeg audio/video strip. The gap is not "logic untested" but +"never executed on real data", which is precisely what this campaign is for. + +#### Bug found by the extension: `--steps` below ~7 crashes inside torch + +Effective timesteps are `int(steps * strength)`. At the vendor-adaptive default strength +(0.15, or 0.10 for OpenAI) any `--steps` under 7 rounds to **zero**, and the pipeline dies +with a raw traceback: + +``` +$ remove-ai-watermarks invisible img.png --steps 5 +RuntimeError: cannot reshape tensor of 0 elements into shape [0, -1, 1, 512] +``` + +Fully valid CLI arguments, no special flags, no `--force`. The value is accepted, the +crash is a torch internal, and nothing tells the user that steps and strength interact. +Fix is either a clamp to >=1 effective step or an up-front validation naming both values. + +Method note: the first run of the knob rows failed 12 times with this identical error, +which read like twelve broken features. It was one bad harness parameter (`--steps 4`) +sitting on top of one real bug. An error that is IDENTICAL across unrelated rows is +evidence of a common cause, not of many faults -- check the shared input first. + +## Tier B -- constructed ground truth (automatable, no labelling) + +Where reality gives no answer key, build one. This is the tier that closes the two biggest +holes: fill quality has **never** been asserted, and recall was measured once at n=240. + +### B1. Fill quality with a true reference + +Real marks have no clean counterpart, so quality has only ever been eyeballed. Construct it +instead: take a clean corpus image, stamp a known mark at a known position (the captured +alpha maps make this exact), remove it, and compare against the **true original**. + +Yields PSNR / SSIM per `--backend` (cv2 / migan / lama), sliced by background class, which +is exactly the axis where the docs say quality varies but no number exists. + +Implemented as `scripts/fill_quality.py`. Two reporting rules are load-bearing: score +INSIDE the footprint (whole-frame PSNR sits near 60 dB whatever the backend does), and use +the MEDIAN (a fill that reproduces a flat background exactly scores PSNR=inf, and one inf +makes a mean inf -- the first run reported "+inf" for every flat bucket). + +#### First run, 2026-07-19, 80 verified-clean sources, n=720 measurements + +Median dB recovered inside the footprint (filled PSNR minus damaged PSNR): + +| mark | bg | cv2 | migan | lama | +|---|---|---|---|---| +| doubao | flat | +10.79 | **+15.62** | +13.97 | +| doubao | mid | +9.25 | **+14.71** | +14.69 | +| doubao | textured | +2.15 | +0.66 | **+1.64** | +| jimeng | flat | +5.52 | +8.80 | **+10.30** | +| jimeng | mid | +10.78 | +11.05 | **+12.33** | +| jimeng | textured | +2.16 | +1.70 | **+3.03** | + +**The `auto` order is CONFIRMED.** Median per-image recovery on the textured tercile: + +| mark (textured) | cv2 | migan | lama | +|---|---|---|---| +| doubao | +0.06 | +1.53 | +1.38 | +| gemini | +1.79 | +2.75 | **+5.59** | +| jimeng | +1.24 | +2.69 | **+3.34** | +| jimeng_pill | +4.27 | +3.81 | **+5.16** | + +LaMa > MI-GAN > cv2 holds on 3 of the 4 marks worth filling, and MI-GAN edges LaMa on +doubao. Nothing here argues for changing `--backend auto`. + +**Correction, and the statistic that caused it.** An earlier version of this section +claimed the opposite -- that MI-GAN was the WORST on texture, below cv2 -- and it was +wrong. The report computed recovery as `median(filled) - median(damaged)`: a **difference +of medians**, not the median of the per-image differences. On skewed data those are +different statistics and here they disagreed in SIGN. A paired per-mark sign test settled +it: on the textured tercile cv2 vs MI-GAN is not significant for any mark (p 0.13-1.00; +pooled n=119, cv2 wins 69, p=0.099), while MI-GAN's medians are higher for 3 of 5. + +The wrong statistic nearly shipped a change to `--backend auto`, which is the resolver a +memory-constrained CPU caller depends on. Two lessons: report the median of the per-image +DIFFERENCES when the question is paired, and confirm a ranking with a paired test before +acting on a table of independently-aggregated columns. + +**Every backend collapses on texture**: recovery falls from ~+10-15 dB to ~+1-3 dB and SSIM +from ~0.79-0.95 to ~0.30-0.44. The documented "textured is where fills struggle" is +confirmed, with numbers, for the first time. + +**The invariant held**: 0 violations of "the fill touches nothing outside its mask" across +all 720 measurements and all three backends. + +#### The Jimeng pill: measure the GATED path or the number is meaningless + +The parity audit calls `get_mark(key).remove`, which bypasses `_keep_pill`. On the pill +that path reports "detector still fires after removal" 68-75% of the time and reads as a +broken feature. It is not: the product gates the pill hard, and `--mark auto` behaves +completely differently. + +Measured through the product path over **all 2,738 pill positives** (2026-07-20): + +| | n | | +|---|---|---| +| raw detections | 2,738 | | +| corroborated real (wordmark or TC260) | 346 | 12.6% | +| the gate lets through to removal | 125 | 4.6% | +| **of those, corroborated real** | **125** | **precision 100% (95% CI 97.0-100)** | + +Every single pill the product removed was corroborated. Headline recall is 36.1%, but that +denominator is wrong: **TC260 provenance maps to BOTH ByteDance products**, so a "TC260 +says Jimeng" image may be a Doubao image with no pill at all. Split by evidence strength: + +| corroboration | n | removed | recall | +|---|---|---|---| +| wordmark (names the product) | 49 | 49 | **100% (CI 92.7-100)** | +| TC260 only (cannot separate Doubao) | 297 | 76 | 25.6% (CI 21.0-30.8) | + +So where the evidence actually names Jimeng, the pill is removed every time. The flatness +guard suppresses 186 TC260-only cases -- its measured recall cost, paid to avoid the +smeared textured fills it exists to prevent. + +Two harness lessons, both of which produced a wrong number before being caught: +- **Gated marks must be measured through the gate.** A per-mark audit answers a question + the product never asks. +- **Match mark labels exactly.** A substring test on `AI生成` also matches Doubao's label + (`Doubao 豆包AI生成 text`), counting Doubao removals as pill removals and inflating the + pill's precision. + +#### The finding that was not being looked for: filling a faint mark is net negative + +Samsung came out negative in every cell, which looked like a broken mark. It is not -- the +sign is set by how strongly the mark perturbs the image, not by which mark it is. Per-mark +recovery by damage band (a HIGH damage-PSNR means a FAINT mark): + +| mark | 0-18 dB | 18-22 dB | 22-26 dB | 26+ dB | +|---|---|---|---|---| +| doubao | +9.78 (n=160) | -1.52 (36) | -3.37 (18) | -8.38 (23) | +| jimeng | +7.70 (187) | -2.49 (21) | -2.54 (15) | -2.34 (15) | +| samsung | - | **+7.23** (48) | -3.78 (102) | -10.78 (90) | + +Samsung is POSITIVE where its mark is strong; doubao and jimeng go NEGATIVE where theirs +are faint. Linear fit over all 720: `recovery = -0.861 * damage_psnr + 19.47`, break-even +at **~22.6 dB**. Samsung's alpha map peaks at 0.37 against doubao 0.68 and jimeng 0.93, so +Samsung simply sits on the faint side of that line most of the time. + +So: **below ~22.6 dB of mark damage, inpainting costs more fidelity than the mark did.** +The pipeline currently fills unconditionally once a mark is detected, so this cost is +invisible today. + +Do not read this as "stop removing faint marks". A user who wants the watermark GONE is not +optimizing PSNR, and a faint mark is still a mark. What it says is that the fill has a real +cost, it is now measurable, and for faint marks it exceeds the thing it removes -- which +makes "how faint is too faint" a product decision that can finally be made on evidence. + +### B2. Detector response curves + +Stamp marks across a controlled grid -- size, contrast, background texture, aspect, +JPEG quality -- and measure detection rate per cell. Produces a recall **curve** instead of +a point estimate, and directly tests the `scale_basis` geometry that hid a 100% landscape +miss for months. Cheap, repeatable, no human in the loop. + +### B3. Invisible round-trip, positive-control gated + +The open DWT-DCT detector is positive-only and carrier-fragile: "not found" on a fragile +carrier proves nothing (measured: `chatgpt-1.png` recovers 114/128, below the 118 gate). +Every invisible assertion must first embed on the SAME carrier and confirm recovery, and +**degrade to a skip rather than a pass** when the control fails. Already implemented in +`smoke_matrix.py`; apply the same discipline anywhere else this detector is used. + +### B4. Resource ceilings + +Peak RSS and wall time per backend x input size, up to 25 MP. The memory-constrained CPU +tier is a real constraint (MI-GAN must stay ~0.6-0.9 GB by cropping around the mask); a +regression here is invisible today and would only surface under load. + +## Tier C -- human-labelled accuracy (bounded by labelling effort) + +The machinery exists: `visible_recall_sample.py` -> `visible_sheets.py` -> +`visible_groundtruth.py` -> `visible_eval.py`. + +- **Recall** is the known weak spot: measured once, unbiased n=240, and that single + measurement is what exposed the landscape miss. Expand per mark, especially jimeng + (n=14) and jimeng_pill (n=6), whose numbers currently rest on almost nothing. +- **Precision** re-runs over the existing 779-cell ground truth; benchmark every detector + change with `--vs `. +- **Coverage**, the largest known gap: ~6% of sampled images carry an uncovered vendor's + mark (千问 / 百度 / 星绘 / 抖音-class) that no registered detector can fire on. This is a + coverage problem, not a tuning problem, and no threshold work will move it. + +Three harness rules are load-bearing and must not be relaxed: score a mark only within its +crop's adjudication scope; take provenance from metadata, never from labels; and never +report recall from the detector-sampled set. + +## Tier D -- external oracles (manual, not automatable here) + +SynthID removal cannot be verified locally by design -- no public decoder exists. Each +vendor has its own oracle and it covers only that vendor's content: `openai.com/verify` for +OpenAI (more accessible, the automation candidate), the Gemini app for Google (manual, +rate-limited). A quiet metadata proxy is **not** proof the pixel watermark is gone. + +Scope honestly: this tier certifies strength floors on a handful of images per vendor, and +that is all it can do. See `docs/synthid.md`. + +### D1. Sampling frame + +The sidecars already classify the corpus: **15,000 images whose watermark list mentions +SynthID**, of which 9,071 carry `verify_oracle=openai` and 5,929 `verify_oracle=google`. +Stratify on the two axes that actually move removal efficacy -- vendor (the certified +floors differ: OpenAI 0.10, Gemini 0.15) and content class (photoreal vs flat graphic, +where the pipelines are documented to diverge). + +### D2. The oracle is the bottleneck, not the GPU + +Measured on MPS (2026-07-19), single invocation of `invisible`: + +| `--max-resolution` | 256 | 384 | 512 | 768 | 1024 | +|---|---|---|---|---|---| +| wall time | 37.9 s | 58.9 s | 59.4 s | 65.2 s | 118.3 s | + +384/512/768 are indistinguishable, so below ~768 the cost is dominated by **fixed model +load, not diffusion**. Confirmed by batching: 4 images in one process took 105.5 s +(26.4 s/image) against 59.4 s/image one at a time -- roughly 40 s fixed overhead per +invocation and ~15 s marginal per image at 512 (rough: run-to-run variance is large). + +Two consequences for the harness: +- **Amortize the fixed cost**: one long-lived process over many images, never one + invocation per image. That is a 2-4x win. Shrinking below 512 is not. +- The binding constraint is the **external oracle's throughput**, which is manual and rate + limited. So do not run a uniform grid; spend each oracle check where the answer is + uncertain -- **bisect strength per content class** to certify a floor in ~10 checks + instead of ~100. + +### D3. Mandatory control before trusting any reduced-size run + +`--max-resolution` is downscale -> diffuse -> Lanczos upscale. If the **resize round trip +alone** damages SynthID, the oracle goes quiet for a reason unrelated to removal and the +result does not transfer to production at native resolution. + +Before any reduced-size sweep, run the resize round trip with **no diffusion** and put the +result through the vendor oracle. If the watermark survives, the reduced size is a valid +test bed; if it does not, reduced-size results are measuring the resizer. This is the same +failure shape as the imwatermark carrier-fragility trap in B3: an oracle that falls silent +for the wrong reason reads exactly like success. + +`docs/synthid.md` cites ~99.98% TPR across 30 transforms including resize, which predicts +the control passes -- but that is Google's claim about their own decoder, not our +measurement, so it is a hypothesis to test, not a reason to skip the control. + +## Tier E -- robustness and adversarial inputs + +Malformed and hostile inputs, since ~0.2% of real uploads are already truncated: truncation +at many offsets, corrupt headers, 16-bit and CMYK, absurd dimensions, decompression bombs, +zero-byte files, unicode and RTL filenames, symlinks, read-only output dirs, concurrent runs +on one file. The bar is never "handles it" but **never raises and never silently degrades**. + +## Build order + +1. **A1 sidecar regression** -- highest value per hour, unattended, needs no new labels. +2. **A2/A3/A4 parity and invariants** -- full corpus, reuses existing audit scripts. +3. **B1 fill quality** -- closes the oldest unmeasured claim in the project. +4. **B2 detector curves** -- cheap, and directly guards the geometry class of bug. +5. **A5 contract sweep at corpus scale**. +6. **B4 resource ceilings**, **E robustness**. +7. **C recall expansion** -- gated by labelling appetite. +8. **D oracles** -- manual, per release. + +Every tier writes a versioned snapshot so runs are comparable over time; a run that cannot +be diffed against the last one is a one-off, not a regression suite. + +## What the measurements imply for detection work + +Recorded here because each item is grounded in a number from this campaign, not because +it is a prioritized plan (that lives elsewhere -- see the note at the end of this section). + +### Metadata absence does not disable the detectors -- it disables the RELAXATION + +The detectors are pixel-based and need no metadata. What metadata does is relax the +false-positive gate (`auto` vs `strict`). So "work better without metadata" means +strengthening the strict-path detectors themselves; it is not a gating problem. + +Per mark, what actually goes away when metadata is stripped: + +- **The pill loses an entire arm.** Its TC260 arm is dead without metadata, leaving only + the wordmark arm. Measured: where the wordmark corroborates, pill recall is **100%** + (49/49); on TC260-only evidence it is 25.6%. So on stripped uploads the pill's fate + rests entirely on Jimeng wordmark detection -- whose own recall is **71% on n=14**. + This is the weakest link with the most leverage: every point of wordmark recall pulls + the pill along with it. +- **The sparkle already runs on pixels**, and the FP-gate tightening cost ~120-200 genuine + detections corpus-wide (12.5% of the 1,256 lost). That headroom exists but the + precision trade behind it was deliberate. +- **The largest gap is metadata-independent by nature**: ~6% of sampled images carry an + uncovered vendor's mark (千问 / 百度 / 星绘 / 抖音-class) that no registered detector + can fire on at all. + +### Where the evidence points + +1. **A generic CJK AI-mark detector.** GB 45438-2025 mandates the shared `AI生成` tail, + and 千問 is already measured as non-separable from Doubao (AUC ~0.5) precisely because + of it. The right shape is to detect the mark CLASS and treat vendor attribution as + optional metadata. Closes the 6% coverage gap and is metadata-free by construction. +2. **Port the `tophat` front-end to the remaining marks.** It took Doubao from 89% to 92% + recall at unchanged 99% precision. But the gate is front-end specific and **must be + recalibrated, never ported**: a naive 0.40 produced 8 false fires instead of 1 and + silently halved the pill's recall (because `_keep_pill` suppresses the pill whenever + Doubao fires). +3. **The Jimeng wordmark.** Weak on its own (71%/71%) and it gates the pill. Its + silhouette is also non-discriminative against Doubao's, which was patched with a 0.85 + threshold -- a patch on a detector problem, not a fix. + +### Measure before improving + +Jimeng recall rests on **n=14** and the pill's on **n=6**. Improving what is measured by +six samples means not knowing whether it improved. Tier B2 (detector response curves on +stamped marks) is the instrument to build first: recall as a function of size, contrast +and background texture, with no new hand labelling, and it catches the geometry class of +bug (`scale_basis`) directly. + +This section records what the measurements imply technically. Prioritization is tracked +separately, outside this repo. + +## Standing gap + +None of this is in `maintain.sh`, and it should not all be -- the sweeps take hours. But +that means **no detector-accuracy or CLI-contract regression is caught automatically** +today. The endpoint of this plan is a cheap subset (fixtures-only smoke + a sidecar diff on +a fixed 500-image slice) that CI can run, with the full sweeps staying pre-release. diff --git a/scripts/fill_quality.py b/scripts/fill_quality.py new file mode 100644 index 0000000..689dace --- /dev/null +++ b/scripts/fill_quality.py @@ -0,0 +1,323 @@ +"""Tier B1: measure visible-mark FILL quality against a constructed ground truth. + +THE PROBLEM THIS SOLVES + A real watermarked image has no clean counterpart, so fill quality has only ever been + eyeballed ("cv2 smears on texture, LaMa is best") and never carried a number. The docs + state a preference order for `--backend auto` that rests on no measurement. + +THE CONSTRUCTION + Take a VERIFIED-CLEAN corpus image, stamp a known mark onto it using the mark's own + captured alpha map (the forward model the reverse-alpha work established: + `stamped = (1-a)*bg + a*white`), then remove it and compare against the TRUE original. + The original is the answer key that reality never provides. + +TWO MEASUREMENT DECISIONS THAT MAKE OR BREAK THIS + * Score INSIDE the footprint only. The fill touches a tiny corner, so whole-frame PSNR + sits near 60 dB whatever the backend does and would rank them all "excellent". + * Report the DAMAGE baseline (stamped vs original) next to the recovery (filled vs + original). Without it a PSNR is uninterpretable: 30 dB is a triumph if the mark cost + 12 dB and a failure if it cost 29 dB. The honest metric is how much of the gap the + fill closes. + +WHAT IT DOES NOT MEASURE + Detection. The mark is localized with `force=True`, so a miss cannot contaminate the + fill numbers -- this isolates the FILL. Detection accuracy is Tier C's job. + +DATA SAFETY + Corpus images are user uploads: read-only, local analysis, output under a gitignored + data/spaces/ path. No image content is written into the report. + + uv run python scripts/fill_quality.py --n 60 +""" + +from __future__ import annotations + +import argparse +import collections +import glob +import json +import random +import sys +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + +sys.path.insert(0, str(Path(__file__).parent.parent)) +sys.path.insert(0, str(Path(__file__).parent)) + +from invisible_quality_audit import _ssim # reuse, do not reimplement a third SSIM + +REPO = Path(__file__).resolve().parents[1] +CORPUS = REPO / "data" / "spaces" / "originals" +OUT = REPO / "data" / "spaces" / "_fill_quality.jsonl" + +# Text marks: a bundled alpha PNG plus the engine's own corner geometry. +STAMPABLE = ("doubao", "jimeng", "samsung") +# Marks with no bundled alpha asset. Both expose a default footprint via +# `footprint_mask(force=True)`, so they are stamped by fitting their own alpha source +# into that slot: Gemini's alpha is derived from its background captures, the pill's +# from its synthetic font-rendered silhouette. +SLOT_STAMPABLE = ("gemini", "jimeng_pill") + + +def psnr(a: np.ndarray, b: np.ndarray) -> float: + mse = float(np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)) + return float("inf") if mse == 0 else 10 * float(np.log10(255.0**2 / mse)) + + +def texture_of(box: np.ndarray) -> float: + """Median Sobel magnitude -- the same texture proxy the pill's flatness gate uses.""" + g = cv2.cvtColor(box, cv2.COLOR_BGR2GRAY) if box.ndim == 3 else box + gx = cv2.Sobel(g, cv2.CV_32F, 1, 0, ksize=3) + gy = cv2.Sobel(g, cv2.CV_32F, 0, 1, ksize=3) + return float(np.median(cv2.magnitude(gx, gy))) + + +def engine_for(mark_key: str) -> Any: + """The TextMarkEngine instance for a mark key (the registry does not expose one).""" + import importlib + + mod = importlib.import_module(f"remove_ai_watermarks.{mark_key}_engine") + cls = next( + obj + for name, obj in vars(mod).items() + # Exclude the imported base class: it takes a `config` arg, the subclasses do not. + if isinstance(obj, type) and name.endswith("Engine") and name != "TextMarkEngine" + ) + return cls() + + +def stamp(image: np.ndarray, mark_key: str) -> tuple[np.ndarray, tuple[int, int, int, int]] | None: + """Composite a mark's alpha glyph into its canonical corner. Returns (stamped, bbox).""" + from remove_ai_watermarks._text_mark_engine import load_alpha_template + + engine = engine_for(mark_key) + cfg = engine.config + alpha = load_alpha_template(cfg.asset_name) + if alpha is None: + return None + + loc = engine.locate(image) + base = engine.scale_base(image) + gw = max(cfg.min_gw, int(cfg.alpha_width_frac * base)) + gh = max(4, int(cfg.alpha_height_frac * base)) + if gw < 8 or gh < 4 or gw > loc.w or gh > loc.h: + return None + + a = cv2.resize(alpha, (gw, gh), interpolation=cv2.INTER_AREA).astype(np.float32) + x = loc.x + (loc.w - gw) // 2 + y = loc.y + (loc.h - gh) // 2 + out = image.copy() + roi = out[y : y + gh, x : x + gw].astype(np.float32) + a3 = a[..., None] + out[y : y + gh, x : x + gw] = np.clip(roi * (1 - a3) + 255.0 * a3, 0, 255).astype(np.uint8) + return out, (x, y, gw, gh) + + +def _slot_alpha(mark_key: str) -> np.ndarray | None: + """The alpha source for a mark that ships no bundled alpha PNG.""" + if mark_key == "gemini": + from remove_ai_watermarks.gemini_engine import _shared_engine + + # _shared_engine is the package's own lru_cache singleton: constructing + # GeminiEngine() per image reloads its captures, recomputes both alpha maps and + # rebuilds the whole template ladder -- the exact cost that singleton exists to + # avoid. Private attribute access is deliberate (no accessor exists); a + # measurement script may reach in, product code must not. + return np.asarray(_shared_engine()._alpha_large, dtype=np.float32) + if mark_key == "jimeng_pill": + from remove_ai_watermarks._text_mark_engine import load_alpha_template + + return load_alpha_template("jimeng_pill.png") + return None + + +def stamp_slot(image: np.ndarray, mark_key: str) -> tuple[np.ndarray, tuple[int, int, int, int]] | None: + """Stamp a mark into its OWN default footprint slot (the `--no-detect` geometry). + + Used for gemini and the pill, which have no bundled alpha asset with corner + fractions. The mark is fitted into the middle of its slot so the fill has to + recover the same kind of region it would in production. + """ + from remove_ai_watermarks.watermark_registry import get_mark + + alpha = _slot_alpha(mark_key) + if alpha is None: + return None + loc = get_mark(mark_key).localize(image, force=True) + if loc.mask is None: + return None + ys, xs = np.nonzero(loc.mask) + if ys.size == 0: + return None + y0, y1, x0, x1 = int(ys.min()), int(ys.max()), int(xs.min()), int(xs.max()) + bw, bh = x1 - x0 + 1, y1 - y0 + 1 + gw, gh = max(8, int(bw * 0.7)), max(8, int(bh * 0.7)) + if gw < 8 or gh < 8 or gw > image.shape[1] or gh > image.shape[0]: + return None + a = cv2.resize(alpha, (gw, gh), interpolation=cv2.INTER_AREA).astype(np.float32) + a = np.clip(a, 0.0, 1.0) + x, y = x0 + (bw - gw) // 2, y0 + (bh - gh) // 2 + out = image.copy() + roi = out[y : y + gh, x : x + gw].astype(np.float32) + a3 = a[..., None] + out[y : y + gh, x : x + gw] = np.clip(roi * (1 - a3) + 255.0 * a3, 0, 255).astype(np.uint8) + return out, (x, y, gw, gh) + + +def stamp_any(image: np.ndarray, mark_key: str) -> tuple[np.ndarray, tuple[int, int, int, int]] | None: + return stamp(image, mark_key) if mark_key in STAMPABLE else stamp_slot(image, mark_key) + + +def clean_sources(n: int, seed: int = 11) -> list[Path]: + """Corpus images with NO metadata signal and NO mark detection -- verified clean.""" + from remove_ai_watermarks.identify import identify + from remove_ai_watermarks.image_io import imread + from remove_ai_watermarks.watermark_registry import detect_marks + + pool = glob.glob(str(CORPUS / "*" / "*")) + random.Random(seed).shuffle(pool) # noqa: S311 -- deterministic sampling, not crypto + out: list[Path] = [] + for p in pool: + if len(out) >= n: + break + path = Path(p) + try: + img = imread(str(path)) + if img is None or min(img.shape[:2]) < 400: + continue + # check_visible=False: identify would otherwise decode the file a second + # time and run the very same visible detectors as detect_marks below. + if identify(path, check_visible=False).signals: + continue + if any(d.detected for d in detect_marks(img)): + continue + except Exception: # noqa: S112 -- a bad corpus file just is not a candidate + continue + out.append(path) + return out + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=60, help="clean source images") + ap.add_argument("--out", type=Path, default=OUT) + a = ap.parse_args() + + from remove_ai_watermarks.image_io import imread + from remove_ai_watermarks.region_eraser import lama_available, migan_available + from remove_ai_watermarks.watermark_registry import fill, get_mark + + backends = ["cv2"] + (["migan"] if migan_available() else []) + (["lama"] if lama_available() else []) + print(f"backends under test: {backends}") + print(f"selecting {a.n} verified-clean sources...", flush=True) + sources = clean_sources(a.n) + print(f"got {len(sources)}\n", flush=True) + + rows: list[dict[str, Any]] = [] + with open(a.out, "w", encoding="utf-8") as fh: + for i, src in enumerate(sources, 1): + base = imread(str(src)) + if base is None: + continue + for key in (*STAMPABLE, *SLOT_STAMPABLE): + st = stamp_any(base, key) + if st is None: + continue + stamped, (x, y, w, h) = st + # Score a slightly padded box: the fill dilates, so a tight glyph box + # would miss damage the backend does just outside the glyph. + pad = 8 + y0, y1 = max(0, y - pad), min(base.shape[0], y + h + pad) + x0, x1 = max(0, x - pad), min(base.shape[1], x + w + pad) + truth = base[y0:y1, x0:x1] + dmg_psnr = psnr(stamped[y0:y1, x0:x1], truth) + dmg_ssim = _ssim( + cv2.cvtColor(stamped[y0:y1, x0:x1], cv2.COLOR_BGR2GRAY), cv2.cvtColor(truth, cv2.COLOR_BGR2GRAY) + ) + tex = texture_of(truth) + + mark = get_mark(key) + loc = mark.localize(stamped, force=True) + if loc.mask is None: + continue + for backend in backends: + try: + filled = fill(stamped, loc.mask, backend=backend) + except Exception as e: + rows.append({"src": src.name, "mark": key, "backend": backend, "error": str(e)[:150]}) + continue + # Invariant: the fill must not touch anything outside its mask. + outside = loc.mask == 0 + untouched = bool(np.array_equal(filled[outside], stamped[outside])) + rec = { + "src": src.name, + "mark": key, + "backend": backend, + "texture": round(tex, 2), + "damage_psnr": round(dmg_psnr, 2), + "damage_ssim": round(dmg_ssim, 4), + "filled_psnr": round(psnr(filled[y0:y1, x0:x1], truth), 2), + "filled_ssim": round( + _ssim( + cv2.cvtColor(filled[y0:y1, x0:x1], cv2.COLOR_BGR2GRAY), + cv2.cvtColor(truth, cv2.COLOR_BGR2GRAY), + ), + 4, + ), + "outside_mask_untouched": untouched, + } + rows.append(rec) + fh.write(json.dumps(rec) + "\n") + if i % 10 == 0: + fh.flush() + print(f" {i}/{len(sources)}", flush=True) + + report(rows, a.out) + + +def report(rows: list[dict[str, Any]], out: Path) -> None: + good = [r for r in rows if "error" not in r] + if not good: + print("no measurements") + return + tex = sorted(r["texture"] for r in good) + t1, t2 = tex[len(tex) // 3], tex[2 * len(tex) // 3] + + def bucket(t: float) -> str: + return "flat" if t <= t1 else ("mid" if t <= t2 else "textured") + + print(f"\n{'=' * 78}\nFILL QUALITY n={len(good)} measurements (texture terciles at {t1:.1f} / {t2:.1f})") + print(f"{'=' * 78}") + print("recovered = filled PSNR minus damaged PSNR; how much of the mark's damage the fill undoes\n") + hdr = f"{'mark':9s} {'backend':8s} {'bg':9s} {'n':>4s} {'damaged':>9s} {'filled':>9s}" + print(f"{hdr} {'recovered':>10s} {'ssim':>7s}") + agg: dict[tuple[str, str, str], list[dict[str, Any]]] = collections.defaultdict(list) + for r in good: + agg[(r["mark"], r["backend"], bucket(r["texture"]))].append(r) + for (mark, backend, bg), rs in sorted(agg.items()): + # MEDIAN, not mean: a fill that reproduces a flat background EXACTLY scores + # PSNR=inf, and a single inf makes the mean inf -- which reported "+inf" for + # every flat bucket on the first run and hid the real numbers. + finite = [r for r in rs if np.isfinite(r["filled_psnr"])] + perfect = len(rs) - len(finite) + d = float(np.median([r["damage_psnr"] for r in rs])) + f = float(np.median([r["filled_psnr"] for r in finite])) if finite else float("inf") + sv = float(np.median([r["filled_ssim"] for r in rs])) + # Recovery is the MEDIAN OF THE PER-IMAGE DIFFERENCES, not the difference of the + # two medians. Those are not the same statistic on skewed data and they can even + # disagree in SIGN: the first version of this report subtracted medians and + # produced a backend ranking that a paired comparison did not support. + rec = float(np.median([r["filled_psnr"] - r["damage_psnr"] for r in finite])) if finite else float("inf") + tail = f" ({perfect} exact)" if perfect else "" + print(f"{mark:9s} {backend:8s} {bg:9s} {len(rs):4d} {d:9.2f} {f:9.2f} {rec:+10.2f} {sv:7.4f}{tail}") + + bad = [r for r in good if not r["outside_mask_untouched"]] + print(f"\noutside-mask invariant violations: {len(bad)}" + (" <-- BUG" if bad else " (none)")) + print(f"records: {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/pill_gate_audit.py b/scripts/pill_gate_audit.py new file mode 100644 index 0000000..297a125 --- /dev/null +++ b/scripts/pill_gate_audit.py @@ -0,0 +1,224 @@ +"""Measure the Jimeng pill on the PRODUCT path, not the raw detector. + +WHY THIS EXISTS + `visible_removal_audit.py` calls `get_mark(key).remove` directly, which bypasses + `_keep_pill`. For the pill that is the wrong path and it reads as a disaster: on a + 300-image sample the raw path was "detector still fires after removal" 68-75% of the + time. The product does not do that -- `--mark auto` gates the pill hard (never on + Doubao; unrestricted only when the bottom-right wordmark fired; otherwise only on a + flat footprint). Measured through the gate on the same sample, 7 of 80 raw detections + survived to removal and 6 of those were corroborated real. + + That sample was too small to state a precision (95% CI 49-97%). This script runs the + gated path over EVERY pill positive in the corpus so the interval is usable. + +THE CORROBORATION PROXY AND ITS BIAS + "Real pill" here means: TC260 metadata names Jimeng, OR the bottom-right "★ 即梦AI" + wordmark is detected. Both are independent of the pill detector, which is the point. + But the proxy MISSES a real pill on a metadata-stripped screenshot with no visible + wordmark -- so measured precision is a LOWER BOUND, not a point estimate. Do not quote + it as if it were exact. + +Corpus images are user uploads: read-only, local analysis, gitignored output. + + uv run python scripts/pill_gate_audit.py --jobs 7 +""" + +from __future__ import annotations + +import argparse +import collections +import json +import math +import os +import sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FutureTimeout +from concurrent.futures.process import BrokenProcessPool +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +REPO = Path(__file__).resolve().parents[1] +POSITIVES = REPO / "data" / "spaces" / "_visible_positives.jsonl" +OUT = REPO / "data" / "spaces" / "_pill_gate_audit.jsonl" + + +def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + if n == 0: + return (0.0, 0.0) + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return (100 * (c - h), 100 * (c + h)) + + +def _one(path_str: str) -> dict[str, object]: + from remove_ai_watermarks.api import visible_provenance + from remove_ai_watermarks.image_io import imread + from remove_ai_watermarks.pill_engine import PillEngine + from remove_ai_watermarks.watermark_registry import detect_marks, get_mark, remove_auto_marks + + rec: dict[str, object] = {"path": Path(path_str).name, "status": "ok"} + try: + img = imread(path_str) + if img is None: + return {**rec, "status": "unreadable"} + prov = visible_provenance(Path(path_str)) + dets = {d.key: d for d in detect_marks(img) if d.detected} + wordmark = "jimeng" in dets + tc260 = "jimeng" in prov + eng = PillEngine() + _, labels = remove_auto_marks(img, sensitivity="auto", provenance=prov, backend="cv2") + # Match the EXACT pill label. A substring test on "AI生成" also matches Doubao's + # label ("Doubao 豆包AI生成 text"), which counted Doubao removals as pill removals + # and inflated the pill's measured precision. + pill_label = get_mark("jimeng_pill").label + removed = any(str(x) == pill_label for x in labels) + # The gate reads the ARBITRATED keys, not strict detection: jimeng can be accepted + # on its relaxed (provenance-confirmed) arm, which `detect_marks` never shows. Take + # the arm from what was actually removed, or every such case lands in "other". + jimeng_decided = any(str(x) == get_mark("jimeng").label for x in labels) + return { + **rec, + "removed_by_product": bool(removed), + "corroborated": bool(wordmark or tc260), + "wordmark": bool(wordmark), + "jimeng_decided": bool(jimeng_decided), + "tc260": bool(tc260), + "doubao_fired": "doubao" in dets, + "footprint_flat": bool(eng.footprint_is_flat(img)), + } + except Exception as e: + return {**rec, "status": f"error:{type(e).__name__}"} + + +def _batch(paths: list[str], jobs: int, timeout: int) -> list[dict[str, object]]: + try: + with ProcessPoolExecutor(max_workers=jobs) as ex: + futs = [ex.submit(_one, p) for p in paths] + return [f.result() for f in as_completed(futs, timeout=timeout)] + except (FutureTimeout, BrokenProcessPool, OSError, RuntimeError): + out = [] + for p in paths: + try: + out.append(_one(p)) + except BaseException: # a native crash costs one file, not the sweep + out.append({"path": Path(p).name, "status": "crashed"}) + return out + + +def report(rows: list[dict[str, object]]) -> None: + ok = [r for r in rows if r.get("status") == "ok"] + if not ok: + print("no usable rows") + return + removed = [r for r in ok if r["removed_by_product"]] + corro = [r for r in ok if r["corroborated"]] + tp = [r for r in removed if r["corroborated"]] + missed = [r for r in corro if not r["removed_by_product"]] + + print(f"\n{'=' * 76}\nJIMENG PILL ON THE PRODUCT PATH n={len(ok)} raw detections\n{'=' * 76}") + print(f"raw detections corroborated as real : {len(corro):5d} ({100 * len(corro) / len(ok):.1f}%)") + print(f"the gate lets through to removal : {len(removed):5d} ({100 * len(removed) / len(ok):.1f}%)") + if removed: + lo, hi = wilson(len(tp), len(removed)) + pct = 100 * len(tp) / len(removed) + print(f" of those, corroborated real : {len(tp):5d} -> precision {pct:.1f}% (95% CI {lo:.1f}-{hi:.1f})") + if corro: + lo, hi = wilson(len(tp), len(corro)) + pct = 100 * len(tp) / len(corro) + head = f"corroborated pills removed : {len(tp):5d}/{len(corro)}" + print(f"{head} -> recall {pct:.1f}% (95% CI {lo:.1f}-{hi:.1f})") + print(f"corroborated pills the gate suppressed: {len(missed):5d}") + print("\nprecision is a LOWER BOUND: the corroboration proxy cannot see a real pill on a") + print("metadata-stripped image whose wordmark is absent or missed.\n") + + print("which gate arm let it through:") + arms: collections.Counter[str] = collections.Counter() + for r in removed: + if r.get("jimeng_decided"): + arms["jimeng wordmark accepted -> unrestricted"] += 1 + elif r["tc260"]: + arms["tc260 metadata + flat footprint"] += 1 + else: + arms["other (unexpected -- the gate has no third arm)"] += 1 + for k, v in arms.most_common(): + print(f" {v:5d} {k}") + + print("\nwhy the rest were suppressed:") + sup: collections.Counter[str] = collections.Counter() + for r in ok: + if r["removed_by_product"]: + continue + if r["doubao_fired"]: + sup["doubao fired (pill never rides on it)"] += 1 + elif not r["corroborated"]: + sup["no corroboration (no wordmark, no TC260)"] += 1 + elif not r["footprint_flat"]: + sup["corroborated but footprint too textured"] += 1 + else: + sup["other"] += 1 + for k, v in sup.most_common(): + print(f" {v:5d} {k}") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2)) + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--batch", type=int, default=200) + ap.add_argument("--timeout", type=int, default=900) + ap.add_argument("--out", type=Path, default=OUT) + ap.add_argument("--restart", action="store_true") + a = ap.parse_args() + + paths = [] + with open(POSITIVES, encoding="utf-8") as fh: + for line in fh: + try: + r = json.loads(line) + except Exception: # noqa: S112 + continue + if "jimeng_pill" in (r.get("keys") or []): + paths.append(r["path"]) + paths.sort() + if a.limit: + paths = paths[: a.limit] + + done: set[str] = set() + if a.restart and a.out.exists(): + a.out.unlink() # --restart must TRUNCATE; appending kept the old rows in the report + if a.out.exists() and not a.restart: + with open(a.out, encoding="utf-8") as fh: + for line in fh: + try: + done.add(json.loads(line)["path"]) + except Exception: # noqa: S112 + continue + todo = [p for p in paths if Path(p).name not in done] + print(f"pill positives {len(paths)} done {len(done)} to do {len(todo)} jobs {a.jobs}", flush=True) + + rows: list[dict[str, object]] = [] + with open(a.out, "a", encoding="utf-8") as fh: + for i in range(0, len(todo), a.batch): + for rec in _batch(todo[i : i + a.batch], a.jobs, a.timeout): + fh.write(json.dumps(rec) + "\n") + rows.append(rec) + fh.flush() + print(f" {len(rows)}/{len(todo)}", flush=True) + + with open(a.out, encoding="utf-8") as fh: + allrows = [] + for line in fh: + try: + allrows.append(json.loads(line)) + except Exception: # noqa: S112 + continue + report(allrows) + print(f"records -> {a.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/sidecar_regression.py b/scripts/sidecar_regression.py new file mode 100644 index 0000000..2e2e352 --- /dev/null +++ b/scripts/sidecar_regression.py @@ -0,0 +1,216 @@ +"""Tier A1: diff today's `identify` against the verdicts recorded in the corpus sidecars. + +`data/spaces/identify//.json` holds the verdict a past run produced for +`data/spaces/originals//_src.`. Re-running identify and diffing turns the +corpus into a ~39k-image behavioral regression suite that needs no new labelling. + +WHAT A DIFF MEANS -- READ THIS BEFORE PANICKING + The sidecars were written by OLDER versions, so an intended improvement shows up as a + diff exactly like a regression does. The output is therefore CLASSIFIED, not pass/fail: + + lost_ai verdict was AI, now is not <- the alarm; almost always a bug + lost_signal a signal family stopped firing <- the alarm + new_ai verdict was not AI, now is <- usually an improvement + new_signal a signal family started firing <- usually an improvement + platform attribution changed + confidence confidence level changed + unchanged identical on every compared axis + + Only `lost_*` is a regression by default. Everything else needs a look before the + baseline is moved. + +WHY FAMILIES, NOT RAW STRINGS + Watermark descriptions are human-readable prose and their WORDING has changed between + versions ("C2PA Content Credentials (OpenAI)" vs "... (OpenAI, Truepic)"). Diffing raw + strings would report every rewording as a lost+new signal pair and bury the real + regressions. Signals are normalized to families (c2pa, synthid, visible_sparkle, ...) + so the comparison tracks BEHAVIOR, not phrasing. + +DATA SAFETY + Corpus images are user uploads: read-only, local analysis. Output goes to a gitignored + path under data/spaces/ and records uids, never image content. + + uv run python scripts/sidecar_regression.py --sample 500 # representative trial + uv run python scripts/sidecar_regression.py # full corpus, resumable + +Use `--sample` for a trial, never `--limit`: the sidecar list is sorted by day, so +`--limit N` reads only the EARLIEST day and its result does not generalize (the first +trial run of this script took 200 files that were all 2026-05-29). +""" + +from __future__ import annotations + +import argparse +import collections +import glob +import json +import os +import random +import sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +REPO = Path(__file__).resolve().parents[1] +IDENTIFY_DIR = REPO / "data" / "spaces" / "identify" +ORIGINALS = REPO / "data" / "spaces" / "originals" +OUT = REPO / "data" / "spaces" / "_sidecar_regression.jsonl" + +# Map a watermark description to a stable behavior family. Order matters: the first +# matching pattern wins, so put the specific tokens above the generic ones. +_FAMILIES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("synthid", ("synthid",)), + ("visible_sparkle", ("sparkle",)), + ("visible_doubao", ("豆包", "doubao")), + ("visible_jimeng", ("即梦", "jimeng")), + ("visible_samsung", ("galaxy ai", "generati dall")), + ("aigc_tc260", ("aigc", "tc260")), + ("iptc", ("iptc", "digitalsourcetype", "made with ai", "made-with-ai")), + ("c2pa", ("c2pa", "content credentials")), + ("trustmark", ("trustmark",)), + ("open_invisible", ("open invisible", "dwt", "stable diffusion xl")), + ("xai_signature", ("xai", "grok signature")), + ("exif_generator", ("exif", "software tag", "png text")), +) + + +def family_of(description: str) -> str: + low = description.lower() + for name, tokens in _FAMILIES: + if any(t in low for t in tokens): + return name + return "other" + + +def families(descriptions: list[str]) -> set[str]: + return {family_of(d) for d in descriptions or []} + + +def compare(sidecar: dict, report: object) -> dict: + """Classify the difference between a recorded verdict and a fresh one.""" + old_ai, new_ai = sidecar.get("is_ai_generated"), getattr(report, "is_ai_generated", None) + old_fam = families(sidecar.get("watermarks") or []) + new_fam = families(list(getattr(report, "watermarks", []) or [])) + old_plat, new_plat = sidecar.get("platform"), getattr(report, "platform", None) + old_conf, new_conf = sidecar.get("confidence"), getattr(report, "confidence", None) + + classes: list[str] = [] + # Treat only a real True->not-True transition as lost. None means "unknown", and the + # library never asserts False, so None->None is not a change. + if bool(old_ai) and not bool(new_ai): + classes.append("lost_ai") + if not bool(old_ai) and bool(new_ai): + classes.append("new_ai") + if old_fam - new_fam: + classes.append("lost_signal") + if new_fam - old_fam: + classes.append("new_signal") + if old_plat != new_plat: + classes.append("platform") + if old_conf != new_conf: + classes.append("confidence") + + return { + "classes": classes or ["unchanged"], + "old_ai": old_ai, + "new_ai": new_ai, + "lost_families": sorted(old_fam - new_fam), + "new_families": sorted(new_fam - old_fam), + "old_platform": old_plat, + "new_platform": new_plat, + "old_confidence": old_conf, + "new_confidence": new_conf, + } + + +def _one(sidecar_path: str) -> dict: + uid = os.path.basename(sidecar_path)[:-5] + day = os.path.basename(os.path.dirname(sidecar_path)) + try: + with open(sidecar_path, encoding="utf-8") as fh: + sidecar = json.load(fh) + except Exception as e: # a corrupt sidecar must not kill the sweep + return {"uid": uid, "day": day, "status": "sidecar_unreadable", "error": str(e)[:200]} + + src = sidecar.get("src") or "" + image = ORIGINALS / day / src + if not src or not image.exists(): + found = glob.glob(str(ORIGINALS / day / f"{uid}*")) + if not found: + return {"uid": uid, "day": day, "status": "image_missing"} + image = Path(found[0]) + + try: + from remove_ai_watermarks.identify import identify + + report = identify(image) + except Exception as e: # record and continue; a crash IS a finding + return {"uid": uid, "day": day, "status": "identify_raised", "error": f"{type(e).__name__}: {e}"[:300]} + + return {"uid": uid, "day": day, "status": "ok", **compare(sidecar, report)} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=0, help="only N sidecars (trial run)") + ap.add_argument( + "--sample", + type=int, + default=0, + help="randomly sample N across ALL days -- use this for a trial, not --limit: " + "the sidecar list is sorted, so --limit takes one day and is not representative", + ) + ap.add_argument("--workers", type=int, default=max(1, (os.cpu_count() or 4) - 2)) + ap.add_argument("--out", type=Path, default=OUT) + ap.add_argument("--restart", action="store_true", help="ignore existing output and start over") + a = ap.parse_args() + + sidecars = sorted(glob.glob(str(IDENTIFY_DIR / "*" / "*.json"))) + if a.sample: + random.Random(7).shuffle(sidecars) # noqa: S311 -- deterministic sampling, not crypto + sidecars = sorted(sidecars[: a.sample]) + elif a.limit: + sidecars = sidecars[: a.limit] + if not sidecars: + raise SystemExit(f"no sidecars under {IDENTIFY_DIR}") + + # Resume: a full sweep is ~1 h, so never redo work an interrupted run already did. + done: set[str] = set() + if a.restart and a.out.exists(): + a.out.unlink() # --restart must TRUNCATE; the file is reopened in append mode below + if a.out.exists() and not a.restart: + with open(a.out, encoding="utf-8") as fh: + for line in fh: + try: + done.add(json.loads(line)["uid"]) + except Exception: # noqa: S112 -- tolerate a torn last line from an interrupted run + continue + todo = [s for s in sidecars if os.path.basename(s)[:-5] not in done] + print(f"sidecars {len(sidecars)} already done {len(done)} to do {len(todo)} workers {a.workers}") + + counts: collections.Counter[str] = collections.Counter() + a.out.parent.mkdir(parents=True, exist_ok=True) + with open(a.out, "a", encoding="utf-8") as fh, ProcessPoolExecutor(max_workers=a.workers) as ex: + futures = {ex.submit(_one, s): s for s in todo} + for i, fut in enumerate(as_completed(futures), 1): + rec = fut.result() + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + if rec["status"] != "ok": + counts[rec["status"]] += 1 + else: + for c in rec["classes"]: + counts[c] += 1 + if i % 500 == 0: + fh.flush() + print(f" {i}/{len(todo)} " + " ".join(f"{k}={v}" for k, v in counts.most_common(6)), flush=True) + + print(f"\n{'=' * 70}\nSIDECAR REGRESSION processed={len(todo)}\n{'=' * 70}") + for k, v in counts.most_common(): + flag = " <-- REGRESSION" if k.startswith("lost_") or k in ("identify_raised",) else "" + print(f" {k:22s} {v:7d}{flag}") + print(f"\nfull records: {a.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke_matrix.py b/scripts/smoke_matrix.py new file mode 100644 index 0000000..580ff07 --- /dev/null +++ b/scripts/smoke_matrix.py @@ -0,0 +1,621 @@ +"""Release smoke matrix: exercise every CLI parameter CHOICE on REAL data. + +Run before a release. This is not a unit-test substitute -- it drives the real CLI as a +subprocess, so it covers argument parsing, exit-code semantics and the file-writing +contracts that unit tests with fakes cannot. + +WHAT IT COVERS AND WHY THAT SHAPE + * Every choice-valued flag at least once (`--backend`, `--sensitivity`, `--mark`, + `--mode`, `--inpaint-method`, ...). Full permutation is combinatorially large and + mostly meaningless; dead branches and typos hide in the CHOICES, not in their + products. + * Every input FORMAT and shape edge case through the pixel paths -- PNG/JPEG/WebP/ + HEIC/AVIF, alpha, unicode names, misnamed extensions, truncated files, tiny and + landscape frames. Real-world breakage lives here far more than in flag combos. + * CONTRACTS, not just exit codes: `visible` must write NO output and exit 2 when no + mark is found (re-serving the input reads as success -- the recurring "it didn't + work" report); a no-op must be byte-identical; `metadata --remove` must actually + strip; a JPEG strip must not touch the pixels. Note the pixel-lossless contract is + the DEFAULT path's -- `--remove-all` deliberately re-encodes (see metadata.py). + * The diffusion bodies under `--diffusion`, at `--max-resolution 512` so they fit MPS + (~1 min/image on 32 GB unified memory). Not just exit codes: `invisible` must + restore the input resolution and must NOT re-stamp SDXL's own open watermark. + +WHAT IT DOES NOT COVER, DELIBERATELY AND LOUDLY + * Without `--diffusion`, the model-running bodies are reported as SKIPPED with a + reason, never as passes -- a green run that quietly skipped half the surface is + worse than a red one. + * Removal STRENGTH is never certified here. Whether a watermark is actually gone + needs the per-vendor oracles (docs/known-limitations.md); these rows prove the + paths run and keep their contracts, nothing more. + * The re-embed row is gated on a POSITIVE CONTROL. imwatermark is positive-only and + fails to round-trip on some pristine carriers, so "no watermark found" proves + nothing there; the row degrades to a skip rather than a false pass. + + uv run python scripts/smoke_matrix.py # corpus + fixtures + uv run python scripts/smoke_matrix.py --quick # fixtures only, no corpus + uv run python scripts/smoke_matrix.py --diffusion # + the SDXL model paths +""" + +from __future__ import annotations + +import argparse +import json +import random +import shutil +import subprocess +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +SAMPLES = REPO / "data" / "samples" +CORPUS = REPO / "data" / "spaces" / "originals" +EXIT_NO_VISIBLE_MARK = 2 + + +def _capture(args: list[str]) -> str: + """Run the CLI and return stdout (for the rows that inspect output, not exit code).""" + exe = shutil.which("uv") or "uv" + p = subprocess.run( # noqa: S603 + [exe, "run", "remove-ai-watermarks", *args], capture_output=True, text=True, cwd=REPO, check=False + ) + return p.stdout + + +@dataclass +class Result: + name: str + status: str # pass | FAIL | skip + detail: str = "" + cmd: str = "" + + +@dataclass +class Runner: + tmp: Path + results: list[Result] = field(default_factory=list) + + def run(self, name: str, args: list[str], *, expect_exit: int | None = 0, timeout: int = 180) -> Result: + exe = shutil.which("uv") or "uv" + cmd = [exe, "run", "remove-ai-watermarks", *args] + try: + p = subprocess.run( # noqa: S603 + cmd, capture_output=True, text=True, timeout=timeout, cwd=REPO, check=False + ) + except subprocess.TimeoutExpired: + r = Result(name, "FAIL", f"timeout after {timeout}s", " ".join(args)) + self.results.append(r) + return r + ok = expect_exit is None or p.returncode == expect_exit + detail = "" if ok else f"exit {p.returncode} (want {expect_exit}): {(p.stderr or p.stdout).strip()[-200:]}" + r = Result(name, "pass" if ok else "FAIL", detail, " ".join(args)) + self.results.append(r) + return r + + def check(self, name: str, ok: bool, detail: str = "") -> None: + self.results.append(Result(name, "pass" if ok else "FAIL", "" if ok else detail)) + + def skip(self, name: str, why: str) -> None: + self.results.append(Result(name, "skip", why)) + + +def corpus_pick(n: int, suffixes: tuple[str, ...]) -> list[Path]: + """Real uploads, chosen deterministically so a failure is reproducible.""" + if not CORPUS.exists(): + return [] + pool = [p for p in CORPUS.glob("*/*") if p.suffix.lower() in suffixes] + random.Random(7).shuffle(pool) # noqa: S311 -- deterministic sampling, not cryptography + return pool[:n] + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--quick", action="store_true", help="fixtures only; skip the corpus rows") + ap.add_argument( + "--diffusion", action="store_true", help="also run the model-running paths (SDXL weights, ~1 min/image)" + ) + a = ap.parse_args() + + tmp = Path(tempfile.mkdtemp(prefix="raiw-smoke-")) + r = Runner(tmp) + doubao = SAMPLES / "doubao-1.png" + chatgpt = SAMPLES / "chatgpt-1.png" + grok = SAMPLES / "grok-1.jpg" + + # ---- identify: every flag, and JSON must actually parse ------------------- + r.run("identify plain", ["identify", str(doubao)]) + r.run("identify --no-visible", ["identify", str(doubao), "--no-visible"]) + try: + json.loads(_capture(["identify", str(doubao), "--json"])) + r.check("identify --json parses", True) + except Exception as e: + r.check("identify --json parses", False, str(e)) + + # ---- metadata: check/remove, and the strip must be REAL ------------------ + r.run("metadata --check", ["metadata", str(doubao), "--check"]) + out = tmp / "meta.png" + r.run("metadata --remove", ["metadata", str(doubao), "--remove", "-o", str(out)]) + if out.exists(): + try: + rep = json.loads(_capture(["identify", str(out), "--json"])) + # Scope the assert to METADATA. doubao-1 also carries a visible pixel mark, + # which `metadata --remove` must NOT touch -- demanding an empty watermark + # list would fail on correct behavior (and hide a real metadata leak behind + # a permanently-red row). + leftover = [s for s in rep.get("signals", []) if not s.get("name", "").startswith("visible_")] + r.check( + "metadata --remove strips every metadata signal", + not leftover and not rep.get("ai_from_metadata"), + f"still reports metadata signals {[s.get('name') for s in leftover]}", + ) + r.check( + "metadata --remove leaves the visible mark alone", + any(s.get("name", "").startswith("visible_") for s in rep.get("signals", [])), + "the pixel mark vanished -- a metadata strip must not touch pixels", + ) + except Exception as e: + r.check("metadata --remove strips every metadata signal", False, str(e)) + + # The pixel-lossless contract belongs to the DEFAULT path. `--remove-all` + # (keep_standard=False) deliberately falls through to the full PIL re-encode + # (metadata.py: the lossless marker walk preserves standard segments, so it cannot + # serve a strip-everything caller), which is lossy for JPEG -- measured ~49 dB. + # Asserting losslessness there tested a contract the code never made. + import numpy as np + + from remove_ai_watermarks.image_io import imread + + out2 = tmp / "meta2.jpg" + r.run("metadata --remove (jpeg, default)", ["metadata", str(grok), "--remove", "-o", str(out2)]) + if out2.exists(): + a_, b_ = imread(str(grok)), imread(str(out2)) + same = a_ is not None and b_ is not None and a_.shape == b_.shape and bool(np.array_equal(a_, b_)) + r.check("jpeg metadata strip is pixel-lossless", same, "pixels changed -- the strip re-encoded") + out3 = tmp / "meta3.jpg" + r.run("metadata --remove --remove-all (jpeg)", ["metadata", str(grok), "--remove", "--remove-all", "-o", str(out3)]) + if out3.exists(): + rep3 = json.loads(_capture(["identify", str(out3), "--json"])) + r.check( + "--remove-all strips the AI metadata too", + not [s for s in rep3.get("signals", []) if not s.get("name", "").startswith("visible_")], + f"still reports {rep3.get('signals')}", + ) + + # ---- visible: every --mark choice, every --backend, every --sensitivity --- + for mark in ("auto", "gemini", "doubao", "jimeng", "samsung", "jimeng_pill"): + # doubao-1 carries only the doubao mark, so every other --mark must exit 2 + want = 0 if mark in ("auto", "doubao") else EXIT_NO_VISIBLE_MARK + r.run( + f"visible --mark {mark}", + ["visible", str(doubao), "--mark", mark, "-o", str(tmp / f"m_{mark}.png")], + expect_exit=want, + ) + # Every backend CHOICE, not just the two that need no extra. migan/lama were + # measured at library level (scripts/fill_quality.py) but had never been driven + # through the CLI, which is a different code path (resolve_backend + the warning). + from remove_ai_watermarks.region_eraser import lama_available, migan_available + + for backend in ("auto", "cv2", "migan", "lama"): + have = {"migan": migan_available(), "lama": lama_available()}.get(backend, True) + if not have: + r.skip(f"visible --backend {backend}", f"the `{backend}` extra is not installed") + continue + r.run( + f"visible --backend {backend}", + ["visible", str(doubao), "--backend", backend, "-o", str(tmp / f"b_{backend}.png")], + timeout=600, + ) + for sens in ("auto", "strict"): + r.run( + f"visible --sensitivity {sens}", + ["visible", str(doubao), "--sensitivity", sens, "-o", str(tmp / f"s_{sens}.png")], + ) + r.run("visible --keep-metadata", ["visible", str(doubao), "--keep-metadata", "-o", str(tmp / "keep.png")]) + r.run("visible --no-detect (forced)", ["visible", str(doubao), "--no-detect", "-o", str(tmp / "force.png")]) + for removed in ("assume-ai", "aggressive"): + r.run( + f"visible rejects --sensitivity {removed}", + ["visible", str(doubao), "--sensitivity", removed], + expect_exit=2, + ) + + # CONTRACT: no mark -> no output file, exit 2 (never re-serve the input as success) + noout = tmp / "must_not_exist.png" + r.run("visible no-mark exits 2", ["visible", str(chatgpt), "-o", str(noout)], expect_exit=EXIT_NO_VISIBLE_MARK) + r.check("visible no-mark writes NO output", not noout.exists(), "wrote an output for an undetected mark") + + # ---- erase: backends, methods, repeated regions, dilate ------------------ + for method in ("telea", "ns"): + r.run( + f"erase --inpaint-method {method}", + [ + "erase", + str(doubao), + "--region", + "10,10,60,30", + "--inpaint-method", + method, + "-o", + str(tmp / f"e_{method}.png"), + ], + ) + r.run( + "erase repeated --region", + ["erase", str(doubao), "--region", "10,10,40,20", "--region", "80,80,40,20", "-o", str(tmp / "e_multi.png")], + ) + r.run( + "erase --dilate", + ["erase", str(doubao), "--region", "10,10,40,20", "--dilate", "5", "-o", str(tmp / "e_dil.png")], + ) + for backend in ("cv2", "migan", "lama"): + have = {"migan": migan_available(), "lama": lama_available()}.get(backend, True) + if not have: + r.skip(f"erase --backend {backend}", f"the `{backend}` extra is not installed") + continue + r.run( + f"erase --backend {backend}", + [ + "erase", + str(doubao), + "--region", + "10,10,40,20", + "--backend", + backend, + "-o", + str(tmp / f"e_{backend}.png"), + ], + timeout=600, + ) + r.run( + "erase --dilate 0 (no dilation)", + ["erase", str(doubao), "--region", "10,10,40,20", "--dilate", "0", "-o", str(tmp / "e_d0.png")], + ) + r.run("erase rejects a malformed --region", ["erase", str(doubao), "--region", "not,a,box"], expect_exit=2) + + # ---- global + explicit-default flags that had never been exercised ------- + r.run("--verbose", ["--verbose", "identify", str(doubao)]) + r.run("--version", ["--version"]) + r.run("metadata --keep-standard (explicit)", ["metadata", str(doubao), "--check", "--keep-standard"]) + r.run( + "visible --strip-metadata (explicit)", + ["visible", str(doubao), "--strip-metadata", "-o", str(tmp / "sm.png")], + ) + r.run("visible --detect (explicit default)", ["visible", str(doubao), "--detect", "-o", str(tmp / "det.png")]) + + # ---- batch: the non-diffusion modes ------------------------------------- + bd = tmp / "batch_in" + bd.mkdir() + for f in (doubao, chatgpt): + shutil.copy(f, bd / f.name) + for mode in ("visible", "metadata"): + r.run( + f"batch --mode {mode}", + ["batch", str(bd), "--mode", mode, "-o", str(tmp / f"batch_{mode}")], + expect_exit=None, + ) + + # ---- diffusion: argument handling always; the model body under --diffusion ---- + r.run("invisible --help parses full knob set", ["invisible", "--help"]) + r.run("all --help parses full knob set", ["all", "--help"]) + _media_rows(r, tmp) + if a.diffusion: + _diffusion_rows(r, tmp, doubao) + else: + for name in ("invisible", "all", "batch --mode invisible"): + r.skip(f"{name} (model-running body)", "pass --diffusion to exercise it (needs the SDXL weights)") + + # ---- real-data formats and shapes --------------------------------------- + if not a.quick: + picks: list[tuple[str, Path]] = [] + for suf, label in ((".heic", "heic"), (".avif", "avif"), (".webp", "webp"), (".jpeg", "jpeg")): + picks += [(label, p) for p in corpus_pick(2, (suf,))] + picks += [("png", p) for p in corpus_pick(3, (".png",))] + if not picks: + r.skip("real-format rows", "corpus not present (data/spaces/originals)") + for label, p in picks: + r.run(f"identify real {label}", ["identify", str(p), "--json"]) + r.run( + f"visible auto real {label}", + ["visible", str(p), "-o", str(tmp / f"r_{label}_{p.stem[:8]}.png")], + expect_exit=None, + ) # 0 or 2 are both correct; a CRASH is not + + # unicode + misnamed extension + truncated: the documented real-world traps + if picks: + src = picks[0][1] + uni = tmp / "тест изображение 测试.png" + shutil.copy(src, uni) + r.run("unicode filename", ["identify", str(uni), "--json"]) + mis = tmp / "actually_png.jpg" # content PNG, extension JPEG + shutil.copy(SAMPLES / "chatgpt-1.png", mis) + r.run("misnamed extension", ["identify", str(mis), "--json"]) + trunc = tmp / "truncated.png" + trunc.write_bytes((SAMPLES / "chatgpt-1.png").read_bytes()[:4096]) + r.run("truncated file does not crash", ["identify", str(trunc), "--json"], expect_exit=None) + + # ---- report -------------------------------------------------------------- + bad = [x for x in r.results if x.status == "FAIL"] + skipped = [x for x in r.results if x.status == "skip"] + ok = [x for x in r.results if x.status == "pass"] + print(f"\n{'=' * 74}\nSMOKE MATRIX pass={len(ok)} FAIL={len(bad)} skipped={len(skipped)}\n{'=' * 74}") + for x in skipped: + print(f" SKIP {x.name:46s} {x.detail}") + for x in bad: + print(f" FAIL {x.name:46s} {x.detail}") + if x.cmd: + print(f" cmd: {x.cmd}") + if not bad: + print(" no failures") + print(f"\ntmp artifacts: {tmp}") + raise SystemExit(1 if bad else 0) + + +def _knob_rows(r: Runner, tmp: Path, img: Path) -> None: + """Every diffusion knob the matrix never touched. + + Deliberately cheap (`--steps 4`, `--max-resolution 384`): these rows answer "is the + knob accepted and does the run complete", NOT "is the output good". Quality per knob + needs a per-knob oracle and most of them have none (`--humanize` has no oracle at + all), so claiming more here would be dishonest. + """ + from remove_ai_watermarks import upscaler + + # --steps 20 is the floor that WORKS, not an arbitrary choice: effective timesteps + # are int(steps * strength), so at the default strength 0.15 anything below + # --steps 7 rounds to ZERO and the pipeline dies inside torch. The first version of + # these rows used --steps 4 and every single one failed with the same reshape error. + fast = ["--max-resolution", "384", "--min-resolution", "0", "--steps", "20", "--force", "--seed", "0"] + + def run(name: str, extra: list[str], *, tag: str, expect: int | None = 0) -> None: + r.run( + name, + ["invisible", str(img), "-o", str(tmp / f"k_{tag}.png"), *fast, *extra], + expect_exit=expect, + timeout=2400, + ) + + run("--pipeline sdxl", ["--pipeline", "sdxl"], tag="sdxl") + run("--pipeline controlnet", ["--pipeline", "controlnet"], tag="cnet") + run("--strength", ["--strength", "0.2"], tag="strength") + run("--guidance-scale", ["--guidance-scale", "5.0"], tag="gs") + run("--controlnet-scale", ["--controlnet-scale", "0.5"], tag="cns") + run("--humanize", ["--humanize", "0.3"], tag="hum") + run("--unsharp", ["--unsharp", "0.5"], tag="uns") + run("--no-adaptive-polish", ["--no-adaptive-polish"], tag="nap") + run("--tile", ["--tile", "--tile-size", "256", "--tile-overlap", "64"], tag="tile") + run("--device mps", ["--device", "mps"], tag="mps") + run("--upscaler lanczos", ["--upscaler", "lanczos"], tag="lanczos") + run("--auto (deprecated no-op)", ["--auto"], tag="auto") + + if upscaler.is_available(): + run("--upscaler esrgan", ["--upscaler", "esrgan"], tag="esrgan") + else: + r.skip("--upscaler esrgan", "the `esrgan` extra is not installed") + + # CPU is correctness-relevant (it is the documented MPS-OOM fallback) but slow, so + # it gets the smallest possible run rather than being skipped. + r.run( + "--device cpu", + [ + "invisible", + str(img), + "-o", + str(tmp / "k_cpu.png"), + "--device", + "cpu", + "--max-resolution", + "256", + "--min-resolution", + "0", + "--steps", + "20", + "--force", + "--seed", + "0", + ], + timeout=3600, + ) + + # qwen is CUDA-class by design (bf16 MMDiT, no MPS fallback). On this host the + # honest outcome is a CLEAN failure, not a crash -- assert it does not hang or + # dump a traceback at the user. + try: + import torch + + cuda = bool(torch.cuda.is_available()) + except Exception: + cuda = False + if cuda: + run("--pipeline qwen", ["--pipeline", "qwen"], tag="qwen") + else: + r.skip("--pipeline qwen", "CUDA-class pipeline; no CUDA device on this host") + + # --model and --hf-token are deliberately not exercised: one would download a second + # multi-GB checkpoint, the other needs a real credential. Skipped loudly, not passed. + r.skip("--model", "would download a second multi-GB checkpoint") + r.skip("--hf-token", "needs a real credential; cannot be exercised meaningfully here") + + # CONTRACT, not just execution: the same seed must reproduce the same pixels. + a_out, b_out = tmp / "seed_a.png", tmp / "seed_b.png" + for out in (a_out, b_out): + r.run( + f"seed determinism run ({out.name})", + ["invisible", str(img), "-o", str(out), *fast], + timeout=2400, + ) + if a_out.exists() and b_out.exists(): + import numpy as np + + from remove_ai_watermarks.image_io import imread + + x, y = imread(str(a_out)), imread(str(b_out)) + r.check( + "same --seed reproduces identical pixels", + x is not None and y is not None and x.shape == y.shape and bool(np.array_equal(x, y)), + "two runs with the same seed differed", + ) + + +def _media_rows(r: Runner, tmp: Path) -> None: + """Audio/video metadata strip via ffmpeg -- a supported path with no corpus coverage. + + The corpus is images only, so the media is synthesized here with ffmpeg rather than + left untested. + """ + if not shutil.which("ffmpeg"): + r.skip("audio/video metadata strip", "ffmpeg not on PATH") + return + for name, gen in ( + ("mp4", ["-f", "lavfi", "-i", "testsrc=duration=1:size=128x128:rate=8", "-pix_fmt", "yuv420p"]), + ("mp3", ["-f", "lavfi", "-i", "sine=frequency=440:duration=1"]), + ): + src = tmp / f"media.{name}" + ff = shutil.which("ffmpeg") or "ffmpeg" + made = subprocess.run( # noqa: S603 + [ff, "-y", *gen, "-metadata", "comment=Made with AI", str(src)], + capture_output=True, + check=False, + ) + if made.returncode != 0 or not src.exists(): + r.skip(f"{name} metadata strip", "ffmpeg could not synthesize the fixture") + continue + out = tmp / f"media_clean.{name}" + r.run(f"{name} metadata strip runs", ["metadata", str(src), "--remove", "-o", str(out)], expect_exit=None) + if out.exists(): + r.check(f"{name} strip produced a non-empty file", out.stat().st_size > 0, "empty output") + + +def _sdxl_watermark_bits(img: object) -> float: + """Bits of the open SDXL DWT-DCT watermark recovered from `img` (128 = perfect).""" + import numpy as np + from imwatermark import WatermarkDecoder + + truth = np.frombuffer(b"StableDiffusionV1"[:16], dtype=np.uint8) + rec = WatermarkDecoder("bytes", 128).decode(img, "dwtDct") + return float(128 - np.unpackbits(truth ^ np.frombuffer(bytes(rec), dtype=np.uint8)).sum()) + + +def _diffusion_rows(r: Runner, tmp: Path, doubao: Path) -> None: + """Exercise the model-running bodies at a reduced resolution (MPS-friendly). + + Bounded with `--max-resolution 512` and a fixed seed: the point is that the paths + RUN and keep their contracts, not to certify removal strength (that needs the + per-vendor oracles, see docs/known-limitations.md). + """ + import numpy as np + + from remove_ai_watermarks.image_io import imread + + small = ["--max-resolution", "512", "--seed", "0"] + + # `invisible` must restore the original resolution and NOT re-stamp SDXL's own + # open watermark (add_watermarker=False; a remover that re-marks its output is the + # regression this row exists for). + # + # The carrier matters: imwatermark is positive-only and fails to round-trip on some + # pristine images, so an "absent" verdict on a fragile carrier proves NOTHING. mj-1 + # is used because it round-trips at 128/128; the control is re-checked on the actual + # OUTPUT and the row degrades to a skip rather than a false pass if it goes fragile. + mj = SAMPLES / "mj-1.png" + inv = tmp / "inv_mj.png" + res = r.run("invisible runs (mps, 512px)", ["invisible", str(mj), "-o", str(inv), "--force", *small], timeout=1800) + if res.status == "pass" and inv.exists(): + src_img, out_img = imread(str(mj)), imread(str(inv)) + r.check( + "invisible restores the input resolution", + src_img is not None and out_img is not None and src_img.shape == out_img.shape, + f"{None if src_img is None else src_img.shape} -> {None if out_img is None else out_img.shape}", + ) + try: + from imwatermark import WatermarkEncoder + + enc = WatermarkEncoder() + enc.set_watermark("bytes", b"StableDiffusionV1"[:16]) + control = _sdxl_watermark_bits(enc.encode(np.array(out_img).copy(), "dwtDct")) + if control < 118: + r.skip("invisible does not re-embed an SDXL watermark", f"carrier fragile (control {control:.0f}/128)") + else: + bits = _sdxl_watermark_bits(out_img) + r.check( + "invisible does not re-embed an SDXL watermark", + bits < 118, + f"re-embedded: {bits:.0f}/128 recovered (control {control:.0f}/128)", + ) + except ImportError: + r.skip("invisible does not re-embed an SDXL watermark", "imwatermark absent (extra `detect`)") + + # `all`: every stage must land -- the visible mark AND the metadata both gone. + allout = tmp / "all_out.png" + res = r.run("all runs (mps, 512px)", ["all", str(doubao), "-o", str(allout), *small], timeout=1800) + if res.status == "pass" and allout.exists(): + rep = json.loads(_capture(["identify", str(allout), "--json"])) + r.check( + "all clears visible + metadata in one pass", + not rep.get("signals"), + f"still reports {[s.get('name') for s in rep.get('signals', [])]}", + ) + + _knob_rows(r, tmp, mj) + + # `batch --mode invisible`: every input must produce an output (a silent short + # write is the failure this row guards). + bd = tmp / "batch_inv" + bd.mkdir(exist_ok=True) + for f in (SAMPLES / "chatgpt-2.png", mj): + shutil.copy(f, bd / f.name) + bout = tmp / "batch_inv_out" + res = r.run( + "batch --mode invisible runs (mps, 512px)", + ["batch", str(bd), "--mode", "invisible", "-o", str(bout), *small], + expect_exit=None, + timeout=3600, + ) + if res.status == "pass": + produced = len(list(bout.glob("*"))) if bout.exists() else 0 + r.check("batch invisible writes one output per input", produced == 2, f"{produced} outputs for 2 inputs") + + # `batch --mode all` -- the only --mode value the matrix never ran. + aout = tmp / "batch_all_out" + r.run( + "batch --mode all runs (mps, 512px)", + ["batch", str(bd), "--mode", "all", "-o", str(aout), *small], + expect_exit=None, + timeout=3600, + ) + + # The AI-enhanced composite path: regenerate ONLY a region and feather it back, + # leaving everything outside the box pixel-exact. Library-level -- the CLI has no + # flag for it, so it would otherwise never be exercised on real data. + try: + import numpy as np + + from remove_ai_watermarks.image_io import imread + from remove_ai_watermarks.noai.watermark_remover import WatermarkRemover + + src = imread(str(mj)) + h, w = src.shape[:2] + box = (w // 4, h // 4, w // 4, h // 4) + rem = WatermarkRemover(pipeline="controlnet") + rout = tmp / "region_composite.png" + rem.remove_watermark(mj, rout, strength=0.15, num_inference_steps=20, seed=0, region=box) + got = imread(str(rout)) + if got is None or got.shape != src.shape: + r.check("region composite keeps the frame outside the box", False, "shape changed or unreadable") + else: + mask = np.ones(src.shape[:2], dtype=bool) + x, y, bw, bh = box + # Outside the box PLUS the feather margin must be untouched. + pad = 96 + mask[max(0, y - pad) : y + bh + pad, max(0, x - pad) : x + bw + pad] = False + r.check( + "region composite keeps the frame outside the box", + bool(np.array_equal(src[mask], got[mask])), + "pixels changed outside the regenerated region", + ) + except Exception as e: + r.skip("region composite (remove_watermark(region=...))", f"{type(e).__name__}: {e}"[:120]) + + +if __name__ == "__main__": + main() diff --git a/scripts/visible_positives.py b/scripts/visible_positives.py new file mode 100644 index 0000000..b630eb8 --- /dev/null +++ b/scripts/visible_positives.py @@ -0,0 +1,141 @@ +"""Parallel detection pass: list every corpus image carrying a known visible mark. + +Why this exists separately from `visible_removal_audit.py`: that audit is single-process, +so a full-corpus sweep costs ~10 h and running it once per backend costs ~30 h. But its +expensive half is DETECTION, and detection does not depend on the fill backend. Splitting +it out means detecting once in parallel and then feeding the positives to the audit via +its `--paths-file` seam, over a few thousand images instead of forty thousand. + +CRASH TOLERANCE IS NOT OPTIONAL AT THIS SCALE + cv2/libpng decode native-crash on some real uploads. A plain `ProcessPoolExecutor.map` + over 39k files then DEADLOCKS: the worker dies without a Python traceback and the parent + waits forever on a result that never arrives (observed 2026-07-19 -- 26 min of work lost + because results were only written at the end). So this script: + * writes every result to JSONL as it arrives -- a kill never costs more than a batch; + * is resumable -- an interrupted run skips what it already recorded; + * runs a FRESH pool per batch with a timeout, so one poisoned file costs one batch, + and that batch is retried serially to find and record the offender. + +Corpus images are user uploads: read-only, local analysis, gitignored output. + + uv run python scripts/visible_positives.py --jobs 6 +""" + +from __future__ import annotations + +import argparse +import collections +import glob +import json +import os +import sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FutureTimeout +from concurrent.futures.process import BrokenProcessPool +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# The package's own format set. An inlined copy here silently skipped .heif, which +# CLAUDE.md documents as supported. +from remove_ai_watermarks.noai.constants import SUPPORTED_FORMATS as _EXTS + +REPO = Path(__file__).resolve().parents[1] +CORPUS = REPO / "data" / "spaces" / "originals" +OUT = REPO / "data" / "spaces" / "_visible_positives.jsonl" +PATHS = REPO / "data" / "spaces" / "_visible_positives.txt" + + +def _one(path: str) -> dict[str, object]: + from remove_ai_watermarks.image_io import imread + from remove_ai_watermarks.watermark_registry import detect_marks + + try: + img = imread(path) + if img is None: + return {"path": path, "keys": [], "status": "unreadable"} + return {"path": path, "keys": [d.key for d in detect_marks(img) if d.detected], "status": "ok"} + except Exception as e: + return {"path": path, "keys": [], "status": f"error:{type(e).__name__}"} + + +def _run_batch(batch: list[str], jobs: int, timeout: int) -> list[dict[str, object]]: + """One batch in a fresh pool. On a native worker crash or timeout, retry serially so + the poisoned file is identified and recorded instead of stalling the whole sweep.""" + try: + with ProcessPoolExecutor(max_workers=jobs) as ex: + futs = {ex.submit(_one, p): p for p in batch} + return [f.result() for f in as_completed(futs, timeout=timeout)] + except (FutureTimeout, BrokenProcessPool, OSError, RuntimeError): + out: list[dict[str, object]] = [] + for p in batch: + try: + out.append(_one(p)) + except BaseException: # a native crash here kills only this file, not the sweep + out.append({"path": p, "keys": [], "status": "crashed"}) + return out + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2)) + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--batch", type=int, default=200) + ap.add_argument("--timeout", type=int, default=600, help="seconds per batch before falling back to serial") + ap.add_argument("--out", type=Path, default=OUT) + ap.add_argument("--restart", action="store_true") + a = ap.parse_args() + + files = sorted(p for p in glob.glob(str(CORPUS / "*" / "*")) if Path(p).suffix.lower() in _EXTS) + if a.limit: + files = files[: a.limit] + + done: set[str] = set() + if a.restart and a.out.exists(): + a.out.unlink() # --restart must TRUNCATE; the file is reopened in append mode below + if a.out.exists() and not a.restart: + with open(a.out, encoding="utf-8") as fh: + for line in fh: + try: + done.add(json.loads(line)["path"]) + except Exception: # noqa: S112 -- tolerate a torn last line + continue + todo = [p for p in files if p not in done] + print(f"images {len(files)} already done {len(done)} to do {len(todo)} jobs {a.jobs}", flush=True) + + counts: collections.Counter[str] = collections.Counter() + status: collections.Counter[str] = collections.Counter() + seen = 0 + with open(a.out, "a", encoding="utf-8") as fh: + for start in range(0, len(todo), a.batch): + for rec in _run_batch(todo[start : start + a.batch], a.jobs, a.timeout): + fh.write(json.dumps(rec) + "\n") + status[str(rec["status"])] += 1 + for k in rec["keys"]: # type: ignore[union-attr] + counts[str(k)] += 1 + seen += 1 + fh.flush() + print(f" {seen}/{len(todo)} {dict(counts)} {dict(status)}", flush=True) + + # Rebuild the paths file from the FULL record set, not just this run's slice. + hits = [] + with open(a.out, encoding="utf-8") as fh: + for line in fh: + try: + r = json.loads(line) + except Exception: # noqa: S112 + continue + if r.get("keys"): + hits.append(r["path"]) + # Derive the paths file from --out so a trial run with a scratch --out cannot + # overwrite the shared list a full sweep produced. + paths_out = a.out.with_suffix(".txt") + paths_out.write_text("\n".join(sorted(set(hits))) + "\n", encoding="utf-8") + print(f"\npositives: {len(set(hits))} images") + for k, v in counts.most_common(): + print(f" {v:6d} {k}") + print(f"statuses: {dict(status)}\npaths -> {paths_out}\nrecords -> {a.out}") + + +if __name__ == "__main__": + main()