diff --git a/data/README.md b/data/README.md index 90d1496..db530e5 100644 --- a/data/README.md +++ b/data/README.md @@ -17,6 +17,8 @@ data/ full-pipeline-quality.csv Reusable full-pipeline evaluation selection evaluations/ + engine-selection/ + Prompt-matched AI content matrix for deterministic auto-engine research fidelity/ Evaluation instructions and hand-verified ground truth video-synthid-oracle.csv Reproducible full-clip Gemini SynthID verdicts @@ -34,6 +36,9 @@ data/ commands, hashes, and oracle verdicts instead of committing another corpus copy. A small curated before-and-after example may live in `docs/images/` when it is part of the public documentation. + `evaluations/engine-selection/` is the bounded exception for ORIGINAL study + inputs: its manifest tracks a prompt-matched OpenAI/Meta content matrix, not + generated model outputs or watermark-oracle evidence. 6. Keep third-party fixture license notices in `licenses/`, outside directories that tests enumerate as media inputs. 7. Runtime detector assets belong in `src/remove_ai_watermarks/assets/`. diff --git a/data/evaluations/engine-selection/README.md b/data/evaluations/engine-selection/README.md new file mode 100644 index 0000000..39ece27 --- /dev/null +++ b/data/evaluations/engine-selection/README.md @@ -0,0 +1,61 @@ +# Engine-selection fixture set + +This directory is the tracked, content-balanced input set for researching a +deterministic `auto` engine choice. It is not a training corpus and it is not a +watermark-oracle corpus. + +## Content matrix + +`content-manifest.csv` contains 19 prompt-matched pairs. Each pair uses the +same prompt and generation index across OpenAI `gpt-image-1-mini` and Meta +`muse-image-1.0`, covering the content classes that materially stress image +regeneration: + +- natural and urban scenes; +- single-person portraits and people in context; +- animals, food, products, interiors, architecture, macro detail, and action; +- flat, isometric, watercolor, and surreal illustration styles. + +The files are exact copies of generation outputs collected for the +`docs/arxiv-paper-review` research branch. The manifest records the SHA-256 of +the committed bytes. The old OpenAI collection manifest stored the API payload +hash under `sha256`; `source_payload_sha256` preserves that value while +`sha256` always means the actual file bytes here. Meta's two hashes are the +same. + +One generation per prompt and provider is enough for the first paired +discovery pass. Any policy inferred from it must be challenged against unused +generation indices from the local research archive before shipping. Generated +outputs belong outside this directory. + +`carrier-manifest.csv` adds the canonical, already tracked signal carriers to +the study without copying their bytes. Together the two manifests cover +ordinary content, text, single faces, face grids, and mixed face/text scenes. + +## Signal carriers + +The content matrix was re-encoded by its collection pipeline and has no local +provenance signal. It must not be used to set or certify watermark-removal +strengths. The tracked oracle/carrier inputs remain in their canonical homes: + +- OpenAI and Google SynthID: `data/synthid/originals/` with + `data/synthid/manifest.csv`; +- Meta Content Seal: `data/contentseal/originals/` with + `data/contentseal/manifest.csv`. + +There is no tracked, publication-cleared Microsoft InvisMark carrier yet. The +three Paint sources used by the earlier calibration are user uploads and remain +outside git. A Microsoft row may be added only from a newly generated or +otherwise publication-cleared source, with a pixel-identical metadata-stripped +control that remains positive in Microsoft's detector. + +## Integrity check + +Run: + +```bash +uv run python scripts/verify_engine_selection_fixtures.py +``` + +The check validates paths, byte hashes, dimensions, prompt pairing, and the +declared provider/content matrix without loading any ML model. diff --git a/data/evaluations/engine-selection/carrier-manifest.csv b/data/evaluations/engine-selection/carrier-manifest.csv new file mode 100644 index 0000000..b711942 --- /dev/null +++ b/data/evaluations/engine-selection/carrier-manifest.csv @@ -0,0 +1,13 @@ +provider,file,content_stratum,signal,verified_via,sha256,verified_text_geometry,verified_text_ground_truth +openai,"../../synthid/originals/ChatGPT Image May 30, 2026, 10_31_08 AM.png",face_grid,OpenAI SynthID,openai-verify,28ff8732b037f98a4ef5bc277bbcdaa32e5eb9ccbd00b6c8c616e46ef68ae8a0,no,no +openai,"../../synthid/originals/ChatGPT Image May 31, 2026, 02_02_23 PM.png",multilingual_typography,OpenAI SynthID,c2pa-metadata,228e54413a3ef62b313176dcdba774a56a58c36b54227d1440bc26cbe0aefd82,yes,yes +openai,"../../synthid/originals/ChatGPT Image May 31, 2026, 02_03_55 PM.png",multilingual_typography,OpenAI SynthID,openai-verify,05b836ecfe40fd689177fda74384ae4fdcc446505bbc4281cd3cbb6523eb669e,yes,yes +google,../../synthid/originals/Gemini_Generated_Image_633uuy633uuy633u.png,cjk_typography_scene,Google SynthID,gemini-app,4affd7f27767a445db6abf741355743ba8d95108ad922c9fff045feed8492236,yes,yes +google,../../synthid/originals/Gemini_Generated_Image_akdbeiakdbeiakdb.png,cyrillic_typography_scene,Google SynthID,gemini-app,8c1a6fb03ef3d45a1f958fb3401e4264e409ff88c2a793061db7f29023454d0e,no,no +google,../../synthid/originals/Gemini_Generated_Image_y48j3cy48j3cy48j.png,crowd_faces,Google SynthID,gemini-app,45d79a683134fcba1b147b2aedb669783d474e1fb8a4df329729a0904fd1b46b,no,no +google,../../synthid/originals/Gemini_Generated_Image_3mc4t93mc4t93mc4.png,faces_and_typography,Google SynthID,gemini-app,2c33e75a2db614ce74c83cc0a6ac6c3ac735aca83ab88c9c9345843b124f7856,no,no +meta,../../contentseal/originals/gen_fox_forest.webp,photoreal_texture,Meta Content Seal,meta-oracle,6d7cecf322a5ac4b2547e2f32c767b3cb3e8a0e49f274cfd00aa31be6697eed4,no,no +meta,../../contentseal/originals/gen_text_poster.webp,typography,Meta Content Seal,meta-oracle,873927662d8198e9c38b4ae71a2919ba5d417dfcd714649db1b0ae231e8cbf9b,no,no +meta,../../contentseal/originals/gen_lighthouse_watercolor.webp,watercolor,Meta Content Seal,meta-oracle,69bee7c376608568361085602c5c7406900885849bd94c5279c3924cf793b27c,no,no +meta,../../contentseal/originals/gen_night_city.webp,urban_night,Meta Content Seal,meta-oracle,345adf6b9f777b15ef7ce7bbdfb3b355a1d9e882740809e966e9f6caafd94d48,no,no +meta,../../contentseal/originals/gen_studio_mug.webp,product_still_life,Meta Content Seal,meta-oracle,2b7bc7fd56fa2250c66ef74f8b3e49d9666fe147d894d5baf9461b5e36077a07,no,no diff --git a/data/evaluations/engine-selection/content-manifest.csv b/data/evaluations/engine-selection/content-manifest.csv new file mode 100644 index 0000000..0367378 --- /dev/null +++ b/data/evaluations/engine-selection/content-manifest.csv @@ -0,0 +1,39 @@ +pair_id,provider,model,file,width,height,content_stratum,sha256,source_payload_sha256,prompt +00,openai,gpt-image-1-mini,originals/openai/00.png,1024,1024,natural_landscape,698896455e0f1de57a227339227bf068143e690873b9c8c9672b31578de93ba0,1345f59fc7b51a1edf837745245bb90811feefd9ba7f9ba6f12dc89846aeba18,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" +00,meta,muse-image-1.0,originals/meta/00.png,1600,1600,natural_landscape,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" +01,openai,gpt-image-1-mini,originals/openai/01.png,1024,1024,street_scene,b885d8cb1490e4fbbe580b87750008780671390097d28e71ff18b50cf320ec85,772518e4686940d2c68df114298fb007a97a5c42548245092e989b6ca9bd5b90,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" +01,meta,muse-image-1.0,originals/meta/01.png,1600,1600,street_scene,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" +02,openai,gpt-image-1-mini,originals/openai/02.png,1024,1024,single_portrait,97618d1a5c39ddeba708df3a9448b27d1d26583ee68b490ba79f7f3e1eb2248d,6bace9d2adae4d630e1830449f0945a30d7ddb3635e3005bc7aad80206a57176,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" +02,meta,muse-image-1.0,originals/meta/02.png,1600,1600,single_portrait,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" +03,openai,gpt-image-1-mini,originals/openai/03.png,1024,1024,animal_action,eb6920aab03bbe6935a01756c8f2d524d383e307369cdfaad3fe44926fbd9e48,2012abb15be639ebb3c596d2210f6d4c50cce0e11dd6cfc88aeb29de1c82de18,"a golden retriever running through a field of sunflowers, action photography" +03,meta,muse-image-1.0,originals/meta/03.png,1600,1600,animal_action,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,"a golden retriever running through a field of sunflowers, action photography" +04,openai,gpt-image-1-mini,originals/openai/04.png,1024,1024,reflective_architecture,275d68de881041de989261ced7570dd460028e0406346642d880d0d1664bb9ae,1808d95c69080aafced4234e4c3ad7714121bad00f3f5dc84a744288d9757245,"a modern architecture building with glass facade reflecting sunset clouds" +04,meta,muse-image-1.0,originals/meta/04.png,1600,1600,reflective_architecture,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,"a modern architecture building with glass facade reflecting sunset clouds" +05,openai,gpt-image-1-mini,originals/openai/05.png,1024,1024,food_closeup,316a743ccf29f025904d1bcd047a6996f987715e4f93e8a0d9cf85b624349b07,a8bac088c350ef849bbf78a85a1e44aa1665d1054a0cdfae29461c1cdca56e4a,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" +05,meta,muse-image-1.0,originals/meta/05.png,1600,1600,food_closeup,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" +06,openai,gpt-image-1-mini,originals/openai/06.png,1024,1024,sports_action,390d87d1f59eb4fa38590c0c751b2848383c77ffcf549571a0bf00db0f783763,6cc24eeb0fce95a86741b9ac0c055002793a586ba37f0cf4d64201a5c5e73513,"a surfer riding a barrel wave at golden hour, backlit spray" +06,meta,muse-image-1.0,originals/meta/06.png,1600,1600,sports_action,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,"a surfer riding a barrel wave at golden hour, backlit spray" +07,openai,gpt-image-1-mini,originals/openai/07.png,1024,1024,interior_low_light,4c77c2814216bdcde3417d2d228c5b6f275e33777f13cbea7911241bc9bc376b,9e858b95a4f842c917cbf47afeb4b7e0e24e04c4df0b19da4afe00b7316a13a9,"a cozy reading nook with warm lamp light and rain on the window" +07,meta,muse-image-1.0,originals/meta/07.png,1600,1600,interior_low_light,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,"a cozy reading nook with warm lamp light and rain on the window" +08,openai,gpt-image-1-mini,originals/openai/08.png,1024,1024,isometric_illustration,d688f6da74d4a7b2bd29ef59ba64864d0915134433febd8c90eb5ce1dc80781f,37a801f66b7c1bf63e613a7a5717dbc00397120a7a32ec45414661e03137a180,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" +08,meta,muse-image-1.0,originals/meta/08.png,1600,1600,isometric_illustration,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" +09,openai,gpt-image-1-mini,originals/openai/09.png,1024,1024,watercolor,d0beefb80ee00a0937a851932a2ed6dab2c670f10104ffe94c5ff22869a9595d,b52401235efc16641d3cb8f9c7c6d20a75c36751ada9409b65ceefa0e57901da,"a watercolor painting of Venice canals at sunset, loose brushstrokes" +09,meta,muse-image-1.0,originals/meta/09.png,1600,1600,watercolor,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,"a watercolor painting of Venice canals at sunset, loose brushstrokes" +10,openai,gpt-image-1-mini,originals/openai/10.png,1024,1024,macro_detail,59d9c6dce5ce4c2b77427144ccbafc8a4b7ab87564673677119859c31598b0d4,3740f9de304f050fc7306447c6668dcaa8156a888709f759f05715e7987a321c,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" +10,meta,muse-image-1.0,originals/meta/10.png,1600,1600,macro_detail,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" +11,openai,gpt-image-1-mini,originals/openai/11.png,1024,1024,urban_night,e4f6f806ed8559973374eae1e4d379807e45046749b7d491edbbc8ad9647728e,06bfc68289ece1b85b169df4cedd5e1cf12d0efa30631bdecd0e3edce4d8c82e,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" +11,meta,muse-image-1.0,originals/meta/11.png,1600,1600,urban_night,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" +12,openai,gpt-image-1-mini,originals/openai/12.png,1024,1024,flat_illustration,6bf230aa0b6f40d7ff0d23b22b8aac2e9b66288eb85c02eebdf43b71a1597f5b,65567d5cecb34241ac755ac4db5eb5a7b271d9a00fa27d826c961d2a2ef6ac3d,"a flat-design poster of Santorini with bold shapes, screen print style" +12,meta,muse-image-1.0,originals/meta/12.png,1600,1600,flat_illustration,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,"a flat-design poster of Santorini with bold shapes, screen print style" +13,openai,gpt-image-1-mini,originals/openai/13.png,1024,1024,storm_landscape,d07b05eef633fcb5b6b54268cc2cdd787dc19f8050198872e462f69e383a054d,a692a8a0daebad375cd57d7c671be6e16ad3ef700f0b9214daacbe80b6148c90,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" +13,meta,muse-image-1.0,originals/meta/13.png,1600,1600,storm_landscape,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" +14,openai,gpt-image-1-mini,originals/openai/14.png,1024,1024,monochrome_portrait,9fb1153f4d0041932b35bcd1408d774b642c68f81d7bb403d22f50f8c891ede7,826acf13df78378499e57c7fe1a8309947ab26810f46482be3a64d3328de1ff6,"a black and white portrait of an elderly craftsman in his workshop" +14,meta,muse-image-1.0,originals/meta/14.png,1600,1600,monochrome_portrait,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,"a black and white portrait of an elderly craftsman in his workshop" +15,openai,gpt-image-1-mini,originals/openai/15.png,1024,1024,surreal_composite,8bf02f9028fd049ebfafd3410d2eb9d618819be1c14b4e05d53a490e22ba109c,e71de5a9a2f6579857d2e24b8e7a1516031f19ee824906cece158101d13fe001,"a surreal double exposure of a forest and a human profile, conceptual art" +15,meta,muse-image-1.0,originals/meta/15.png,1600,1600,surreal_composite,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,"a surreal double exposure of a forest and a human profile, conceptual art" +16,openai,gpt-image-1-mini,originals/openai/16.png,1024,1024,brutalist_architecture,e91ca5128ce8644510c5c4def6bba085e45e1716105b3b6a2761cc18d42018b3,0aa8a1819f160ce26dfc9e59b996f969698fdf4eba3402fe999af598fcad80d4,"a low-angle shot of a brutalist building against dramatic clouds" +16,meta,muse-image-1.0,originals/meta/16.png,1600,1600,brutalist_architecture,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,"a low-angle shot of a brutalist building against dramatic clouds" +17,openai,gpt-image-1-mini,originals/openai/17.png,1024,1024,product_still_life,4f41f4a46b80c0ded61a9a3ce1273412b1728476a0734519d1af34a1797013b0,1f60c25c59138c14bacb0dd485475037e7d6222bb45daa66ed85f9dd91c83e4e,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" +17,meta,muse-image-1.0,originals/meta/17.png,1600,1600,product_still_life,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" +18,openai,gpt-image-1-mini,originals/openai/18.png,1024,1024,person_in_context,9b049d3112a654a87022d591b93a01087693c0664be5af560d3c0ec490291963,ba873bc7989720245821f5298f12e36bd7b6051909694b72569645c29cdd6bd0,"a lifestyle photo of a person working on a laptop in a bright cafe" +18,meta,muse-image-1.0,originals/meta/18.png,1600,1600,person_in_context,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,"a lifestyle photo of a person working on a laptop in a bright cafe" diff --git a/data/evaluations/engine-selection/originals/meta/00.png b/data/evaluations/engine-selection/originals/meta/00.png new file mode 100644 index 0000000..bd803e4 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/00.png differ diff --git a/data/evaluations/engine-selection/originals/meta/01.png b/data/evaluations/engine-selection/originals/meta/01.png new file mode 100644 index 0000000..c870563 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/01.png differ diff --git a/data/evaluations/engine-selection/originals/meta/02.png b/data/evaluations/engine-selection/originals/meta/02.png new file mode 100644 index 0000000..fec4a42 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/02.png differ diff --git a/data/evaluations/engine-selection/originals/meta/03.png b/data/evaluations/engine-selection/originals/meta/03.png new file mode 100644 index 0000000..42295d6 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/03.png differ diff --git a/data/evaluations/engine-selection/originals/meta/04.png b/data/evaluations/engine-selection/originals/meta/04.png new file mode 100644 index 0000000..3e0e34e Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/04.png differ diff --git a/data/evaluations/engine-selection/originals/meta/05.png b/data/evaluations/engine-selection/originals/meta/05.png new file mode 100644 index 0000000..7178204 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/05.png differ diff --git a/data/evaluations/engine-selection/originals/meta/06.png b/data/evaluations/engine-selection/originals/meta/06.png new file mode 100644 index 0000000..e4ee3ac Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/06.png differ diff --git a/data/evaluations/engine-selection/originals/meta/07.png b/data/evaluations/engine-selection/originals/meta/07.png new file mode 100644 index 0000000..e08851f Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/07.png differ diff --git a/data/evaluations/engine-selection/originals/meta/08.png b/data/evaluations/engine-selection/originals/meta/08.png new file mode 100644 index 0000000..80508f8 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/08.png differ diff --git a/data/evaluations/engine-selection/originals/meta/09.png b/data/evaluations/engine-selection/originals/meta/09.png new file mode 100644 index 0000000..c7feebe Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/09.png differ diff --git a/data/evaluations/engine-selection/originals/meta/10.png b/data/evaluations/engine-selection/originals/meta/10.png new file mode 100644 index 0000000..e0a719f Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/10.png differ diff --git a/data/evaluations/engine-selection/originals/meta/11.png b/data/evaluations/engine-selection/originals/meta/11.png new file mode 100644 index 0000000..4c4c522 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/11.png differ diff --git a/data/evaluations/engine-selection/originals/meta/12.png b/data/evaluations/engine-selection/originals/meta/12.png new file mode 100644 index 0000000..1ffce6e Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/12.png differ diff --git a/data/evaluations/engine-selection/originals/meta/13.png b/data/evaluations/engine-selection/originals/meta/13.png new file mode 100644 index 0000000..3bfba82 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/13.png differ diff --git a/data/evaluations/engine-selection/originals/meta/14.png b/data/evaluations/engine-selection/originals/meta/14.png new file mode 100644 index 0000000..0b21abe Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/14.png differ diff --git a/data/evaluations/engine-selection/originals/meta/15.png b/data/evaluations/engine-selection/originals/meta/15.png new file mode 100644 index 0000000..630edf0 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/15.png differ diff --git a/data/evaluations/engine-selection/originals/meta/16.png b/data/evaluations/engine-selection/originals/meta/16.png new file mode 100644 index 0000000..f3309fe Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/16.png differ diff --git a/data/evaluations/engine-selection/originals/meta/17.png b/data/evaluations/engine-selection/originals/meta/17.png new file mode 100644 index 0000000..4689b7d Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/17.png differ diff --git a/data/evaluations/engine-selection/originals/meta/18.png b/data/evaluations/engine-selection/originals/meta/18.png new file mode 100644 index 0000000..9ada905 Binary files /dev/null and b/data/evaluations/engine-selection/originals/meta/18.png differ diff --git a/data/evaluations/engine-selection/originals/openai/00.png b/data/evaluations/engine-selection/originals/openai/00.png new file mode 100644 index 0000000..b0e8c87 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/00.png differ diff --git a/data/evaluations/engine-selection/originals/openai/01.png b/data/evaluations/engine-selection/originals/openai/01.png new file mode 100644 index 0000000..c173450 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/01.png differ diff --git a/data/evaluations/engine-selection/originals/openai/02.png b/data/evaluations/engine-selection/originals/openai/02.png new file mode 100644 index 0000000..290edef Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/02.png differ diff --git a/data/evaluations/engine-selection/originals/openai/03.png b/data/evaluations/engine-selection/originals/openai/03.png new file mode 100644 index 0000000..a0680f8 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/03.png differ diff --git a/data/evaluations/engine-selection/originals/openai/04.png b/data/evaluations/engine-selection/originals/openai/04.png new file mode 100644 index 0000000..ca3a64f Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/04.png differ diff --git a/data/evaluations/engine-selection/originals/openai/05.png b/data/evaluations/engine-selection/originals/openai/05.png new file mode 100644 index 0000000..607c9f7 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/05.png differ diff --git a/data/evaluations/engine-selection/originals/openai/06.png b/data/evaluations/engine-selection/originals/openai/06.png new file mode 100644 index 0000000..01bcb24 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/06.png differ diff --git a/data/evaluations/engine-selection/originals/openai/07.png b/data/evaluations/engine-selection/originals/openai/07.png new file mode 100644 index 0000000..170c332 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/07.png differ diff --git a/data/evaluations/engine-selection/originals/openai/08.png b/data/evaluations/engine-selection/originals/openai/08.png new file mode 100644 index 0000000..24c9e10 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/08.png differ diff --git a/data/evaluations/engine-selection/originals/openai/09.png b/data/evaluations/engine-selection/originals/openai/09.png new file mode 100644 index 0000000..d8cdb0f Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/09.png differ diff --git a/data/evaluations/engine-selection/originals/openai/10.png b/data/evaluations/engine-selection/originals/openai/10.png new file mode 100644 index 0000000..58964d7 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/10.png differ diff --git a/data/evaluations/engine-selection/originals/openai/11.png b/data/evaluations/engine-selection/originals/openai/11.png new file mode 100644 index 0000000..65ab17a Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/11.png differ diff --git a/data/evaluations/engine-selection/originals/openai/12.png b/data/evaluations/engine-selection/originals/openai/12.png new file mode 100644 index 0000000..5e6d9f2 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/12.png differ diff --git a/data/evaluations/engine-selection/originals/openai/13.png b/data/evaluations/engine-selection/originals/openai/13.png new file mode 100644 index 0000000..aa97408 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/13.png differ diff --git a/data/evaluations/engine-selection/originals/openai/14.png b/data/evaluations/engine-selection/originals/openai/14.png new file mode 100644 index 0000000..5faaa77 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/14.png differ diff --git a/data/evaluations/engine-selection/originals/openai/15.png b/data/evaluations/engine-selection/originals/openai/15.png new file mode 100644 index 0000000..da2b716 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/15.png differ diff --git a/data/evaluations/engine-selection/originals/openai/16.png b/data/evaluations/engine-selection/originals/openai/16.png new file mode 100644 index 0000000..10675b9 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/16.png differ diff --git a/data/evaluations/engine-selection/originals/openai/17.png b/data/evaluations/engine-selection/originals/openai/17.png new file mode 100644 index 0000000..b238c6c Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/17.png differ diff --git a/data/evaluations/engine-selection/originals/openai/18.png b/data/evaluations/engine-selection/originals/openai/18.png new file mode 100644 index 0000000..ba7c997 Binary files /dev/null and b/data/evaluations/engine-selection/originals/openai/18.png differ diff --git a/docs/chroma1-engine-research.md b/docs/chroma1-engine-research.md index 5497818..34e333b 100644 --- a/docs/chroma1-engine-research.md +++ b/docs/chroma1-engine-research.md @@ -475,6 +475,51 @@ That is not a YuNet split: `face_count > 0` would over-strengthen the spread (0.075 - 0.06) is already absorbed by the shipped 0.09 floor. Images API generations remain non-carriers (C2PA only). +## Content-balanced engine selection check (2026-08-31) + +The earlier tables were intentionally carrier-focused and therefore small. A +separate discovery pass tested whether ordinary image content contradicts the +cohort choice at the shipped floors. The tracked matrix contains 19 prompts, +matched across OpenAI `gpt-image-1-mini` and Meta `muse-image-1.0`, spanning +photoreal scenes, faces, action, text-like flat art, products, architecture, +watercolor, and surreal illustration. These files were re-encoded by collection +and are not signal carriers, so this pass measures fidelity only; removal remains +established by the provider-oracle calibration above. + +Qwen and Chroma ran in separate sequential H100 invocations, once per source, +with the production provider floor and fixed seed. The comparison isolated the +global stage because both profiles inherit the same source-based Z-Image face +repair. Per-image differences, rather than differences between aggregate means, +were evaluated with exact two-sided sign tests: + +| provider | metric | Chroma wins | Qwen wins | two-sided p | +|---|---|---:|---:|---:| +| OpenAI | SSIM | 19 | 0 | 0.0000038 | +| OpenAI | PSNR | 19 | 0 | 0.0000038 | +| OpenAI | MAE | 18 | 1 | 0.000076 | +| OpenAI | edge F1 | 15 | 4 | 0.019 | +| OpenAI | LPIPS | 12 | 7 | 0.359 | +| Meta | LPIPS | 1 | 18 | 0.000076 | +| Meta | edge F1 | 1 | 18 | 0.000076 | +| Meta | SSIM | 5 | 14 | 0.064 | + +Meta's remaining pixel metrics disagree: Chroma wins PSNR on 16/19 and MAE on +14/19, while losing both perceptual distance and edge preservation on 18/19 at +its higher removal floor. This is not evidence for a genre classifier. The +direction instead follows the already measured provenance cohort: Chroma for +OpenAI, Qwen for Meta. On OpenAI, the seven LPIPS exceptions include unrelated +classes (landscape, sport, low-light interior, watercolor, monochrome portrait, +surreal art, and brutalism), while Chroma still wins SSIM and PSNR on every one; +there is no coherent input feature that justifies overriding the cohort choice. + +Inputs and integrity manifests are under +`data/evaluations/engine-selection/`. Raw generated outputs remain gitignored +under `out/engine-selection-study/`; `scripts/engine_selection_study.py` +reproduces the run and `scripts/analyze_engine_selection_study.py` reproduces +the paired statistics. One generation per prompt is a discovery set, not a +license to fit thresholds. Any future content override must first predict a +rule here and then survive unused generation indices without changing it. + ## Oracle session notes - openai.com/verify throttles anonymous sessions through Cloudflare Turnstile diff --git a/docs/cli.md b/docs/cli.md index db9ebdd..a68b1bf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -468,17 +468,18 @@ user can act on rather than after a model load. ### Restore operator-verified text `--text-manifest` enables the experimental `vae-glyphs` post-pass. It reconstructs -the source with the Qwen VAE, erases the annotated candidate glyphs with LaMa, and -composites only the reconstructed glyph cores through source-derived silhouettes. It -does not run OCR or choose which strings are correct. The optional `--fidelity-anchor` -described below additionally blends 15% of the reconstruction across the full frame. +the source with the selected profile's VAE, erases the annotated candidate glyphs +with LaMa, and composites only the reconstructed glyph cores through source-derived +silhouettes. It does not run OCR or choose which strings are correct. The optional +Qwen-only `--fidelity-anchor` described below additionally blends 15% of the +reconstruction across the full frame. Install the combined extra and run only with an operator-verified manifest: ```bash uv tool install --force "remove-ai-watermarks[text-restoration]" remove-ai-watermarks invisible image.png -o clean.png \ - --pipeline qwen-zimage --text-manifest verified-lines.json --force + --pipeline auto --text-manifest verified-lines.json --force ``` ``verified: true`` may also be set by an automated operator that attests @@ -486,7 +487,8 @@ machine-verified geometry: stability-gated detector boxes inside sane caps. Such operators should use the geometry-only schema 2, which carries no transcription or script metadata. -Since 0.27.1 the global 15% Qwen-VAE fidelity-anchor blend is off by default: it +Since 0.27.1 the global 15% Qwen-VAE fidelity-anchor blend is off by default and +remains available only with `qwen-zimage`: it was measured to return detector-visible OpenAI SynthID on poster-scale manifests (official Content Provenance API, 2026-08-19). `--fidelity-anchor` restores the 0.27.0 research behavior; text-box fidelity lost by the default is well under one @@ -503,8 +505,9 @@ container changes remain valid while a resized or edited source fails closed. Th experimental helper `remove_ai_watermarks._internal.text_restoration.source_pixel_sha256` computes it. -This mode is supported only by `qwen-zimage` at native geometry with -`humanize=0`, `unsharp=0`, and adaptive polish disabled. `all` also accepts the flag, +This mode is supported by `qwen-zimage`, `chroma-zimage`, and `auto` at native +geometry with `humanize=0`, `unsharp=0`, and adaptive polish disabled. The legacy +`sdxl-zimage` profile does not expose a verified-text VAE donor. `all` also accepts the flag, but its manifest must match the pixels entering the invisible stage; if visible-mark removal changes those pixels, the hash check rejects the run. One oracle verdict does not certify another manifest, seed, model/runtime version, or output hash. diff --git a/docs/installation.md b/docs/installation.md index 3fbbd04..6a05aec 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -117,7 +117,7 @@ application actually uses: | `migan` | MI-GAN ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch | | `lama` | big-LaMa ONNX fill backend | `visible`, ONNX Runtime | Model download, no Torch | | `qwen-zimage` | Invisible image-watermark removal, all CUDA-only profiles | `diffusion`, DiffSynth | Yes | -| `text-restoration` | Opt-in verified Qwen-VAE glyph restoration | `qwen-zimage`, `lama` | Yes | +| `text-restoration` | Opt-in verified profile-VAE glyph restoration | `qwen-zimage`, `lama` | Yes | | `text-draft` | Draft OCR proposals for operator verification | PaddleOCR, PaddlePaddle | Model download, no Torch | | `all` | Every production feature available on the active Python | All compatible rows above | Yes | | `dev` | Tests, linting, typing, and upstream parity checks | `video`, `detect`, upstream invisible-watermark | Yes, for parity tests | diff --git a/docs/module-internals.md b/docs/module-internals.md index 12d38eb..ccab018 100644 --- a/docs/module-internals.md +++ b/docs/module-internals.md @@ -976,9 +976,10 @@ is the source of truth for: The current profiles are `qwen-zimage` (the default), `sdxl-zimage`, `chroma-zimage`, and `auto`, and all four are CUDA-only. `controlnet`, `sdxl`, `qwen` and `default` were removed rather than kept as a CPU path, and are -rejected rather than aliased onward. `auto` is a per-cohort engine router: +rejected rather than aliased onward. `auto` is a deterministic per-cohort +selection policy: chroma-zimage for OpenAI and Microsoft, qwen-zimage for Google, Meta, and -unknown. It is not a content-class split inside one engine. +unknown. It does not run a learned router or classify the image's genre. `qwen-zimage` normally resolves global denoise from image area for unknown content. Measured provider cohorts bypass that curve with flat operating points. The values, @@ -1136,13 +1137,14 @@ geometry alone in schema 2, plus a SHA-256 over decoded RGB width, height, and p Validation happens before model loading. The library never treats OCR confidence as verification, and geometry-only operators do not need to invent text or script fields. -When enabled, `QwenZImagePipeline` reconstructs the source once through its already -loaded Qwen VAE, runs the ordinary global and face stages, and calls the shared -restoration compositor. The optional fidelity anchor first blends 15% of the VAE +When enabled, the one profile selected before model loading reconstructs the source +once through its already loaded VAE, runs the ordinary global and face stages, and +calls the shared restoration compositor. Qwen and Chroma implement this donor hook; +SDXL does not. The optional Qwen-only fidelity anchor first blends 15% of the VAE reconstruction into the clean result; it is off by default because the blend returned detector-visible OpenAI SynthID in the measured poster fixtures. The compositor derives binary source and candidate silhouettes, groups nearby lines, uses LaMa for the initial -and residual-glyph erase passes, paints fresh silhouette edges, then copies the Qwen-VAE +and residual-glyph erase passes, paints fresh silhouette edges, then copies the profile-VAE core with a 0.5-pixel feather. The evaluation script imports these same mask and compositing helpers so the two implementations cannot silently drift. Silhouette crops start 12% of line height beyond each horizontal @@ -1158,6 +1160,21 @@ geometry or final pixels after the verified layer and have no measured oracle re It remains opt-in because annotations are manual and provider verdicts apply only to the exact tested output hashes, not to the mechanism in general. +The first full-path H100 check on 2026-08-31 ran each generative engine exactly +once per source and used that SAME engine's VAE donor; it did not compose Qwen and +Chroma in one production result. On the two verified multilingual OpenAI posters, +restoration reduced PaddleOCR CER from 0.303 to 0.160 and 0.279 to 0.136 for Qwen, +and from 0.213 to 0.130 and 0.204 to 0.108 for Chroma. PSNR improved on all four +arms (Qwen 26.1 to 29.9 dB and 27.7 to 31.6 dB; Chroma 29.6 to 33.1 dB and 29.7 +to 32.9 dB). The verified CJK Google fixture is the counterexample that prevents +an unconditional quality claim: Qwen CER stayed 0.074, while Chroma moved from +0.037 to 0.074 despite better LPIPS, SSIM, PSNR, and text-box pixel error. The +stage is therefore implemented and fidelity-positive on both OpenAI sources, but +not assumed to improve every OCR reading. This is a fidelity check, not a provider +oracle certification; exact output hashes and per-box metrics are produced by +`scripts/chroma_text_restoration_study.py` under the gitignored +`out/text-restoration-engine-study/` directory. + A matched stage-isolation check on the 18-face Gemini portrait grid confirms the division of responsibility. The visible-cleaned, metadata-stripped control and the Z-Image face-only output were both SynthID-positive; Qwen global-only and the full @@ -1196,6 +1213,18 @@ strength-controlled img2img in that family; the full research record, including why FLUX.2 itself is not integrable this way, is [`chroma1-engine-research.md`](chroma1-engine-research.md). +The profile also implements the shared verified-text donor hook through its already +loaded Chroma VAE. It edge-pads native geometry to the /16 latent grid, takes the +deterministic mode of the encoder distribution rather than sampling it, decodes with +Diffusers' own image processor, and crops back to the exact source dimensions. Thus +`auto` still loads and runs only the profile selected by the measured vendor table; +the presence of a manifest does not introduce a second generative model or override +that decision. A CPU smoke run against the official Chroma1-HD `AutoencoderKL` +weights on 2026-08-31 verified the real encode/decode contract and exact restoration +of a non-grid-aligned source size; model-free tests pin mode-not-sample and padding. +Chroma-specific full-pipeline provider-oracle certification remains an output-hash +measurement, not something either check can establish. + The profile swaps only the global stage (same inheritance invariant as sdxl-zimage, asserted by the same test shape). Three things are bound to the calibration and must not drift: the NEUTRAL prompt (`high quality, sharp, @@ -1227,6 +1256,24 @@ that rung, which is the margin the 0.17 floor already holds. OpenAI stays on the clear at or below 0.06, including a 2-face file, and only the 9-face grid needs 0.075 (seed-stable on 0, 1, 2 via openai.com/research/verify). +A separate content-balanced check on 2026-08-31 tested the exact production +OpenAI and Meta floors over 19 prompt-matched image strata per provider. Each +engine ran in its own sequential H100 invocation, and the comparison isolated +the global stage because the shared face repair does not change with this choice. +On OpenAI content, Chroma won SSIM and PSNR on 19/19 pairs, MAE on 18/19, edge +F1 on 15/19, and LPIPS on 12/19. On Meta content, Qwen won LPIPS and edge F1 on +18/19 pairs; Chroma's better PSNR on 16/19 did not overcome its systematically +worse perceptual and edge preservation at the higher required floor. Exact +two-sided sign-test p-values were 3.8e-6 for the OpenAI SSIM and PSNR directions +and 7.6e-5 for the Meta LPIPS and edge-F1 directions. The tracked discovery +inputs and manifests are in +[`data/evaluations/engine-selection/`](../data/evaluations/engine-selection/), +the run is reproduced by `scripts/engine_selection_study.py`, and paired analysis +by `scripts/analyze_engine_selection_study.py`. These content files were locally +re-encoded and are not watermark oracles; they validate fidelity at already +oracle-calibrated floors, not removal by themselves. No content-stratum override +is supported by this pass, so `auto` keeps the measured provenance-cohort table. + ### SDXL plus Z-Image [`_internal/sdxl_zimage_pipeline.py`](../src/remove_ai_watermarks/_internal/sdxl_zimage_pipeline.py) diff --git a/docs/python-api.md b/docs/python-api.md index 5962a6d..5eb6334 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -672,11 +672,13 @@ engine.remove_watermark( Install `remove-ai-watermarks[text-restoration]`. The manifest schema and safety constraints are documented in the CLI guide. The engine verifies its decoded RGB -hash before loading the diffusion models and rejects SDXL, downscaling, and -postprocessing combinations that were not evaluated. Tiling is also rejected because -the combined tiled-restoration path has no provider-oracle calibration. `InvisibleOptions` -exposes the same field for `remove_all`; after a visible-stage edit, the manifest must -be built against the staged pixels rather than the pristine source. +hash before loading the diffusion models. Qwen and Chroma reconstruct the donor with +the VAE already loaded for the one profile selected by `auto`; no second generative +profile runs. The engine rejects SDXL, downscaling, and postprocessing combinations +that were not evaluated. Tiling is also rejected because the combined +tiled-restoration path has no provider-oracle calibration. `InvisibleOptions` exposes +the same field for `remove_all`; after a visible-stage edit, the manifest must be built +against the staged pixels rather than the pristine source. Use manifest schema 1 for manually reviewed text plus script metadata. Automated operators that verify only text-region geometry should emit schema 2 lines with a @@ -687,7 +689,8 @@ default** (`fidelity_anchor=False`): that whole-frame blend was measured to return detector-visible OpenAI SynthID on poster-scale manifests (official Content Provenance API, 2026-08-19 - detected x6 with the anchor, clean x6 without it, controls and base outputs validated in the same sessions). Pass -`fidelity_anchor=True` to reproduce the 0.27.0 research behavior. +`fidelity_anchor=True` with `qwen-zimage` to reproduce the 0.27.0 research +behavior. Chroma rejects that Qwen-specific reproduction flag. ### Drafting manifest lines diff --git a/scripts/README.md b/scripts/README.md index d8abf02..447e298 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -45,13 +45,17 @@ explicitly names a tracked canonical result. | `visible_sheets.py` | Build blinded contact sheets for relaxation candidates. | | `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. | | `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-models`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. | +| `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. | ## Research and diagnostic prototypes | Script | Purpose | | --- | --- | +| `chroma_text_restoration_study.py` | Compare verified-text restoration for Qwen and Chroma on H100 without running both engines in production. | | `cjk_tail_probe.py` | Test a generic template for otherwise uncovered CJK labels. | | `controlnet_sweep.py` | Sweep the historical ControlNet removal prototype. | +| `engine_selection_study.py` | Run one global engine at a time over the tracked prompt-matched content matrix. | +| `analyze_engine_selection_study.py` | Reduce engine-study output to paired deltas and exact sign tests. | | `infer_text_lines.py` | Draft stable source-text lines without modifying pixels. | | `qwen_scrub_prototype.py` | Probe low-strength Qwen regeneration on a GPU. | | `selective_text_restoration.py` | Evaluate text restoration over a scrubbed image. | diff --git a/scripts/analyze_engine_selection_study.py b/scripts/analyze_engine_selection_study.py new file mode 100644 index 0000000..2c1abcd --- /dev/null +++ b/scripts/analyze_engine_selection_study.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Summarize paired fidelity results from the engine-selection study. + +Run from the repository root after ``engine_selection_study.py``: + + uv run scripts/analyze_engine_selection_study.py + +The report uses per-image paired differences and exact two-sided sign tests. +It does not turn the discovery set into a production routing rule. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import math +import statistics +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + +log = logging.getLogger(__name__) + +_METRICS: dict[str, tuple[Callable[[float], float], str]] = { + "lpips_512": (lambda value: value, "lower"), + "ssim": (lambda value: -value, "higher"), + "psnr_db": (lambda value: -value, "higher"), + "mae": (lambda value: value, "lower"), + "edge_f1": (lambda value: -value, "higher"), + "laplacian_ratio": (lambda value: abs(math.log(max(value, 1e-12))), "closer_to_one"), +} + + +def _sign_test_two_sided(wins: int, losses: int) -> float: + """Return the exact two-sided sign-test p-value, excluding ties.""" + trials = wins + losses + if trials == 0: + return 1.0 + tail = sum(math.comb(trials, index) for index in range(min(wins, losses) + 1)) + return min(1.0, 2.0 * tail / (2**trials)) + + +def _index(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, Any]]: + return {(row["provider"], row["pair_id"]): row for row in rows} + + +def analyze(report: dict[str, Any]) -> dict[str, Any]: + qwen = _index(report["qwen-zimage"]["rows"]) + chroma = _index(report["chroma-zimage"]["rows"]) + if qwen.keys() != chroma.keys(): + raise ValueError("Qwen and Chroma result keys differ") + + by_provider: dict[str, list[tuple[dict[str, Any], dict[str, Any]]]] = defaultdict(list) + for key in sorted(qwen): + by_provider[key[0]].append((qwen[key], chroma[key])) + + providers: dict[str, Any] = {} + for provider, pairs in sorted(by_provider.items()): + metrics: dict[str, Any] = {} + for metric, (loss_transform, direction) in _METRICS.items(): + deltas: list[float] = [] + chroma_wins = 0 + qwen_wins = 0 + ties = 0 + for qwen_row, chroma_row in pairs: + qwen_value = float(qwen_row["fidelity"][metric]) + chroma_value = float(chroma_row["fidelity"][metric]) + delta = loss_transform(chroma_value) - loss_transform(qwen_value) + deltas.append(delta) + if math.isclose(delta, 0.0, abs_tol=1e-12): + ties += 1 + elif delta < 0: + chroma_wins += 1 + else: + qwen_wins += 1 + metrics[metric] = { + "preferred_direction": direction, + "delta_definition": "chroma_loss_minus_qwen_loss; negative favors Chroma", + "chroma_wins": chroma_wins, + "qwen_wins": qwen_wins, + "ties": ties, + "mean_delta": statistics.fmean(deltas), + "median_delta": statistics.median(deltas), + "sign_test_two_sided_p": _sign_test_two_sided(chroma_wins, qwen_wins), + } + providers[provider] = {"pairs": len(pairs), "metrics": metrics} + + return { + "method": "paired per-image differences with exact two-sided sign tests; ties excluded", + "warning": "Discovery-set evidence only; challenge any proposed rule on unused generations.", + "providers": providers, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--input", + type=Path, + default=Path("out/engine-selection-study/summary.json"), + ) + parser.add_argument( + "--output", + type=Path, + default=Path("out/engine-selection-study/analysis.json"), + ) + args = parser.parse_args() + report = json.loads(args.input.read_text(encoding="utf-8")) + analysis = analyze(report) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(analysis, indent=2) + "\n", encoding="utf-8") + log.info("Wrote %s", args.output) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + main() diff --git a/scripts/chroma_text_restoration_study.py b/scripts/chroma_text_restoration_study.py new file mode 100644 index 0000000..49a7947 --- /dev/null +++ b/scripts/chroma_text_restoration_study.py @@ -0,0 +1,232 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["modal>=1.5"] +# /// +"""Paired offline study of verified-text restoration on Qwen and Chroma. + +Each generative profile runs exactly once per fixture. Its already loaded VAE +produces the restoration donor, then the shared production compositor creates +the restored candidate from that one base output. The experiment therefore +measures four outputs (Qwen/Chroma x base/restored) without accidentally timing +or evaluating a second generation for either restoration arm. + +Run from the repository root: + + uvx modal run scripts/chroma_text_restoration_study.py + +Outputs stay under ``out/text-restoration-engine-study``. Submit exact output +hashes to the matching provider oracle before using fidelity metrics to choose +between engines; fidelity only ranks candidates after the scrub verdict passes. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import modal + +_ROOT = Path(__file__).resolve().parent.parent +_GPU = "H100" + +app = modal.App("chroma-text-restoration-study") + +image = ( + modal.Image.debian_slim(python_version="3.12") + .pip_install( + "torch", + "torchvision>=0.20.0", + "diffusers>=0.38,<1", + "diffsynth>=2.0.17,<3", + "transformers>=4.53", + "accelerate", + "sentencepiece", + "protobuf", + "pillow", + "numpy", + "opencv-python-headless<5", + "onnxruntime>=1.24.0", + "huggingface-hub>=0.20.0", + ) + .env({"HF_HOME": "/cache"}) + .add_local_dir(_ROOT / "src" / "remove_ai_watermarks", remote_path="/pkg/remove_ai_watermarks") +) + +cache = modal.Volume.from_name("chroma-hf-cache", create_if_missing=True) + + +def _png_bytes(image: Any) -> bytes: + import io + + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +def _metric_inputs(source: Any, boxes: list[tuple[int, int, int, int]]) -> tuple[Any, Any]: + import numpy as np + + source_rgb = np.asarray(source.convert("RGB"), dtype=np.uint8) + mask = np.zeros(source_rgb.shape[:2], dtype=bool) + for x1, y1, x2, y2 in boxes: + mask[y1:y2, x1:x2] = True + return source_rgb, mask + + +def _metrics(source_rgb: Any, candidate: Any, mask: Any) -> dict[str, float]: + import math + + import numpy as np + + candidate_rgb = np.asarray(candidate.convert("RGB"), dtype=np.uint8) + signed_delta = source_rgb.astype(np.float32) - candidate_rgb + absolute_delta = np.abs(signed_delta) + mse = float((signed_delta**2).mean()) + return { + "whole_mae": round(float(absolute_delta.mean()), 6), + "text_box_mae": round(float(absolute_delta[mask].mean()), 6), + "outside_box_mae": round(float(absolute_delta[~mask].mean()), 6), + "psnr_db": round(99.0 if mse <= 1e-12 else 10.0 * math.log10(255.0**2 / mse), 6), + } + + +@app.function(image=image, gpu=_GPU, volumes={"/cache": cache}, timeout=3600) +def run(payload: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: + import gc + import io + import sys + import time + + import torch + from PIL import Image + + sys.path.insert(0, "/pkg") + + from remove_ai_watermarks._internal.chroma_zimage_pipeline import ChromaZImagePipeline + from remove_ai_watermarks._internal.qwen_zimage_pipeline import QwenZImagePipeline + from remove_ai_watermarks._internal.text_restoration import VerifiedTextLine, restore_verified_text + from remove_ai_watermarks._internal.two_stage_pipeline import detect_faces + from remove_ai_watermarks._internal.watermark_profiles import resolve_strength + + sources: dict[str, tuple[Image.Image, str, tuple[VerifiedTextLine, ...]]] = {} + for name, item in payload.items(): + source = Image.open(io.BytesIO(item["source"])).convert("RGB") + lines = tuple( + VerifiedTextLine( + tuple(line["box"]), + line.get("text"), + line.get("script"), + float(line.get("angle", 0.0)), + ) + for line in item["lines"] + ) + sources[name] = source, str(item["vendor"]), lines + + results: dict[str, dict[str, Any]] = { + name: {"vendor": vendor, "engines": {}} for name, (_, vendor, _) in sources.items() + } + metric_inputs = { + name: _metric_inputs(source, [line.box for line in lines]) for name, (source, _vendor, lines) in sources.items() + } + + for profile, pipeline_type in ( + ("qwen-zimage", QwenZImagePipeline), + ("chroma-zimage", ChromaZImagePipeline), + ): + print(f"[{profile}] loading...", flush=True) + pipeline = pipeline_type( + device="cuda", + torch_dtype=torch.bfloat16, + keep_global_models_on_device=True, + keep_face_models_on_device=True, + progress_callback=lambda message, p=profile: print(f"[{p}] {message}", flush=True), + ) + for name, (source, vendor, lines) in sources.items(): + face_count = len(detect_faces(source)) if profile == "chroma-zimage" else 0 + strength = resolve_strength( + None, + vendor, + profile, + size=source.size, + face_count=face_count, + ) + print(f"[{profile}] {name}: strength={strength:.6f}, faces={face_count}", flush=True) + + started = time.perf_counter() + donor = pipeline._vae_roundtrip(source) + donor_seconds = time.perf_counter() - started + + started = time.perf_counter() + base = pipeline.run(source, strength=strength, seed=0) + generation_seconds = time.perf_counter() - started + + started = time.perf_counter() + restored = restore_verified_text(source, base, donor, lines) + restoration_seconds = time.perf_counter() - started + + source_rgb, text_mask = metric_inputs[name] + base_png = _png_bytes(base) + restored_png = _png_bytes(restored) + donor_png = _png_bytes(donor) + results[name]["engines"][profile] = { + "strength": strength, + "face_count": face_count, + "donor_seconds": round(donor_seconds, 3), + "generation_seconds": round(generation_seconds, 3), + "restoration_seconds": round(restoration_seconds, 3), + "base_sha256": hashlib.sha256(base_png).hexdigest(), + "restored_sha256": hashlib.sha256(restored_png).hexdigest(), + "donor_sha256": hashlib.sha256(donor_png).hexdigest(), + "base_metrics": _metrics(source_rgb, base, text_mask), + "restored_metrics": _metrics(source_rgb, restored, text_mask), + "base_png": base_png, + "restored_png": restored_png, + "donor_png": donor_png, + } + + del pipeline + gc.collect() + torch.cuda.empty_cache() + + cache.commit() + return results + + +@app.local_entrypoint() +def main(out: str = "out/text-restoration-engine-study") -> None: + annotations = json.loads((_ROOT / "data/evaluations/fidelity/text-lines.json").read_text(encoding="utf-8")) + vendors = { + "ChatGPT Image May 31, 2026, 02_02_23 PM.png": "openai", + "ChatGPT Image May 31, 2026, 02_03_55 PM.png": "openai", + "Gemini_Generated_Image_633uuy633uuy633u.png": "google", + } + source_root = _ROOT / "data/synthid/originals" + payload = { + name: { + "source": (source_root / name).read_bytes(), + "vendor": vendor, + "lines": annotations[name], + } + for name, vendor in vendors.items() + } + + results = run.remote(payload) + out_root = Path(out) + summary: dict[str, Any] = {"gpu": _GPU, "fixtures": {}} + for name, fixture in results.items(): + fixture_dir = out_root / Path(name).stem + fixture_dir.mkdir(parents=True, exist_ok=True) + summary["fixtures"][name] = {"vendor": fixture["vendor"], "engines": {}} + for profile, result in fixture["engines"].items(): + for variant in ("base", "restored", "donor"): + (fixture_dir / f"{profile}_{variant}.png").write_bytes(result[f"{variant}_png"]) + summary["fixtures"][name]["engines"][profile] = { + key: value for key, value in result.items() if not key.endswith("_png") + } + + out_root.mkdir(parents=True, exist_ok=True) + (out_root / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + print(f"outputs in {out_root}") diff --git a/scripts/engine_selection_study.py b/scripts/engine_selection_study.py new file mode 100644 index 0000000..5ab531c --- /dev/null +++ b/scripts/engine_selection_study.py @@ -0,0 +1,221 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["modal>=1.5"] +# /// +"""Compare one engine at a time on the tracked content-selection matrix. + +This is an offline research harness, not a production router. Each Modal call +loads exactly one global engine and runs all 38 fixed inputs at that engine's +shipped provider floor. The shared Z-Image face repair is deliberately omitted: +the model choice changes only the global stage, while the face stage regenerates +the same original crops for both profiles. + +Run from the repository root: + + uvx modal run scripts/engine_selection_study.py + +The combined metrics land in the gitignored +``out/engine-selection-study/summary.json``. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +import modal + +log = logging.getLogger(__name__) + +_ROOT = Path(__file__).resolve().parent.parent +_FIXTURE_ROOT = _ROOT / "data" / "evaluations" / "engine-selection" +_GPU = "H100" + +app = modal.App("engine-selection-study") + +image = ( + modal.Image.debian_slim(python_version="3.12") + .pip_install( + "torch", + "torchvision>=0.20.0", + "diffusers>=0.38,<1", + "diffsynth>=2.0.17,<3", + "transformers>=4.53", + "accelerate", + "sentencepiece", + "protobuf", + "pillow", + "numpy", + "opencv-python-headless", + "scikit-image", + "lpips", + ) + .env({"HF_HOME": "/cache", "TORCH_HOME": "/cache/torch"}) + .add_local_dir( + _ROOT / "src" / "remove_ai_watermarks", + remote_path="/pkg/remove_ai_watermarks", + ) + .add_local_dir(_FIXTURE_ROOT, remote_path="/fixtures") +) + +cache = modal.Volume.from_name("chroma-hf-cache", create_if_missing=True) + +_STRENGTHS = { + "qwen-zimage": {"openai": 0.07675, "meta": 0.10}, + "chroma-zimage": {"openai": 0.09, "meta": 0.17}, +} + + +@app.function(image=image, gpu=_GPU, volumes={"/cache": cache}, timeout=3600) +def run_engine(engine: str) -> dict[str, Any]: + import csv + import hashlib + import sys + import time + + import cv2 + import lpips + import numpy as np + import torch + from PIL import Image + from skimage.metrics import peak_signal_noise_ratio, structural_similarity + + sys.path.insert(0, "/pkg") + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + if engine == "qwen-zimage": + from remove_ai_watermarks._internal.qwen_zimage_pipeline import QwenZImagePipeline + + pipeline_type: Any = QwenZImagePipeline + elif engine == "chroma-zimage": + from remove_ai_watermarks._internal.chroma_zimage_pipeline import ChromaZImagePipeline + + pipeline_type = ChromaZImagePipeline + else: + raise ValueError(f"unknown engine: {engine}") + pipeline = pipeline_type( + device="cuda", + torch_dtype=torch.bfloat16, + keep_global_models_on_device=True, + keep_face_models_on_device=False, + ) + + perceptual = lpips.LPIPS(net="alex", verbose=False).to("cuda").eval() + + def rgb_array(pil_image: Image.Image) -> np.ndarray: + return np.asarray(pil_image.convert("RGB"), dtype=np.uint8) + + def gray(rgb: np.ndarray) -> np.ndarray: + return cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY) + + def lpips_distance(source: np.ndarray, candidate: np.ndarray) -> float: + height, width = source.shape[:2] + scale = min(1.0, 512.0 / max(height, width)) + size = (max(1, round(width * scale)), max(1, round(height * scale))) + + def tensor(array: np.ndarray) -> torch.Tensor: + resized = cv2.resize(array, size, interpolation=cv2.INTER_AREA) + normalized = resized.astype(np.float32) / 127.5 - 1.0 + return torch.from_numpy(normalized).permute(2, 0, 1).unsqueeze(0).to("cuda") + + with torch.inference_mode(): + return float(perceptual(tensor(source), tensor(candidate)).item()) + + def edge_f1(source_edges: np.ndarray, candidate_gray: np.ndarray) -> float: + candidate_edges = cv2.Canny(candidate_gray, 100, 200) > 0 + kernel = np.ones((3, 3), np.uint8) + source_near = cv2.dilate(source_edges.astype(np.uint8), kernel) > 0 + candidate_near = cv2.dilate(candidate_edges.astype(np.uint8), kernel) > 0 + precision = float((candidate_edges & source_near).sum()) / max(1, int(candidate_edges.sum())) + recall = float((source_edges & candidate_near).sum()) / max(1, int(source_edges.sum())) + return 2.0 * precision * recall / max(1e-9, precision + recall) + + def source_features( + source: np.ndarray, + source_gray: np.ndarray, + source_edges: np.ndarray, + source_lapvar: float, + ) -> dict[str, float]: + histogram = np.bincount(source_gray.reshape(-1), minlength=256).astype(np.float64) + probabilities = histogram[histogram > 0] / histogram.sum() + hsv = cv2.cvtColor(source, cv2.COLOR_RGB2HSV) + return { + "edge_density": round(float(source_edges.mean()), 6), + "laplacian_variance": round(source_lapvar, 6), + "entropy_bits": round(float(-(probabilities * np.log2(probabilities)).sum()), 6), + "mean_saturation": round(float(hsv[..., 1].mean() / 255.0), 6), + } + + def fidelity( + source: np.ndarray, + candidate: np.ndarray, + source_gray: np.ndarray, + source_edges: np.ndarray, + source_lapvar: float, + ) -> dict[str, float]: + candidate_gray = gray(candidate) + candidate_lapvar = float(cv2.Laplacian(candidate_gray, cv2.CV_64F).var()) + return { + "lpips_512": round(lpips_distance(source, candidate), 6), + "ssim": round(float(structural_similarity(source_gray, candidate_gray)), 6), + "psnr_db": round(float(peak_signal_noise_ratio(source, candidate)), 6), + "mae": round(float(np.abs(source.astype(np.float32) - candidate).mean()), 6), + "edge_f1": round(edge_f1(source_edges, candidate_gray), 6), + "laplacian_ratio": round(candidate_lapvar / max(source_lapvar, 1e-9), 6), + } + + with Path("/fixtures/content-manifest.csv").open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + + results: list[dict[str, Any]] = [] + for index, row in enumerate(rows, start=1): + source_path = Path("/fixtures") / row["file"] + source_image = Image.open(source_path).convert("RGB") + strength = _STRENGTHS[engine][row["provider"]] + started = time.perf_counter() + output = pipeline._run_global(source_image, strength, 0) + elapsed = time.perf_counter() - started + source = rgb_array(source_image) + candidate = rgb_array(output) + source_gray = gray(source) + source_edges = cv2.Canny(source_gray, 100, 200) > 0 + source_lapvar = float(cv2.Laplacian(source_gray, cv2.CV_64F).var()) + result = { + "pair_id": row["pair_id"], + "provider": row["provider"], + "content_stratum": row["content_stratum"], + "strength": strength, + "seconds": round(elapsed, 3), + "output_rgb_sha256": hashlib.sha256(output.tobytes()).hexdigest(), + "source_features": source_features(source, source_gray, source_edges, source_lapvar), + "fidelity": fidelity(source, candidate, source_gray, source_edges, source_lapvar), + } + results.append(result) + log.info( + "%s %s/%s (%s/%s)", + engine, + index, + len(rows), + row["provider"], + row["content_stratum"], + ) + + cache.commit() + return {"engine": engine, "gpu": _GPU, "rows": results} + + +@app.local_entrypoint() +def main(out: str = "out/engine-selection-study") -> None: + qwen = run_engine.remote("qwen-zimage") + chroma = run_engine.remote("chroma-zimage") + report = { + "protocol": "one engine per sequential H100 invocation; global-stage factorial ablation", + "qwen-zimage": qwen, + "chroma-zimage": chroma, + } + destination = Path(out) / "summary.json" + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, indent=2), encoding="utf-8") + log.info("Wrote %s", destination) diff --git a/scripts/verify_engine_selection_fixtures.py b/scripts/verify_engine_selection_fixtures.py new file mode 100644 index 0000000..10f5ed6 --- /dev/null +++ b/scripts/verify_engine_selection_fixtures.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Verify the tracked content matrix used by the auto-engine study.""" + +from __future__ import annotations + +import csv +import hashlib +import logging +from collections import defaultdict +from pathlib import Path + +from PIL import Image + +log = logging.getLogger(__name__) + +_ROOT = Path(__file__).resolve().parent.parent +_FIXTURE_ROOT = _ROOT / "data" / "evaluations" / "engine-selection" +_MANIFEST = _FIXTURE_ROOT / "content-manifest.csv" +_CARRIER_MANIFEST = _FIXTURE_ROOT / "carrier-manifest.csv" +_EXPECTED_PROVIDERS = {"meta", "openai"} + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + errors: list[str] = [] + pairs: dict[str, list[dict[str, str]]] = defaultdict(list) + paths: set[Path] = set() + hashes: set[str] = set() + + with _MANIFEST.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + with _CARRIER_MANIFEST.open(newline="", encoding="utf-8") as stream: + carrier_rows = list(csv.DictReader(stream)) + + for row in rows: + pair_id = row["pair_id"] + relative_path = Path(row["file"]) + path = _FIXTURE_ROOT / relative_path + pairs[pair_id].append(row) + + if relative_path in paths: + errors.append(f"duplicate path: {relative_path}") + paths.add(relative_path) + + expected_hash = row["sha256"] + if expected_hash in hashes: + errors.append(f"duplicate file hash: {expected_hash}") + hashes.add(expected_hash) + + if not path.is_file(): + errors.append(f"missing file: {relative_path}") + continue + actual_hash = _sha256(path) + if actual_hash != expected_hash: + errors.append(f"hash mismatch for {relative_path}: expected {expected_hash}, got {actual_hash}") + + with Image.open(path) as image: + actual_size = image.size + actual_format = image.format + expected_size = (int(row["width"]), int(row["height"])) + if actual_size != expected_size: + errors.append(f"size mismatch for {relative_path}: expected {expected_size}, got {actual_size}") + if actual_format != "PNG": + errors.append(f"format mismatch for {relative_path}: expected PNG, got {actual_format}") + + for pair_id, pair_rows in sorted(pairs.items()): + providers = {row["provider"] for row in pair_rows} + prompts = {row["prompt"] for row in pair_rows} + strata = {row["content_stratum"] for row in pair_rows} + if providers != _EXPECTED_PROVIDERS: + errors.append(f"pair {pair_id} providers: expected {_EXPECTED_PROVIDERS}, got {providers}") + if len(prompts) != 1: + errors.append(f"pair {pair_id} has mismatched prompts") + if len(strata) != 1: + errors.append(f"pair {pair_id} has mismatched content strata") + + expected_pair_ids = {f"{index:02d}" for index in range(19)} + if set(pairs) != expected_pair_ids: + errors.append(f"pair ids: expected {sorted(expected_pair_ids)}, got {sorted(pairs)}") + + committed_files = { + path.relative_to(_FIXTURE_ROOT) for path in (_FIXTURE_ROOT / "originals").glob("**/*") if path.is_file() + } + unlisted = committed_files - paths + if unlisted: + errors.append(f"unlisted files: {sorted(str(path) for path in unlisted)}") + + carrier_providers: set[str] = set() + for row in carrier_rows: + relative_path = Path(row["file"]) + path = (_FIXTURE_ROOT / relative_path).resolve() + carrier_providers.add(row["provider"]) + if not path.is_relative_to(_ROOT / "data"): + errors.append(f"carrier outside data/: {relative_path}") + continue + if not path.is_file(): + errors.append(f"missing carrier: {relative_path}") + continue + actual_hash = _sha256(path) + if actual_hash != row["sha256"]: + errors.append(f"carrier hash mismatch for {relative_path}: expected {row['sha256']}, got {actual_hash}") + if carrier_providers != {"google", "meta", "openai"}: + errors.append(f"carrier providers: expected google/meta/openai; got {sorted(carrier_providers)}") + + if errors: + for error in errors: + log.error("%s", error) + return 1 + + log.info( + "Verified %s content files in %s matched pairs and %s canonical carriers", + len(rows), + len(pairs), + len(carrier_rows), + ) + return 0 + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + raise SystemExit(main()) diff --git a/src/remove_ai_watermarks/_internal/chroma_zimage_pipeline.py b/src/remove_ai_watermarks/_internal/chroma_zimage_pipeline.py index a89bd41..48e8710 100644 --- a/src/remove_ai_watermarks/_internal/chroma_zimage_pipeline.py +++ b/src/remove_ai_watermarks/_internal/chroma_zimage_pipeline.py @@ -36,6 +36,7 @@ from remove_ai_watermarks._internal.two_stage_pipeline import ( _load_prompt_payload, _prompt_cache_path, _store_prompt_payload, + edge_pad_to_grid, ) log = logging.getLogger(__name__) @@ -189,3 +190,31 @@ class ChromaZImagePipeline(TwoStageZImagePipeline): if result.size != image.size: result = result.resize(image.size, Image.Resampling.LANCZOS) return result.convert("RGB") + + def _vae_roundtrip(self, image: Image.Image) -> Image.Image: + """Reconstruct source pixels through the already loaded Chroma VAE.""" + import torch + + pipe = self._load_global() + source_width, source_height = image.size + padded = edge_pad_to_grid(image, _LATENT_GRID) + + tensor = pipe.image_processor.preprocess( + padded, + height=padded.height, + width=padded.width, + ).to(device=self.device, dtype=self.torch_dtype) + with torch.inference_mode(): + encoded = pipe.vae.encode(tensor) + if hasattr(encoded, "latent_dist"): + # A restoration donor must be deterministic. Diffusers samples this + # distribution for img2img noise initialization; its mode is the + # faithful VAE reconstruction needed here. + latents = encoded.latent_dist.mode() + elif hasattr(encoded, "latents"): + latents = encoded.latents + else: + raise AttributeError("Chroma VAE encoder output contains no latents") + decoded = pipe.vae.decode(latents, return_dict=False)[0] + reconstructed = pipe.image_processor.postprocess(decoded, output_type="pil")[0] + return reconstructed.crop((0, 0, source_width, source_height)).convert("RGB") diff --git a/src/remove_ai_watermarks/_internal/qwen_zimage_pipeline.py b/src/remove_ai_watermarks/_internal/qwen_zimage_pipeline.py index d1c9f99..22da37e 100644 --- a/src/remove_ai_watermarks/_internal/qwen_zimage_pipeline.py +++ b/src/remove_ai_watermarks/_internal/qwen_zimage_pipeline.py @@ -18,7 +18,6 @@ import os from dataclasses import dataclass from typing import Any, ClassVar -import numpy as np from PIL import Image from remove_ai_watermarks._internal.two_stage_pipeline import ( @@ -28,6 +27,7 @@ from remove_ai_watermarks._internal.two_stage_pipeline import ( _cache_static_prompt_embeddings, _resize_to_target, build_canny_control_image, + edge_pad_to_grid, ) log = logging.getLogger(__name__) @@ -236,17 +236,7 @@ class QwenZImagePipeline(TwoStageZImagePipeline): pipe, _controlnet_input_cls = self._load_global() source_width, source_height = image.size - pad_width = (-source_width) % 8 - pad_height = (-source_height) % 8 - padded = image.convert("RGB") - if pad_width or pad_height: - padded = Image.fromarray( - np.pad( - np.asarray(padded), - ((0, pad_height), (0, pad_width), (0, 0)), - mode="edge", - ) - ) + padded = edge_pad_to_grid(image, 8) pipe.load_models_to_device(["vae"]) tensor = pipe.preprocess_image(padded).to(device=self.device, dtype=self.torch_dtype) with torch.inference_mode(): diff --git a/src/remove_ai_watermarks/_internal/text_restoration.py b/src/remove_ai_watermarks/_internal/text_restoration.py index 286e776..7b644ea 100644 --- a/src/remove_ai_watermarks/_internal/text_restoration.py +++ b/src/remove_ai_watermarks/_internal/text_restoration.py @@ -1,4 +1,4 @@ -"""Opt-in restoration of verified text from a Qwen VAE reconstruction.""" +"""Opt-in restoration of verified text from a profile VAE reconstruction.""" # pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportUnknownParameterType=false, reportMissingTypeArgument=false, reportMissingTypeStubs=false, reportMissingImports=false, reportArgumentType=false, reportAssignmentType=false, reportReturnType=false, reportCallIssue=false, reportIndexIssue=false, reportOperatorIssue=false from __future__ import annotations @@ -151,7 +151,7 @@ def restore_verified_text( donor: Image.Image, lines: tuple[VerifiedTextLine, ...], ) -> Image.Image: - """Erase candidate glyphs, then composite verified Qwen-VAE glyph cores.""" + """Erase candidate glyphs, then composite verified profile-VAE glyph cores.""" from remove_ai_watermarks import region_eraser if not region_eraser.lama_available(): @@ -162,7 +162,7 @@ def restore_verified_text( candidate_rgb = np.asarray(candidate.convert("RGB")) donor_rgb = np.asarray(donor.convert("RGB")) if source_rgb.shape != candidate_rgb.shape or source_rgb.shape != donor_rgb.shape: - raise ValueError("Source, candidate, and Qwen-VAE donor dimensions must match") + raise ValueError("Source, candidate, and profile-VAE donor dimensions must match") source_masks = [source_silhouette_mask(source_rgb, line.box, line.angle) for line in lines] for index, mask in enumerate(source_masks): diff --git a/src/remove_ai_watermarks/_internal/two_stage_pipeline.py b/src/remove_ai_watermarks/_internal/two_stage_pipeline.py index 3c4f42c..4150d7e 100644 --- a/src/remove_ai_watermarks/_internal/two_stage_pipeline.py +++ b/src/remove_ai_watermarks/_internal/two_stage_pipeline.py @@ -78,6 +78,23 @@ _FACE_PROMPT = "" _FACE_NEGATIVE = "blurry, ugly, bad quality," +def edge_pad_to_grid(image: Image.Image, grid: int) -> Image.Image: + """Pad RGB pixels at the lower/right edges to a latent-grid multiple.""" + width, height = image.size + pad_width = (-width) % grid + pad_height = (-height) % grid + rgb = image.convert("RGB") + if not (pad_width or pad_height): + return rgb + return Image.fromarray( + np.pad( + np.asarray(rgb), + ((0, pad_height), (0, pad_width), (0, 0)), + mode="edge", + ) + ) + + def resolve_face_model_residency( requested: bool | None, *, diff --git a/src/remove_ai_watermarks/_internal/watermark_profiles.py b/src/remove_ai_watermarks/_internal/watermark_profiles.py index 7f1cb41..8ac3f24 100644 --- a/src/remove_ai_watermarks/_internal/watermark_profiles.py +++ b/src/remove_ai_watermarks/_internal/watermark_profiles.py @@ -181,7 +181,7 @@ _ALIASES = { "auto": AUTO_PROFILE, } -# The per-cohort engine router: which profile wins on which vendor, from the +# The deterministic per-cohort selection policy: which profile wins on which vendor, from the # 2026-08-29/30 four-cohort calibration (docs/chroma1-engine-research.md). # chroma-zimage has lower floors AND better matched-strength fidelity on OpenAI # and Microsoft; qwen-zimage wins on Google and Meta. Unknown stays on qwen diff --git a/src/remove_ai_watermarks/_internal/watermark_remover.py b/src/remove_ai_watermarks/_internal/watermark_remover.py index c1097bf..eb0c741 100644 --- a/src/remove_ai_watermarks/_internal/watermark_remover.py +++ b/src/remove_ai_watermarks/_internal/watermark_remover.py @@ -16,6 +16,7 @@ from remove_ai_watermarks._internal.watermark_profiles import ( DEFAULT_PROFILE, INVISIBLE_EXTRA, PROFILE_CHOICES, + QWEN_ZIMAGE_PROFILE, REMOVAL_MODULES, SDXL_ZIMAGE_PROFILE, normalize_profile, @@ -236,7 +237,7 @@ class WatermarkRemover: with Image.open(image_path) as opened: source = opened.convert("RGB") - # The auto router resolves per-image once the provenance vendor is + # The auto policy resolves per-image once the provenance vendor is # known, BEFORE the strength resolution so the right engine's floors # are used. If the vendor changed the engine, reset the cached pipeline. if self._auto: @@ -244,13 +245,15 @@ class WatermarkRemover: if resolved != self.model_profile: self.model_profile = resolved self._qwen_zimage_pipeline = None + if text_manifest is not None and self.model_profile == SDXL_ZIMAGE_PROFILE: + raise ValueError("Verified text restoration is not supported by the sdxl-zimage profile") + if text_manifest is not None and tile: + raise ValueError("Verified text restoration is not calibrated with tiled diffusion") + if fidelity_anchor and self.model_profile != QWEN_ZIMAGE_PROFILE: + raise ValueError("The fidelity anchor is supported only by the qwen-zimage profile") resolved_strength = self._resolve_chroma_strength(strength, vendor, source) if not 0.0 <= resolved_strength <= 1.0: raise ValueError(f"Strength must be between 0.0 and 1.0, got {resolved_strength}") - if text_manifest is not None and self.model_profile in (SDXL_ZIMAGE_PROFILE, CHROMA_ZIMAGE_PROFILE): - raise ValueError("Verified text restoration is supported only by the qwen-zimage profile") - if text_manifest is not None and tile: - raise ValueError("Verified text restoration is not calibrated with tiled diffusion") result = self._load_qwen_zimage_pipeline().run( source, diff --git a/src/remove_ai_watermarks/cli.py b/src/remove_ai_watermarks/cli.py index bdb8063..b443a96 100644 --- a/src/remove_ai_watermarks/cli.py +++ b/src/remove_ai_watermarks/cli.py @@ -345,8 +345,9 @@ _text_manifest_option = click.option( type=click.Path(exists=True, dir_okay=False, path_type=Path), default=None, help=( - "Experimental verified-text restoration manifest. Requires qwen-zimage, " - "the text-restoration extra, native untiled geometry, and no postprocessing." + "Experimental verified-text restoration manifest. Requires qwen-zimage or " + "chroma-zimage, the text-restoration extra, native untiled geometry, and no " + "postprocessing." ), ) diff --git a/src/remove_ai_watermarks/invisible_engine.py b/src/remove_ai_watermarks/invisible_engine.py index 0067520..2e945a2 100644 --- a/src/remove_ai_watermarks/invisible_engine.py +++ b/src/remove_ai_watermarks/invisible_engine.py @@ -18,8 +18,8 @@ from typing import TYPE_CHECKING from ._internal.watermark_profiles import ( DEFAULT_PROFILE, - QWEN_ZIMAGE_PROFILE, REMOVAL_MODULES, + SDXL_ZIMAGE_PROFILE, resolve_adaptive_polish, resolve_seed, ) @@ -186,18 +186,19 @@ class InvisibleEngine: tile_size: Tile dimension in px (default 1024). tile_overlap: Overlap between adjacent tiles in px (default 128). text_manifest: Operator-verified text lines bound to the decoded source - pixels. Enables the experimental Qwen-VAE ``vae-glyphs`` post-pass. - Requires the ``text-restoration`` extra and the ``qwen-zimage`` - profile. Incompatible with downscaling, humanize, unsharp, and - adaptive polish. Tiling is rejected because that combination has - no provider-oracle calibration. + pixels. Enables the experimental profile-VAE ``vae-glyphs`` post-pass. + Requires the ``text-restoration`` extra and either ``qwen-zimage`` + or ``chroma-zimage``. Incompatible with downscaling, humanize, + unsharp, and adaptive polish. Tiling is rejected because that + combination has no provider-oracle calibration. fidelity_anchor: Blend 15% of the Qwen-VAE donor across the whole frame before glyph restoration. OFF by default since 0.27.1: that global blend was measured to return detector-visible OpenAI SynthID on poster-scale manifests (detected x6 with the anchor vs clean x6 without it, base clean; official Content Provenance API, 2026-08-19 - docs/text-protection-research.md). ``True`` reproduces - the 0.27.0 research behavior. Requires ``text_manifest``. + the 0.27.0 research behavior. Requires ``text_manifest`` and the + ``qwen-zimage`` profile. Returns: Path to the cleaned image. @@ -210,8 +211,8 @@ class InvisibleEngine: if fidelity_anchor and text_manifest is None: raise ValueError("fidelity_anchor requires a text manifest") if text_manifest is not None: - if self._remover.model_profile != QWEN_ZIMAGE_PROFILE: - raise ValueError("--text-manifest is supported only by the qwen-zimage profile") + if self._remover.model_profile == SDXL_ZIMAGE_PROFILE: + raise ValueError("--text-manifest is not supported by the sdxl-zimage profile") if tile: raise ValueError("--text-manifest is not calibrated with --tile") if max_resolution != 0: diff --git a/tests/test_engine_selection_analysis.py b/tests/test_engine_selection_analysis.py new file mode 100644 index 0000000..baac3cb --- /dev/null +++ b/tests/test_engine_selection_analysis.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "analyze_engine_selection_study.py" +SPEC = importlib.util.spec_from_file_location("analyze_engine_selection_study", SCRIPT) +assert SPEC is not None +assert SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def _row(*, ssim: float, lpips: float) -> dict[str, Any]: + common = { + "psnr_db": 30.0, + "mae": 2.0, + "edge_f1": 0.8, + "laplacian_ratio": 0.9, + } + return { + "pair_id": "00", + "provider": "openai", + "content_stratum": "test", + "fidelity": {**common, "ssim": ssim, "lpips_512": lpips}, + } + + +def test_exact_sign_test_two_sided() -> None: + assert MODULE._sign_test_two_sided(19, 0) == pytest.approx(2 / 2**19) + assert MODULE._sign_test_two_sided(12, 7) == pytest.approx(0.359283447265625) + assert MODULE._sign_test_two_sided(0, 0) == 1.0 + + +def test_analysis_respects_each_metrics_preferred_direction() -> None: + report = { + "qwen-zimage": {"rows": [_row(ssim=0.8, lpips=0.1)]}, + "chroma-zimage": {"rows": [_row(ssim=0.9, lpips=0.2)]}, + } + + metrics = MODULE.analyze(report)["providers"]["openai"]["metrics"] + + assert metrics["ssim"]["chroma_wins"] == 1 + assert metrics["ssim"]["qwen_wins"] == 0 + assert metrics["lpips_512"]["chroma_wins"] == 0 + assert metrics["lpips_512"]["qwen_wins"] == 1 diff --git a/tests/test_invisible_engine.py b/tests/test_invisible_engine.py index e98ead6..2b3eba0 100644 --- a/tests/test_invisible_engine.py +++ b/tests/test_invisible_engine.py @@ -59,7 +59,7 @@ class TestVerifiedTextMode: manifest = tmp_path / "manifest.json" manifest.write_text("{}", encoding="utf-8") cases = ( - ("sdxl-zimage", {}, "qwen-zimage"), + ("sdxl-zimage", {}, "not supported by the sdxl-zimage profile"), ("qwen-zimage", {"tile": True}, "not calibrated with --tile"), ("qwen-zimage", {"max_resolution": 1024}, "max-resolution 0"), ("qwen-zimage", {"humanize": 1.0}, "humanize=0"), @@ -113,7 +113,9 @@ class TestVerifiedTextMode: Image.open(kwargs["image_path"]).save(kwargs["output_path"]) return kwargs["output_path"] - engine = self._engine() + # The public engine must not reject auto before WatermarkRemover resolves + # the measured per-vendor profile. + engine = self._engine("auto") engine._remover.remove_watermark = fake_remove monkeypatch.setattr(region_eraser, "lama_available", lambda: True) diff --git a/tests/test_qwen_zimage_pipeline.py b/tests/test_qwen_zimage_pipeline.py index 88508b1..1a66539 100644 --- a/tests/test_qwen_zimage_pipeline.py +++ b/tests/test_qwen_zimage_pipeline.py @@ -1154,9 +1154,8 @@ def test_chroma_zimage_floors_to_its_own_latent_grid(): assert chroma_target_size(3, 3) == (16, 16) -def test_watermark_remover_rejects_text_manifest_for_chroma(tmp_path, monkeypatch): - """Verified text restoration needs the Qwen VAE donor; chroma-zimage and - sdxl-zimage must both be rejected at the boundary, not deep inside.""" +def test_watermark_remover_accepts_text_manifest_for_chroma_but_not_sdxl(tmp_path, monkeypatch): + """Chroma owns a VAE donor; SDXL still fails at the public boundary.""" from remove_ai_watermarks._internal.text_restoration import VerifiedTextLine, VerifiedTextManifest from remove_ai_watermarks._internal.watermark_remover import WatermarkRemover @@ -1165,10 +1164,22 @@ def test_watermark_remover_rejects_text_manifest_for_chroma(tmp_path, monkeypatc Image.new("RGB", (96, 80)).save(source) manifest = VerifiedTextManifest("0" * 64, 96, 80, (VerifiedTextLine((4, 4, 20, 16), "x", "alphabetic"),)) - for profile in ("chroma-zimage", "sdxl-zimage"): - remover = WatermarkRemover(device="cuda", pipeline=profile) - with pytest.raises(ValueError, match="only by the qwen-zimage profile"): - remover.remove_watermark(source, text_manifest=manifest) + chroma = WatermarkRemover(device="cuda", pipeline="chroma-zimage") + chroma_runtime = MagicMock() + chroma_runtime.run.return_value = Image.new("RGB", (96, 80)) + monkeypatch.setattr(chroma, "_load_qwen_zimage_pipeline", lambda: chroma_runtime) + + chroma.remove_watermark(source, text_manifest=manifest) + + assert chroma_runtime.run.call_count == 1 + assert chroma_runtime.run.call_args.kwargs["text_manifest"] is manifest + with pytest.raises(ValueError, match=r"fidelity anchor.*qwen-zimage"): + chroma.remove_watermark(source, text_manifest=manifest, fidelity_anchor=True) + assert chroma_runtime.run.call_count == 1 + + sdxl = WatermarkRemover(device="cuda", pipeline="sdxl-zimage") + with pytest.raises(ValueError, match="not supported by the sdxl-zimage profile"): + sdxl.remove_watermark(source, text_manifest=manifest) def test_watermark_remover_dispatches_to_chroma_pipeline(monkeypatch): @@ -1209,17 +1220,56 @@ def test_chroma_adaptive_polish_defaults_off(): assert resolve_adaptive_polish(True, "chroma-zimage") is True -def test_chroma_vae_roundtrip_raises_not_implemented(): - """The base class provides no VAE donor for chroma-zimage; the failure must - be the explicit NotImplementedError, not an opaque crash from a missing - Qwen stack.""" +def test_chroma_vae_roundtrip_uses_deterministic_mode_and_restores_native_size(monkeypatch): + """The Chroma donor pads without distorting geometry and never samples.""" import torch from remove_ai_watermarks._internal.chroma_zimage_pipeline import ChromaZImagePipeline - pipeline = ChromaZImagePipeline(device="cuda", torch_dtype=torch.bfloat16) - with pytest.raises(NotImplementedError, match="does not expose a VAE donor"): - pipeline._vae_roundtrip(Image.new("RGB", (32, 32))) + seen: dict[str, object] = {} + latent = torch.ones(1, 4, 4, 6) + + class LatentDistribution: + def mode(self): + seen["mode_calls"] = int(seen.get("mode_calls", 0)) + 1 + return latent + + def sample(self, *_args, **_kwargs): + raise AssertionError("A verified-text donor must not sample VAE latents") + + class FakeVae: + @staticmethod + def encode(tensor): + seen["encoded_shape"] = tuple(tensor.shape) + return type("Encoded", (), {"latent_dist": LatentDistribution()})() + + @staticmethod + def decode(received, *, return_dict): + assert received is latent + assert return_dict is False + return (torch.zeros(1, 3, 32, 48),) + + class FakeProcessor: + @staticmethod + def preprocess(image, *, height, width): + seen["preprocess"] = (image.size, height, width) + return torch.zeros(1, 3, height, width) + + @staticmethod + def postprocess(decoded, *, output_type): + assert output_type == "pil" + return [Image.new("RGB", (decoded.shape[3], decoded.shape[2]))] + + pipe = type("FakePipe", (), {"vae": FakeVae(), "image_processor": FakeProcessor()})() + pipeline = ChromaZImagePipeline(device="cpu", torch_dtype=torch.float32) + monkeypatch.setattr(pipeline, "_load_global", lambda: pipe) + + result = pipeline._vae_roundtrip(Image.new("RGB", (35, 19))) + + assert seen["preprocess"] == ((48, 32), 32, 48) + assert seen["encoded_shape"] == (1, 3, 32, 48) + assert seen["mode_calls"] == 1 + assert result.size == (35, 19) def test_chroma_google_face_content_gets_the_lower_adaptive_floor(): @@ -1296,9 +1346,8 @@ def test_auto_profile_strength_uses_the_resolved_engine_floors(tmp_path, monkeyp assert kwargs["strength"] == pytest.approx(0.09) -def test_auto_profile_text_manifest_stays_on_qwen(tmp_path, monkeypatch): - """Auto routes OpenAI to chroma-zimage, which rejects text manifests; - the rejection must reach the caller at the boundary.""" +def test_auto_profile_text_manifest_uses_the_measured_chroma_engine_once(tmp_path, monkeypatch): + """A manifest does not override the researched OpenAI-to-Chroma decision.""" from remove_ai_watermarks._internal.text_restoration import VerifiedTextLine, VerifiedTextManifest from remove_ai_watermarks._internal.watermark_remover import WatermarkRemover @@ -1308,5 +1357,12 @@ def test_auto_profile_text_manifest_stays_on_qwen(tmp_path, monkeypatch): manifest = VerifiedTextManifest("0" * 64, 96, 80, (VerifiedTextLine((4, 4, 20, 16), "x", "alphabetic"),)) remover = WatermarkRemover(device="cuda", pipeline="auto") - with pytest.raises(ValueError, match="only by the qwen-zimage profile"): - remover.remove_watermark(source, vendor="openai", text_manifest=manifest) + runtime = MagicMock() + runtime.run.return_value = Image.new("RGB", (96, 80)) + monkeypatch.setattr(remover, "_load_qwen_zimage_pipeline", lambda: runtime) + + remover.remove_watermark(source, vendor="openai", text_manifest=manifest) + + assert remover.model_profile == "chroma-zimage" + assert runtime.run.call_count == 1 + assert runtime.run.call_args.kwargs["text_manifest"] is manifest