diff --git a/data/README.md b/data/README.md index a4580d6..00e80a4 100644 --- a/data/README.md +++ b/data/README.md @@ -19,7 +19,8 @@ data/ Reusable full-pipeline evaluation selection evaluations/ engine-selection/ - Prompt-matched AI content matrix for deterministic auto-engine research + Prompt-matched, publication-cleared AI content matrix for + deterministic auto-engine and watermark benchmark research fidelity/ Evaluation instructions and hand-verified ground truth video-synthid-oracle.csv Reproducible full-clip Gemini SynthID verdicts diff --git a/data/evaluations/engine-selection/README.md b/data/evaluations/engine-selection/README.md index 39ece27..8c7d7c9 100644 --- a/data/evaluations/engine-selection/README.md +++ b/data/evaluations/engine-selection/README.md @@ -23,6 +23,11 @@ hash under `sha256`; `source_payload_sha256` preserves that value while `sha256` always means the actual file bytes here. Meta's two hashes are the same. +The project maintainer has cleared these exact committed outputs for permanent +public test and benchmark reuse. `reuse_basis` records that clearance on every +row. It is permission for this fixed fixture set, not a claim that either +provider grants a general-purpose dataset license for arbitrary generations. + One generation per prompt and provider is enough for the first paired discovery pass. Any policy inferred from it must be challenged against unused generation indices from the local research archive before shipping. Generated diff --git a/data/evaluations/engine-selection/content-manifest.csv b/data/evaluations/engine-selection/content-manifest.csv index 0367378..02bbcf9 100644 --- a/data/evaluations/engine-selection/content-manifest.csv +++ b/data/evaluations/engine-selection/content-manifest.csv @@ -1,39 +1,39 @@ -pair_id,provider,model,file,width,height,content_stratum,sha256,source_payload_sha256,prompt -00,openai,gpt-image-1-mini,originals/openai/00.png,1024,1024,natural_landscape,698896455e0f1de57a227339227bf068143e690873b9c8c9672b31578de93ba0,1345f59fc7b51a1edf837745245bb90811feefd9ba7f9ba6f12dc89846aeba18,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" -00,meta,muse-image-1.0,originals/meta/00.png,1600,1600,natural_landscape,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" -01,openai,gpt-image-1-mini,originals/openai/01.png,1024,1024,street_scene,b885d8cb1490e4fbbe580b87750008780671390097d28e71ff18b50cf320ec85,772518e4686940d2c68df114298fb007a97a5c42548245092e989b6ca9bd5b90,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" -01,meta,muse-image-1.0,originals/meta/01.png,1600,1600,street_scene,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" -02,openai,gpt-image-1-mini,originals/openai/02.png,1024,1024,single_portrait,97618d1a5c39ddeba708df3a9448b27d1d26583ee68b490ba79f7f3e1eb2248d,6bace9d2adae4d630e1830449f0945a30d7ddb3635e3005bc7aad80206a57176,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" -02,meta,muse-image-1.0,originals/meta/02.png,1600,1600,single_portrait,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" -03,openai,gpt-image-1-mini,originals/openai/03.png,1024,1024,animal_action,eb6920aab03bbe6935a01756c8f2d524d383e307369cdfaad3fe44926fbd9e48,2012abb15be639ebb3c596d2210f6d4c50cce0e11dd6cfc88aeb29de1c82de18,"a golden retriever running through a field of sunflowers, action photography" -03,meta,muse-image-1.0,originals/meta/03.png,1600,1600,animal_action,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,"a golden retriever running through a field of sunflowers, action photography" -04,openai,gpt-image-1-mini,originals/openai/04.png,1024,1024,reflective_architecture,275d68de881041de989261ced7570dd460028e0406346642d880d0d1664bb9ae,1808d95c69080aafced4234e4c3ad7714121bad00f3f5dc84a744288d9757245,"a modern architecture building with glass facade reflecting sunset clouds" -04,meta,muse-image-1.0,originals/meta/04.png,1600,1600,reflective_architecture,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,"a modern architecture building with glass facade reflecting sunset clouds" -05,openai,gpt-image-1-mini,originals/openai/05.png,1024,1024,food_closeup,316a743ccf29f025904d1bcd047a6996f987715e4f93e8a0d9cf85b624349b07,a8bac088c350ef849bbf78a85a1e44aa1665d1054a0cdfae29461c1cdca56e4a,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" -05,meta,muse-image-1.0,originals/meta/05.png,1600,1600,food_closeup,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" -06,openai,gpt-image-1-mini,originals/openai/06.png,1024,1024,sports_action,390d87d1f59eb4fa38590c0c751b2848383c77ffcf549571a0bf00db0f783763,6cc24eeb0fce95a86741b9ac0c055002793a586ba37f0cf4d64201a5c5e73513,"a surfer riding a barrel wave at golden hour, backlit spray" -06,meta,muse-image-1.0,originals/meta/06.png,1600,1600,sports_action,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,"a surfer riding a barrel wave at golden hour, backlit spray" -07,openai,gpt-image-1-mini,originals/openai/07.png,1024,1024,interior_low_light,4c77c2814216bdcde3417d2d228c5b6f275e33777f13cbea7911241bc9bc376b,9e858b95a4f842c917cbf47afeb4b7e0e24e04c4df0b19da4afe00b7316a13a9,"a cozy reading nook with warm lamp light and rain on the window" -07,meta,muse-image-1.0,originals/meta/07.png,1600,1600,interior_low_light,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,"a cozy reading nook with warm lamp light and rain on the window" -08,openai,gpt-image-1-mini,originals/openai/08.png,1024,1024,isometric_illustration,d688f6da74d4a7b2bd29ef59ba64864d0915134433febd8c90eb5ce1dc80781f,37a801f66b7c1bf63e613a7a5717dbc00397120a7a32ec45414661e03137a180,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" -08,meta,muse-image-1.0,originals/meta/08.png,1600,1600,isometric_illustration,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" -09,openai,gpt-image-1-mini,originals/openai/09.png,1024,1024,watercolor,d0beefb80ee00a0937a851932a2ed6dab2c670f10104ffe94c5ff22869a9595d,b52401235efc16641d3cb8f9c7c6d20a75c36751ada9409b65ceefa0e57901da,"a watercolor painting of Venice canals at sunset, loose brushstrokes" -09,meta,muse-image-1.0,originals/meta/09.png,1600,1600,watercolor,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,"a watercolor painting of Venice canals at sunset, loose brushstrokes" -10,openai,gpt-image-1-mini,originals/openai/10.png,1024,1024,macro_detail,59d9c6dce5ce4c2b77427144ccbafc8a4b7ab87564673677119859c31598b0d4,3740f9de304f050fc7306447c6668dcaa8156a888709f759f05715e7987a321c,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" -10,meta,muse-image-1.0,originals/meta/10.png,1600,1600,macro_detail,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" -11,openai,gpt-image-1-mini,originals/openai/11.png,1024,1024,urban_night,e4f6f806ed8559973374eae1e4d379807e45046749b7d491edbbc8ad9647728e,06bfc68289ece1b85b169df4cedd5e1cf12d0efa30631bdecd0e3edce4d8c82e,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" -11,meta,muse-image-1.0,originals/meta/11.png,1600,1600,urban_night,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" -12,openai,gpt-image-1-mini,originals/openai/12.png,1024,1024,flat_illustration,6bf230aa0b6f40d7ff0d23b22b8aac2e9b66288eb85c02eebdf43b71a1597f5b,65567d5cecb34241ac755ac4db5eb5a7b271d9a00fa27d826c961d2a2ef6ac3d,"a flat-design poster of Santorini with bold shapes, screen print style" -12,meta,muse-image-1.0,originals/meta/12.png,1600,1600,flat_illustration,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,"a flat-design poster of Santorini with bold shapes, screen print style" -13,openai,gpt-image-1-mini,originals/openai/13.png,1024,1024,storm_landscape,d07b05eef633fcb5b6b54268cc2cdd787dc19f8050198872e462f69e383a054d,a692a8a0daebad375cd57d7c671be6e16ad3ef700f0b9214daacbe80b6148c90,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" -13,meta,muse-image-1.0,originals/meta/13.png,1600,1600,storm_landscape,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" -14,openai,gpt-image-1-mini,originals/openai/14.png,1024,1024,monochrome_portrait,9fb1153f4d0041932b35bcd1408d774b642c68f81d7bb403d22f50f8c891ede7,826acf13df78378499e57c7fe1a8309947ab26810f46482be3a64d3328de1ff6,"a black and white portrait of an elderly craftsman in his workshop" -14,meta,muse-image-1.0,originals/meta/14.png,1600,1600,monochrome_portrait,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,"a black and white portrait of an elderly craftsman in his workshop" -15,openai,gpt-image-1-mini,originals/openai/15.png,1024,1024,surreal_composite,8bf02f9028fd049ebfafd3410d2eb9d618819be1c14b4e05d53a490e22ba109c,e71de5a9a2f6579857d2e24b8e7a1516031f19ee824906cece158101d13fe001,"a surreal double exposure of a forest and a human profile, conceptual art" -15,meta,muse-image-1.0,originals/meta/15.png,1600,1600,surreal_composite,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,"a surreal double exposure of a forest and a human profile, conceptual art" -16,openai,gpt-image-1-mini,originals/openai/16.png,1024,1024,brutalist_architecture,e91ca5128ce8644510c5c4def6bba085e45e1716105b3b6a2761cc18d42018b3,0aa8a1819f160ce26dfc9e59b996f969698fdf4eba3402fe999af598fcad80d4,"a low-angle shot of a brutalist building against dramatic clouds" -16,meta,muse-image-1.0,originals/meta/16.png,1600,1600,brutalist_architecture,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,"a low-angle shot of a brutalist building against dramatic clouds" -17,openai,gpt-image-1-mini,originals/openai/17.png,1024,1024,product_still_life,4f41f4a46b80c0ded61a9a3ce1273412b1728476a0734519d1af34a1797013b0,1f60c25c59138c14bacb0dd485475037e7d6222bb45daa66ed85f9dd91c83e4e,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" -17,meta,muse-image-1.0,originals/meta/17.png,1600,1600,product_still_life,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" -18,openai,gpt-image-1-mini,originals/openai/18.png,1024,1024,person_in_context,9b049d3112a654a87022d591b93a01087693c0664be5af560d3c0ec490291963,ba873bc7989720245821f5298f12e36bd7b6051909694b72569645c29cdd6bd0,"a lifestyle photo of a person working on a laptop in a bright cafe" -18,meta,muse-image-1.0,originals/meta/18.png,1600,1600,person_in_context,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,"a lifestyle photo of a person working on a laptop in a bright cafe" +pair_id,provider,model,file,width,height,content_stratum,sha256,source_payload_sha256,reuse_basis,prompt +00,openai,gpt-image-1-mini,originals/openai/00.png,1024,1024,natural_landscape,698896455e0f1de57a227339227bf068143e690873b9c8c9672b31578de93ba0,1345f59fc7b51a1edf837745245bb90811feefd9ba7f9ba6f12dc89846aeba18,project-maintainer-public-test-clearance,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" +00,meta,muse-image-1.0,originals/meta/00.png,1600,1600,natural_landscape,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,83362c189b79cd0e700296b975e9ef8e7f2b2fe89813e407bc795299400ca5e0,project-maintainer-public-test-clearance,"a stunning photograph of a mountain lake at sunrise, highly detailed, professional photography" +01,openai,gpt-image-1-mini,originals/openai/01.png,1024,1024,street_scene,b885d8cb1490e4fbbe580b87750008780671390097d28e71ff18b50cf320ec85,772518e4686940d2c68df114298fb007a97a5c42548245092e989b6ca9bd5b90,project-maintainer-public-test-clearance,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" +01,meta,muse-image-1.0,originals/meta/01.png,1600,1600,street_scene,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,5ba08cf9ab07bf00f64f39fe93f1766a64f737817b00086783476566c5f3317f,project-maintainer-public-test-clearance,"a vibrant street market in Bangkok with colorful fruits and spices, candid photo" +02,openai,gpt-image-1-mini,originals/openai/02.png,1024,1024,single_portrait,97618d1a5c39ddeba708df3a9448b27d1d26583ee68b490ba79f7f3e1eb2248d,6bace9d2adae4d630e1830449f0945a30d7ddb3635e3005bc7aad80206a57176,project-maintainer-public-test-clearance,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" +02,meta,muse-image-1.0,originals/meta/02.png,1600,1600,single_portrait,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,4daf07fc8bdfe007b96a70b00ec0cfde87f180eb8c2d5501723e218ed29e7589,project-maintainer-public-test-clearance,"a close-up portrait of a woman with freckles and green eyes, studio lighting, 85mm lens" +03,openai,gpt-image-1-mini,originals/openai/03.png,1024,1024,animal_action,eb6920aab03bbe6935a01756c8f2d524d383e307369cdfaad3fe44926fbd9e48,2012abb15be639ebb3c596d2210f6d4c50cce0e11dd6cfc88aeb29de1c82de18,project-maintainer-public-test-clearance,"a golden retriever running through a field of sunflowers, action photography" +03,meta,muse-image-1.0,originals/meta/03.png,1600,1600,animal_action,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,93dce3a5049f773adb882ef22243a84e4822b9c43ad30774741c40f3844ef405,project-maintainer-public-test-clearance,"a golden retriever running through a field of sunflowers, action photography" +04,openai,gpt-image-1-mini,originals/openai/04.png,1024,1024,reflective_architecture,275d68de881041de989261ced7570dd460028e0406346642d880d0d1664bb9ae,1808d95c69080aafced4234e4c3ad7714121bad00f3f5dc84a744288d9757245,project-maintainer-public-test-clearance,"a modern architecture building with glass facade reflecting sunset clouds" +04,meta,muse-image-1.0,originals/meta/04.png,1600,1600,reflective_architecture,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,209c71f0b737a9ef60b4d620c66f518b34dbc42af4532b2b8e0b3086183c90ec,project-maintainer-public-test-clearance,"a modern architecture building with glass facade reflecting sunset clouds" +05,openai,gpt-image-1-mini,originals/openai/05.png,1024,1024,food_closeup,316a743ccf29f025904d1bcd047a6996f987715e4f93e8a0d9cf85b624349b07,a8bac088c350ef849bbf78a85a1e44aa1665d1054a0cdfae29461c1cdca56e4a,project-maintainer-public-test-clearance,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" +05,meta,muse-image-1.0,originals/meta/05.png,1600,1600,food_closeup,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,4a50ca3e33ddacd2897b926cdc54615a389416fcd2edf4d8803ad882b85b33af,project-maintainer-public-test-clearance,"a bowl of ramen with steam rising, moody food photography, shallow depth of field" +06,openai,gpt-image-1-mini,originals/openai/06.png,1024,1024,sports_action,390d87d1f59eb4fa38590c0c751b2848383c77ffcf549571a0bf00db0f783763,6cc24eeb0fce95a86741b9ac0c055002793a586ba37f0cf4d64201a5c5e73513,project-maintainer-public-test-clearance,"a surfer riding a barrel wave at golden hour, backlit spray" +06,meta,muse-image-1.0,originals/meta/06.png,1600,1600,sports_action,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,1d9fcbf0f471f733ae96ca94dff892afd9f5852767000e379d3486dc94e30b78,project-maintainer-public-test-clearance,"a surfer riding a barrel wave at golden hour, backlit spray" +07,openai,gpt-image-1-mini,originals/openai/07.png,1024,1024,interior_low_light,4c77c2814216bdcde3417d2d228c5b6f275e33777f13cbea7911241bc9bc376b,9e858b95a4f842c917cbf47afeb4b7e0e24e04c4df0b19da4afe00b7316a13a9,project-maintainer-public-test-clearance,"a cozy reading nook with warm lamp light and rain on the window" +07,meta,muse-image-1.0,originals/meta/07.png,1600,1600,interior_low_light,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,ecec76c086a7b33beec1b3cb787f62cad68c09cb362a8888a8af734e9d4f3425,project-maintainer-public-test-clearance,"a cozy reading nook with warm lamp light and rain on the window" +08,openai,gpt-image-1-mini,originals/openai/08.png,1024,1024,isometric_illustration,d688f6da74d4a7b2bd29ef59ba64864d0915134433febd8c90eb5ce1dc80781f,37a801f66b7c1bf63e613a7a5717dbc00397120a7a32ec45414661e03137a180,project-maintainer-public-test-clearance,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" +08,meta,muse-image-1.0,originals/meta/08.png,1600,1600,isometric_illustration,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,d4683d16a55c5f358f5b3b876b89fbc810ae1e3332e74399395ae20c9f646ef5,project-maintainer-public-test-clearance,"an isometric illustration of a cozy coffee shop interior, warm colors, detailed" +09,openai,gpt-image-1-mini,originals/openai/09.png,1024,1024,watercolor,d0beefb80ee00a0937a851932a2ed6dab2c670f10104ffe94c5ff22869a9595d,b52401235efc16641d3cb8f9c7c6d20a75c36751ada9409b65ceefa0e57901da,project-maintainer-public-test-clearance,"a watercolor painting of Venice canals at sunset, loose brushstrokes" +09,meta,muse-image-1.0,originals/meta/09.png,1600,1600,watercolor,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,c76ffbca59fbcb2dba9b66c94598f0c0062b0ac4679e348cdf1e18a06d3eb64d,project-maintainer-public-test-clearance,"a watercolor painting of Venice canals at sunset, loose brushstrokes" +10,openai,gpt-image-1-mini,originals/openai/10.png,1024,1024,macro_detail,59d9c6dce5ce4c2b77427144ccbafc8a4b7ab87564673677119859c31598b0d4,3740f9de304f050fc7306447c6668dcaa8156a888709f759f05715e7987a321c,project-maintainer-public-test-clearance,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" +10,meta,muse-image-1.0,originals/meta/10.png,1600,1600,macro_detail,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,c86fd9c3a59aa0bec709c5bce7c2ef1be4ecb1d780f3a3f35460b73cd97d1a12,project-maintainer-public-test-clearance,"a macro photograph of a dewdrop on a spider web at dawn, bokeh background" +11,openai,gpt-image-1-mini,originals/openai/11.png,1024,1024,urban_night,e4f6f806ed8559973374eae1e4d379807e45046749b7d491edbbc8ad9647728e,06bfc68289ece1b85b169df4cedd5e1cf12d0efa30631bdecd0e3edce4d8c82e,project-maintainer-public-test-clearance,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" +11,meta,muse-image-1.0,originals/meta/11.png,1600,1600,urban_night,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,7877cf0c13b07fbc6651178d838db4ab253369896292af1cf3f85a957a559db8,project-maintainer-public-test-clearance,"a cinematic wide shot of Tokyo at night with neon reflections on wet streets" +12,openai,gpt-image-1-mini,originals/openai/12.png,1024,1024,flat_illustration,6bf230aa0b6f40d7ff0d23b22b8aac2e9b66288eb85c02eebdf43b71a1597f5b,65567d5cecb34241ac755ac4db5eb5a7b271d9a00fa27d826c961d2a2ef6ac3d,project-maintainer-public-test-clearance,"a flat-design poster of Santorini with bold shapes, screen print style" +12,meta,muse-image-1.0,originals/meta/12.png,1600,1600,flat_illustration,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,1b71bd94aa18d737c8a1c319a33dd0f0e9e15c2e93210c9c3aa98242b9a67955,project-maintainer-public-test-clearance,"a flat-design poster of Santorini with bold shapes, screen print style" +13,openai,gpt-image-1-mini,originals/openai/13.png,1024,1024,storm_landscape,d07b05eef633fcb5b6b54268cc2cdd787dc19f8050198872e462f69e383a054d,a692a8a0daebad375cd57d7c671be6e16ad3ef700f0b9214daacbe80b6148c90,project-maintainer-public-test-clearance,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" +13,meta,muse-image-1.0,originals/meta/13.png,1600,1600,storm_landscape,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,e669128fe2d3b59dc83ae0b216c5378fe5930e77fa3a692219da238dbf55d5f6,project-maintainer-public-test-clearance,"a dramatic storm cloud over a lonely lighthouse, moody, high contrast photography" +14,openai,gpt-image-1-mini,originals/openai/14.png,1024,1024,monochrome_portrait,9fb1153f4d0041932b35bcd1408d774b642c68f81d7bb403d22f50f8c891ede7,826acf13df78378499e57c7fe1a8309947ab26810f46482be3a64d3328de1ff6,project-maintainer-public-test-clearance,"a black and white portrait of an elderly craftsman in his workshop" +14,meta,muse-image-1.0,originals/meta/14.png,1600,1600,monochrome_portrait,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,8339475f0585491ea08992d29890f3c759f24db2184212c6bf4f7efc0932e177,project-maintainer-public-test-clearance,"a black and white portrait of an elderly craftsman in his workshop" +15,openai,gpt-image-1-mini,originals/openai/15.png,1024,1024,surreal_composite,8bf02f9028fd049ebfafd3410d2eb9d618819be1c14b4e05d53a490e22ba109c,e71de5a9a2f6579857d2e24b8e7a1516031f19ee824906cece158101d13fe001,project-maintainer-public-test-clearance,"a surreal double exposure of a forest and a human profile, conceptual art" +15,meta,muse-image-1.0,originals/meta/15.png,1600,1600,surreal_composite,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,20628541cc924e623984604337807c7afd453aa8a7652f087add742a13828f6a,project-maintainer-public-test-clearance,"a surreal double exposure of a forest and a human profile, conceptual art" +16,openai,gpt-image-1-mini,originals/openai/16.png,1024,1024,brutalist_architecture,e91ca5128ce8644510c5c4def6bba085e45e1716105b3b6a2761cc18d42018b3,0aa8a1819f160ce26dfc9e59b996f969698fdf4eba3402fe999af598fcad80d4,project-maintainer-public-test-clearance,"a low-angle shot of a brutalist building against dramatic clouds" +16,meta,muse-image-1.0,originals/meta/16.png,1600,1600,brutalist_architecture,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,d22702d26343b6575e89751c95d865696c6085bed053f6008f989434a96bac02,project-maintainer-public-test-clearance,"a low-angle shot of a brutalist building against dramatic clouds" +17,openai,gpt-image-1-mini,originals/openai/17.png,1024,1024,product_still_life,4f41f4a46b80c0ded61a9a3ce1273412b1728476a0734519d1af34a1797013b0,1f60c25c59138c14bacb0dd485475037e7d6222bb45daa66ed85f9dd91c83e4e,project-maintainer-public-test-clearance,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" +17,meta,muse-image-1.0,originals/meta/17.png,1600,1600,product_still_life,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,67bde86edfd3db08b88c0dbc5572958408875e188dac6fa8ed64f821b7e7e994,project-maintainer-public-test-clearance,"a minimalist product photo of a ceramic mug on a marble surface, clean commercial look" +18,openai,gpt-image-1-mini,originals/openai/18.png,1024,1024,person_in_context,9b049d3112a654a87022d591b93a01087693c0664be5af560d3c0ec490291963,ba873bc7989720245821f5298f12e36bd7b6051909694b72569645c29cdd6bd0,project-maintainer-public-test-clearance,"a lifestyle photo of a person working on a laptop in a bright cafe" +18,meta,muse-image-1.0,originals/meta/18.png,1600,1600,person_in_context,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,cc4b2bcfc4e80d7dfe6646beb12529168137aef04d2b6765c6382b830ce19ed7,project-maintainer-public-test-clearance,"a lifestyle photo of a person working on a laptop in a bright cafe" diff --git a/docs/benchmark-kernel.md b/docs/benchmark-kernel.md index 11e79c7..6b7c129 100644 --- a/docs/benchmark-kernel.md +++ b/docs/benchmark-kernel.md @@ -102,7 +102,8 @@ output as a separate `removed` state. ```bash uv run python scripts/watermark_benchmark_cohort.py \ - --output-dir .local-eval/watermark-benchmark-cohort-v1 + --output-dir .local-eval/watermark-benchmark-cohort-v1 \ + --size 512 uv run python scripts/watermark_benchmark.py \ .local-eval/watermark-benchmark-cohort-v1/manifest.jsonl \ --output .local-eval/watermark-benchmark-cohort-v1-results.jsonl @@ -115,11 +116,15 @@ from the production detector so the generated positive cannot silently drift from the matcher. TrustMark's optional package and model weights must already be available. +`--size` accepts square carrier sides of at least 256 pixels, divisible by 16. +The size is part of every case identity and transform record, so resource sweeps +at multiple boundaries can be aggregated without merging unlike artifacts. + Attacked and removed rows intentionally use `expected: unresolved`: they are measurements, not assertions smuggled into fixtures. The cohort is too small and too synthetic for prevalence, confidence, or release-quality claims. It is a deterministic first baseline that proves the harness and exposes gross -detector behavior before a licensed, stratified real-image cohort is added. +detector behavior before the real-image cohort below is run. ### Development baseline, 2026-09-05 @@ -168,6 +173,108 @@ TrustMark cold start matters only when validation is the first TrustMark use in the process; a workflow that already identified the source pays the warm cost. The sample is deliberately diagnostic rather than statistically representative. +## Real-image cohort v1 + +`scripts/watermark_benchmark_real_cohort.py` replaces procedural carriers with +the 38 committed OpenAI/Meta images in the engine-selection content matrix. The +matrix has 19 prompt-matched content strata, one image per provider in every +stratum, and no duplicate file hash. Its images are project-collected model +outputs rather than camera originals; "real-image" here means real, decoded +content carriers rather than generated gradients or checkerboards. + +Every source row carries +`reuse_basis: project-maintainer-public-test-clearance`. This records permanent +public test and benchmark permission for those exact committed bytes. It does +not claim that provider terms grant a general dataset license for arbitrary +generations. The builder refuses a source without that explicit basis, a hash +or dimension mismatch, repeated pixels, an incomplete provider pair, or a +duplicate provider/content stratum. + +```bash +uv run python scripts/watermark_benchmark_real_cohort.py \ + --output-dir .local-eval/watermark-benchmark-real-v1 \ + --size 512 +uv run python scripts/watermark_benchmark.py \ + .local-eval/watermark-benchmark-real-v1/manifest.jsonl \ + --output .local-eval/watermark-benchmark-real-v1-results.jsonl +``` + +Each source is deterministically center-cropped only if necessary, resized to +the selected square geometry with LANCZOS, converted to RGB, and written as a +PNG reference. At the default size all current sources are already square, so +only the downscale and RGB conversion apply. The source manifest hash, original +file and payload hashes, provider, model, content stratum, native dimensions, +reuse basis, prompt hash, and standardization recipe travel with every case. + +The default build has 798 cases: all three production DWT-DCT patterns plus +TrustMark Variant P, each with matched negatives, marked positives, the same +three fixed attacks as the synthetic cohort, and TrustMark remover outputs. +Clean sources are expected negative only for the selected DWT-DCT or TrustMark +adapter; the result must not be generalized into a claim that no other +provenance signal exists. The aggregate report adds provider and content-stratum +tables when this source metadata is present. + +The 12 existing SynthID and Content Seal signal carriers are deliberately not +mixed into this matched-negative baseline. They remain suitable for a separate +cross-watermark interference layer, where their original signal and oracle +provenance can be named instead of being mislabeled as globally clean. + +### Development baseline, 2026-09-06 + +Three sequential local runs reproduced all 798 case verdicts with no unstable +case, adapter error, or unavailable result. Both adapters rejected all 38 +matched negatives. The carrier set therefore removed the synthetic cohort's +false impression that TrustMark was broadly carrier-fragile, while confirming +and sharpening the DWT-DCT carrier-dependence finding. + +| Adapter and state | Unique artifacts | Detected | Not detected | +| --- | ---: | ---: | ---: | +| DWT-DCT marked | 114 | 74 | 40 | +| DWT-DCT JPEG q90 | 114 | 76 | 38 | +| DWT-DCT resize 75% round-trip | 114 | 77 | 37 | +| DWT-DCT crop 5% per edge | 114 | 0 | 114 | +| TrustMark marked | 38 | 38 | 0 | +| TrustMark, all three attacked arms | 114 | 114 | 0 | +| TrustMark removed | 38 | 0 | 38 | + +DWT-DCT marked recall was 26/38 for FLUX, 23/38 for Stable Diffusion +1.x, and 25/38 for SDXL. JPEG and resize retained every one of the 74 base +detections, then moved two and three previously missed cases above the decoder +threshold respectively. They did not create new ground-truth marks; this is why +the report presents pairwise transitions rather than describing 76/114 and +77/114 as ordinary attack retention. The crop destroyed every recognized +DWT-DCT signal. + +The content strata expose a strong dependence that three procedural carriers +could not measure. Interior low light, person in context, and single portrait +were 0/6 on the un-attacked DWT-DCT embeds, while nine strata were 6/6. Meta +carriers were detected in 42/57 marked cases and their prompt-matched OpenAI +partners in 32/57. Those are descriptive counts over one image per provider +and stratum, not a provider-population estimate. + +TrustMark was detected before and after every fixed attack in every stratum. +All 38 remover outputs produced no recognized TrustMark signal, with 41.26 to +49.80 dB PSNR and 45.27 dB median PSNR against the standardized carriers. This +remains a decoder-scoped post-removal observation, not proof of erasure against +another implementation. + +The run was based on commit `f61a7ec` with tracked benchmark changes still +uncommitted. The exact inputs and implementations are therefore bound by hash: + +- source manifest: `sha256:aa37aa97ed3b32a59242fd506fa3cee545df615f98690d373735929a9d7d0089`; +- content-manifest validator: `sha256:72ddb2e489d5a7f92cd948b880a0fd308d4083ed86d68fc49897d646aceb7ec5`; +- real-cohort recipe: `sha256:f3851dae5d4eee69510c3dc5b5b15b5116720d944cfeed631894e635c2a5e9c9`; +- shared embed/attack helper: `sha256:c78342f0d125d6636259b641e67e71b72b577455057e568052d2a0a4df00c9dd`; +- benchmark kernel: `sha256:a96874a41fa0a38cd29d957c6a69683de887e942b480ce7668090b0f1a9e3329`; +- generated manifest: `sha256:77dc6094fcfbbaa6fdb9a15d96dee6fb3d4349829b2b1d7cf5f006aa7fb0ae8a`; +- case-level results: `sha256:bb483be015c012fd28d0979510244c834475a23bbc215226c998fada73485ed3`, + `sha256:35d735de9654a36d6d2a6e096f7a2c3f8a70b1b22ead4f6f414d6c1494e53b2a`, and + `sha256:52a9a2bb78dcc9f40609b5ca13812470d21477e5e553892c2bbef4b803fee433`; +- aggregate report: `sha256:4496edf231a20cc7977f003c30313af5acbe8c99075e22d2cb2eeec5e104f3eb`. + +The local timing rows were host-contended and are not used for a new latency +claim. The controlled resource profile below remains the performance evidence. + ## Aggregate repeated runs `scripts/watermark_benchmark_report.py` turns one or more result JSONL files @@ -189,3 +296,70 @@ expected-result mismatches, unstable repeated verdicts, errors, and unavailable adapters as separate evidence. A repeated `case_id` whose artifact, transform, or other case identity changes is rejected rather than merged. Input file digests in the final report bind every aggregate to its exact case-level source. +The runner decodes an artifact used by multiple cases only once per process and +passes the decoded pixels to both local adapters; the adapter timing therefore +does not include file decoding. + +## Profile process cost + +`scripts/watermark_benchmark_resources.py` runs each manifest in fresh Python +processes. It records parent-observed wall time and the child's absolute peak +RSS, while preserving the ordinary case-level JSONL outputs for the detector +report above. + +```bash +uv run python scripts/watermark_benchmark_resources.py \ + .local-eval/cohort-256/manifest.jsonl \ + .local-eval/cohort-512/manifest.jsonl \ + .local-eval/cohort-1024/manifest.jsonl \ + --repeat 3 \ + --output-dir .local-eval/resource-profile +``` + +Wall time includes interpreter startup, imports, image decoding, detector calls, +fidelity metrics, and result writing. Peak RSS is the process's absolute +high-water mark, not incremental memory caused only by validation. Run DWT-DCT +and TrustMark manifests separately to avoid attributing one adapter's model load +to the other. The profiler uses only local adapters and never calls an oracle or +provider API. + +### Development resource profile, 2026-09-06 + +Ten sequential fresh-process runs per size were measured on macOS arm64 with +Python 3.12.13. DWT-DCT manifests contained 46 cases; TrustMark manifests +contained 19. Every status count was stable across all ten repetitions. + +| Adapter | Max geometry | Cases | Process wall p50 | p90 | Absolute peak RSS p50 | p90 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| DWT-DCT | 256x256 | 46 | 263.07 ms | 322.61 ms | 74.66 MiB | 75.31 MiB | +| DWT-DCT | 512x512 | 46 | 482.67 ms | 498.07 ms | 90.56 MiB | 91.02 MiB | +| DWT-DCT | 1024x1024 | 46 | 1,360.10 ms | 1,389.40 ms | 159.58 MiB | 160.00 MiB | +| TrustMark | 256x256 | 19 | 3,191.18 ms | 6,074.68 ms | 741.11 MiB | 743.66 MiB | +| TrustMark | 512x512 | 19 | 9,172.26 ms | 15,430.20 ms | 752.92 MiB | 758.17 MiB | +| TrustMark | 1024x1024 | 19 | 7,301.26 ms | 9,797.40 ms | 848.91 MiB | 852.92 MiB | + +The TrustMark wall-time variance, including one 18.34-second 512 px maximum, +did not coincide with comparable memory movement. The non-monotonic 512/1024 +times prove that this host was contended, so the TrustMark wall rows are a +diagnostic range and cannot establish size scaling or throughput. Repeat them +on an idle host before making a performance claim. Peak RSS stayed much more +stable: its nearly flat 256/512 values establish the fixed model cost, while +1024 px adds about 100 MiB of high-water memory. + +These whole-process rows are deliberately not divided by the case count. A +fresh-process average would hide the fixed model load and is not the cost of one +post-removal check. The current three-run real-image cohort measured warm p50 +adapter calls of 0.57 ms for DWT-DCT and 29.11 ms for TrustMark at 512 px, +excluding file decoding. A workflow that identified the source before removal +already holds TrustMark's singleton decoder, so validation reuses that fixed +model cost. Starting a new process or loading TrustMark only after removal is +the expensive design and should be avoided. + +The exact profiler source was +`sha256:caf9674dffceddebb8cbf26067581250334fbc11e0fbcb567e8dd45555357b54`. +The local DWT-DCT and TrustMark resource JSONL files were respectively +`sha256:b7ceb012db5c2f549720c37d6234526396c7120fb68e040705322be35dfd6743` +and +`sha256:f3639f46a9257a3161ed0197016c089ca5c084b2de2e07cb4b65fc774e20544d`. +They remain untracked because they contain host-specific paths and timings; +these hashes bind the summarized numbers to the exact local evidence. diff --git a/scripts/README.md b/scripts/README.md index e30f9e0..1340550 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -27,6 +27,7 @@ explicitly names a tracked canonical result. | `visible_removal_audit.py` | Audit visible-removal results over a local corpus. | | `watermark_benchmark.py` | Run hash-pinned image cases while keeping detection, removal observation, and fidelity separate. | | `watermark_benchmark_report.py` | Aggregate repeated benchmark results with artifact-aware counts and separate cold/warm timing. | +| `watermark_benchmark_resources.py` | Measure fresh-process wall time and absolute peak RSS for benchmark manifests. | ## Calibration and corpus preparation @@ -46,6 +47,7 @@ explicitly names a tracked canonical result. | `visible_recall_sample.py` | Build an unbiased blinded sample for recall measurement. | | `visible_sheets.py` | Build blinded contact sheets for relaxation candidates. | | `watermark_benchmark_cohort.py` | Build the deterministic synthetic DWT-DCT and TrustMark cohort consumed by `watermark_benchmark.py`. | +| `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. | | `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. | | `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. | | `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. | @@ -82,3 +84,5 @@ running the sync command without `--check` is the explicit snapshot update. `_plain_console.py` provides plain-text fallbacks for Rich output, and `_text_eval.py` contains normalization helpers shared by text-evaluation scripts. +`engine_selection_manifest.py` owns the content-matrix schema and validation +shared by its integrity check and the real-image benchmark builder. diff --git a/scripts/engine_selection_manifest.py b/scripts/engine_selection_manifest.py new file mode 100644 index 0000000..d12cb80 --- /dev/null +++ b/scripts/engine_selection_manifest.py @@ -0,0 +1,171 @@ +"""Shared validation for the publication-cleared engine-selection image matrix.""" + +from __future__ import annotations + +import csv +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image +from watermark_benchmark import sha256_file + +REUSE_BASIS = "project-maintainer-public-test-clearance" +CONTENT_FIELDS = ( + "pair_id", + "provider", + "model", + "file", + "width", + "height", + "content_stratum", + "sha256", + "source_payload_sha256", + "reuse_basis", + "prompt", +) +_SAFE_ID = re.compile(r"[a-z0-9][a-z0-9-]*") + + +@dataclass(frozen=True) +class ContentFixture: + """One validated source row from the publication-cleared content matrix.""" + + pair_id: str + provider: str + model: str + path: Path + width: int + height: int + content_stratum: str + sha256: str + source_payload_sha256: str + reuse_basis: str + prompt: str + + @property + def name(self) -> str: + """Return a stable filename-safe carrier identity.""" + return f"{self.pair_id}-{self.provider}" + + @property + def seed(self) -> int: + """Derive a stable 60-bit seed from the source bytes.""" + return int(self.sha256[:15], 16) + + +def _require_text(row: dict[str, str | None], field: str, *, location: str) -> str: + value = row.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{location}: {field} must be non-empty") + return value.strip() + + +def _require_sha256(row: dict[str, str | None], field: str, *, location: str) -> str: + value = _require_text(row, field, location=location) + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{location}: {field} must be a lowercase sha256") + return value + + +def load_content_manifest(source_manifest: Path) -> list[ContentFixture]: + """Load and validate one balanced, content-stratified source manifest.""" + manifest = source_manifest.resolve() + with manifest.open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if tuple(reader.fieldnames or ()) != CONTENT_FIELDS: + raise ValueError(f"{manifest}: expected columns {', '.join(CONTENT_FIELDS)}") + rows = list(reader) + if not rows: + raise ValueError(f"{manifest}: source manifest contains no rows") + + fixtures: list[ContentFixture] = [] + paths: set[Path] = set() + hashes: set[str] = set() + pair_provider: set[tuple[str, str]] = set() + stratum_provider: set[tuple[str, str]] = set() + pairs: dict[str, list[ContentFixture]] = defaultdict(list) + + for line_number, row in enumerate(rows, start=2): + location = f"{manifest}:{line_number}" + if None in row: + raise ValueError(f"{location}: row contains extra columns") + pair_id = _require_text(row, "pair_id", location=location) + provider = _require_text(row, "provider", location=location) + if not _SAFE_ID.fullmatch(pair_id) or not _SAFE_ID.fullmatch(provider): + raise ValueError(f"{location}: pair_id and provider must be lowercase filename-safe identifiers") + source_sha256 = _require_sha256(row, "sha256", location=location) + source_payload_sha256 = _require_sha256(row, "source_payload_sha256", location=location) + reuse_basis = _require_text(row, "reuse_basis", location=location) + if reuse_basis != REUSE_BASIS: + raise ValueError(f"{location}: unsupported reuse_basis {reuse_basis!r}") + try: + width = int(_require_text(row, "width", location=location)) + height = int(_require_text(row, "height", location=location)) + except ValueError as exc: + raise ValueError(f"{location}: width and height must be positive integers") from exc + if width <= 0 or height <= 0: + raise ValueError(f"{location}: width and height must be positive integers") + + relative_path = Path(_require_text(row, "file", location=location)) + path = (manifest.parent / relative_path).resolve() + if not path.is_relative_to(manifest.parent): + raise ValueError(f"{location}: file must stay inside {manifest.parent}") + if path in paths: + raise ValueError(f"{location}: duplicate source path {relative_path}") + paths.add(path) + if source_sha256 in hashes: + raise ValueError(f"{location}: duplicate source sha256 {source_sha256}") + hashes.add(source_sha256) + identity = (pair_id, provider) + if identity in pair_provider: + raise ValueError(f"{location}: duplicate pair/provider {pair_id}/{provider}") + pair_provider.add(identity) + + content_stratum = _require_text(row, "content_stratum", location=location) + stratum_identity = (content_stratum, provider) + if stratum_identity in stratum_provider: + raise ValueError(f"{location}: duplicate stratum/provider {content_stratum}/{provider}") + stratum_provider.add(stratum_identity) + + if not path.is_file(): + raise ValueError(f"{location}: source file is missing: {relative_path}") + actual_sha256 = sha256_file(path) + if actual_sha256 != source_sha256: + raise ValueError( + f"{location}: source sha256 mismatch for {relative_path}: expected {source_sha256}, got {actual_sha256}" + ) + with Image.open(path) as image: + actual_size = image.size + actual_format = image.format + if actual_size != (width, height): + raise ValueError(f"{location}: expected size {(width, height)}, got {actual_size}") + if actual_format != "PNG": + raise ValueError(f"{location}: expected PNG source, got {actual_format}") + + fixture = ContentFixture( + pair_id=pair_id, + provider=provider, + model=_require_text(row, "model", location=location), + path=path, + width=width, + height=height, + content_stratum=content_stratum, + sha256=source_sha256, + source_payload_sha256=source_payload_sha256, + reuse_basis=reuse_basis, + prompt=_require_text(row, "prompt", location=location), + ) + fixtures.append(fixture) + pairs[pair_id].append(fixture) + + providers = {fixture.provider for fixture in fixtures} + if len(providers) < 2: + raise ValueError(f"{manifest}: a stratified cohort requires at least two providers") + for pair_id, pair in sorted(pairs.items()): + if {fixture.provider for fixture in pair} != providers: + raise ValueError(f"{manifest}: pair {pair_id} does not contain every provider") + if len({fixture.prompt for fixture in pair}) != 1 or len({fixture.content_stratum for fixture in pair}) != 1: + raise ValueError(f"{manifest}: pair {pair_id} does not preserve one prompt and content stratum") + return fixtures diff --git a/scripts/verify_engine_selection_fixtures.py b/scripts/verify_engine_selection_fixtures.py index 10f5ed6..1a6cf6a 100644 --- a/scripts/verify_engine_selection_fixtures.py +++ b/scripts/verify_engine_selection_fixtures.py @@ -4,12 +4,11 @@ from __future__ import annotations import csv -import hashlib import logging -from collections import defaultdict from pathlib import Path -from PIL import Image +from engine_selection_manifest import load_content_manifest +from watermark_benchmark import sha256_file log = logging.getLogger(__name__) @@ -20,71 +19,26 @@ _CARRIER_MANIFEST = _FIXTURE_ROOT / "carrier-manifest.csv" _EXPECTED_PROVIDERS = {"meta", "openai"} -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - def main() -> int: errors: list[str] = [] - pairs: dict[str, list[dict[str, str]]] = defaultdict(list) - paths: set[Path] = set() - hashes: set[str] = set() - - with _MANIFEST.open(newline="", encoding="utf-8") as stream: - rows = list(csv.DictReader(stream)) + try: + rows = load_content_manifest(_MANIFEST) + except (OSError, ValueError) as exc: + errors.append(str(exc)) + rows = [] with _CARRIER_MANIFEST.open(newline="", encoding="utf-8") as stream: carrier_rows = list(csv.DictReader(stream)) - for row in rows: - pair_id = row["pair_id"] - relative_path = Path(row["file"]) - path = _FIXTURE_ROOT / relative_path - pairs[pair_id].append(row) - - if relative_path in paths: - errors.append(f"duplicate path: {relative_path}") - paths.add(relative_path) - - expected_hash = row["sha256"] - if expected_hash in hashes: - errors.append(f"duplicate file hash: {expected_hash}") - hashes.add(expected_hash) - - if not path.is_file(): - errors.append(f"missing file: {relative_path}") - continue - actual_hash = _sha256(path) - if actual_hash != expected_hash: - errors.append(f"hash mismatch for {relative_path}: expected {expected_hash}, got {actual_hash}") - - with Image.open(path) as image: - actual_size = image.size - actual_format = image.format - expected_size = (int(row["width"]), int(row["height"])) - if actual_size != expected_size: - errors.append(f"size mismatch for {relative_path}: expected {expected_size}, got {actual_size}") - if actual_format != "PNG": - errors.append(f"format mismatch for {relative_path}: expected PNG, got {actual_format}") - - for pair_id, pair_rows in sorted(pairs.items()): - providers = {row["provider"] for row in pair_rows} - prompts = {row["prompt"] for row in pair_rows} - strata = {row["content_stratum"] for row in pair_rows} - if providers != _EXPECTED_PROVIDERS: - errors.append(f"pair {pair_id} providers: expected {_EXPECTED_PROVIDERS}, got {providers}") - if len(prompts) != 1: - errors.append(f"pair {pair_id} has mismatched prompts") - if len(strata) != 1: - errors.append(f"pair {pair_id} has mismatched content strata") + providers = {row.provider for row in rows} + if rows and providers != _EXPECTED_PROVIDERS: + errors.append(f"content providers: expected {_EXPECTED_PROVIDERS}, got {providers}") expected_pair_ids = {f"{index:02d}" for index in range(19)} - if set(pairs) != expected_pair_ids: - errors.append(f"pair ids: expected {sorted(expected_pair_ids)}, got {sorted(pairs)}") + pair_ids = {row.pair_id for row in rows} + if rows and pair_ids != expected_pair_ids: + errors.append(f"pair ids: expected {sorted(expected_pair_ids)}, got {sorted(pair_ids)}") + paths = {row.path.relative_to(_FIXTURE_ROOT) for row in rows} committed_files = { path.relative_to(_FIXTURE_ROOT) for path in (_FIXTURE_ROOT / "originals").glob("**/*") if path.is_file() } @@ -103,7 +57,7 @@ def main() -> int: if not path.is_file(): errors.append(f"missing carrier: {relative_path}") continue - actual_hash = _sha256(path) + actual_hash = sha256_file(path) if actual_hash != row["sha256"]: errors.append(f"carrier hash mismatch for {relative_path}: expected {row['sha256']}, got {actual_hash}") if carrier_providers != {"google", "meta", "openai"}: @@ -117,7 +71,7 @@ def main() -> int: log.info( "Verified %s content files in %s matched pairs and %s canonical carriers", len(rows), - len(pairs), + len(pair_ids), len(carrier_rows), ) return 0 diff --git a/scripts/watermark_benchmark.py b/scripts/watermark_benchmark.py index 153c90f..1a40933 100644 --- a/scripts/watermark_benchmark.py +++ b/scripts/watermark_benchmark.py @@ -35,7 +35,7 @@ import tempfile import time from collections import Counter from dataclasses import dataclass -from functools import cache, lru_cache +from functools import cache from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Protocol, cast @@ -352,10 +352,16 @@ def _detect_dwt_dct(path: Path, image: NDArray[Any]) -> str | None: return detect_invisible_watermark(path, image=to_bgr(image)) -def _detect_trustmark(path: Path, _image: NDArray[Any]) -> str | None: - from remove_ai_watermarks.trustmark_detector import detect_trustmark +def _detect_trustmark(path: Path, image: NDArray[Any]) -> str | None: + import cv2 + import numpy as np + from PIL import Image - return detect_trustmark(path) + from remove_ai_watermarks.image_io import to_bgr + from remove_ai_watermarks.trustmark_detector import detect_trustmark_image + + rgb = cv2.cvtColor(to_bgr(image), cv2.COLOR_BGR2RGB) + return detect_trustmark_image(Image.fromarray(np.ascontiguousarray(rgb), "RGB"), source=path) def default_adapters() -> dict[str, DetectorAdapter]: @@ -422,7 +428,7 @@ def _image_fidelity( import numpy as np - reference = reference_decoder(case.reference_path) + reference = artifact if case.reference_path == case.path else reference_decoder(case.reference_path) if artifact is None or reference is None: return {"status": "not_measured", "reason": "decoded_pixels_unavailable"} if artifact.shape != reference.shape: @@ -505,6 +511,7 @@ def evaluate_case( *, adapters: Mapping[str, DetectorAdapter], repository: Mapping[str, str | bool], + artifact_decoder: Callable[[Path], NDArray[Any] | None] = _decode_image, reference_decoder: Callable[[Path], NDArray[Any] | None] = _decode_image, clock_ns: Callable[[], int] = time.perf_counter_ns, ) -> dict[str, Any]: @@ -513,7 +520,7 @@ def evaluate_case( adapter = adapters[case.adapter] except KeyError as exc: raise ValueError(f"unknown adapter {case.adapter!r}") from exc - artifact = _decode_image(case.path) + artifact = artifact_decoder(case.path) adapter_elapsed_ms: float | None = None if artifact is None: outcome = DetectorOutcome(status="error", label=None, error="artifact could not be decoded") @@ -585,7 +592,16 @@ def run_benchmark(manifest: Path, output: Path) -> Counter[str]: raise ValueError(f"unknown adapters: {', '.join(unknown)}") repository = repository_state() counts: Counter[str] = Counter() - reference_decoder = lru_cache(maxsize=8)(_decode_image) + path_counts = Counter(case.path for case in cases) + path_counts.update(case.reference_path for case in cases if case.reference_path is not None) + recurring_images: dict[Path, NDArray[Any] | None] = {} + + def decode_image(path: Path) -> NDArray[Any] | None: + if path_counts[path] < 2: + return _decode_image(path) + if path not in recurring_images: + recurring_images[path] = _decode_image(path) + return recurring_images[path] def records() -> Iterable[dict[str, Any]]: for case in cases: @@ -593,7 +609,8 @@ def run_benchmark(manifest: Path, output: Path) -> Counter[str]: case, adapters=adapters, repository=repository, - reference_decoder=reference_decoder, + artifact_decoder=decode_image, + reference_decoder=decode_image, ) counts[record["detection"]["status"]] += 1 yield record diff --git a/scripts/watermark_benchmark_cohort.py b/scripts/watermark_benchmark_cohort.py index 21d73b3..6d1fdec 100755 --- a/scripts/watermark_benchmark_cohort.py +++ b/scripts/watermark_benchmark_cohort.py @@ -30,7 +30,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from PIL import Image if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Mapping, Sequence REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO / "src")) @@ -67,6 +67,27 @@ class CarrierSpec: seed: int +def validate_cohort_options( + adapters: Sequence[AdapterName], + dwt_schemes: Sequence[str], + attacks: Sequence[AttackName], + size: int, +) -> None: + """Validate options shared by synthetic and real-image cohort builders.""" + unknown_adapters = sorted(set(adapters) - set(_ADAPTERS)) + if not adapters or unknown_adapters: + detail = ", ".join(unknown_adapters) if unknown_adapters else "none selected" + raise ValueError(f"invalid adapters: {detail}") + unknown_schemes = sorted(set(dwt_schemes) - _DWT_SCHEMES) + if unknown_schemes: + raise ValueError(f"unknown DWT-DCT schemes: {', '.join(unknown_schemes)}") + unknown_attacks = sorted(set(attacks) - _ATTACKS) + if unknown_attacks: + raise ValueError(f"unknown attacks: {', '.join(unknown_attacks)}") + if size < 256 or size % 16: + raise ValueError("size must be at least 256 and divisible by 16") + + def trustmark_available() -> bool: """Return whether the optional TrustMark package can be imported.""" from remove_ai_watermarks.trustmark_detector import is_available @@ -87,7 +108,8 @@ def _package_version(name: str) -> str: return "unavailable" -def _dependencies() -> dict[str, str]: +def benchmark_dependencies() -> dict[str, str]: + """Return versions of dependencies that can affect cohort artifacts.""" return { "numpy": _package_version("numpy"), "pillow": _package_version("pillow"), @@ -149,12 +171,14 @@ def _carrier(spec: CarrierSpec, *, size: int = 512) -> Image.Image: raise ValueError(f"unknown carrier {spec.name!r}") -def _save(image: Image.Image, path: Path) -> Path: +def save_rgb_png(image: Image.Image, path: Path) -> Path: + """Write one cohort artifact as an RGB PNG and return its path.""" image.convert("RGB").save(path, format="PNG") return path -def _attack(image: Image.Image, attack: AttackName) -> Image.Image: +def attack_image(image: Image.Image, attack: AttackName) -> Image.Image: + """Apply one deterministic robustness transform to an image.""" image = image.convert("RGB") if attack == "jpeg-q90": buffer = io.BytesIO() @@ -163,7 +187,10 @@ def _attack(image: Image.Image, attack: AttackName) -> Image.Image: with Image.open(buffer) as decoded: return decoded.convert("RGB").copy() if attack == "resize-75": - reduced = image.resize((384, 384), Image.Resampling.LANCZOS) + reduced = image.resize( + (round(image.width * 0.75), round(image.height * 0.75)), + Image.Resampling.LANCZOS, + ) return reduced.resize(image.size, Image.Resampling.LANCZOS) if attack == "crop-5": margin_x = round(image.width * 0.05) @@ -173,7 +200,8 @@ def _attack(image: Image.Image, attack: AttackName) -> Image.Image: raise ValueError(f"unknown attack {attack!r}") -def _embed_dwt_dct(image: Image.Image, scheme: str) -> Image.Image: +def embed_dwt_dct(image: Image.Image, scheme: str) -> Image.Image: + """Embed one production DWT-DCT pattern into an image.""" import cv2 import numpy as np @@ -200,7 +228,8 @@ def _embed_dwt_dct(image: Image.Image, scheme: str) -> Image.Image: return Image.fromarray(cv2.cvtColor(marked_bgr, cv2.COLOR_BGR2RGB), "RGB") -def _trustmark_runtime() -> Any: +def trustmark_runtime() -> Any: + """Load the TrustMark encoder and remover used by cohort builders.""" if not trustmark_available(): raise RuntimeError("TrustMark cohort requested but the trustmark extra is unavailable") @@ -217,7 +246,7 @@ def _relative(path: Path, root: Path) -> str: return path.relative_to(root).as_posix() -def _row( +def benchmark_row( *, root: Path, case_id: str, @@ -234,6 +263,7 @@ def _row( seed: int, expected: str, ) -> dict[str, Any]: + """Build one strict benchmark-manifest row for a generated artifact.""" return { "schema_version": SCHEMA_VERSION, "case_id": case_id, @@ -257,7 +287,8 @@ def _row( } -def _attack_parameters(attack: AttackName) -> dict[str, Any]: +def attack_parameters(attack: AttackName) -> dict[str, Any]: + """Return the exact parameters represented by an attack name.""" if attack == "jpeg-q90": values: dict[str, Any] = {"quality": 90, "subsampling": 0} elif attack == "resize-75": @@ -275,16 +306,17 @@ def _append_hard_negative( adapter: AdapterName, source_revision: str, dependencies: dict[str, str], + size: int, ) -> None: spec = CarrierSpec("hard-checker", 101) - path = artifacts / "hard-checker.png" + path = artifacts / f"hard-checker--{size}px.png" if not path.exists(): - _save(_carrier(spec), path) + save_rgb_png(_carrier(spec, size=size), path) rows.append( - _row( + benchmark_row( root=root, - case_id=f"{adapter}--hard-checker--clean", - pair_id=f"{adapter}--hard-checker", + case_id=f"{adapter}--hard-checker--{size}px--clean", + pair_id=f"{adapter}--hard-checker--{size}px", adapter=adapter, arm="hard_negative", state="clean", @@ -293,14 +325,14 @@ def _append_hard_negative( source_revision=source_revision, transform_name="synthetic-carrier", transform_revision=RECIPE_VERSION, - parameters={"carrier": spec.name, "dependencies": dependencies}, + parameters={"carrier": spec.name, "size": [size, size], "dependencies": dependencies}, seed=spec.seed, expected="not_detected", ) ) -def _append_dwt_rows( +def append_dwt_rows( rows: list[dict[str, Any]], *, root: Path, @@ -310,14 +342,27 @@ def _append_dwt_rows( attacks: Sequence[AttackName], source_revision: str, dependencies: dict[str, str], + size: int, + carrier_metadata: Mapping[str, Mapping[str, Any]] | None = None, + clean_transform_name: str = "synthetic-carrier", + clean_transform_revision: str = RECIPE_VERSION, + attack_transform_revision: str = RECIPE_VERSION, ) -> None: + """Append matched DWT-DCT cases for prepared carrier images.""" for spec, clean in carriers: - clean_path = _save(clean, artifacts / f"{spec.name}--clean.png") + clean_path = save_rgb_png(clean, artifacts / f"{spec.name}--{size}px--clean.png") for scheme in schemes: - pair_id = f"dwt-dct--{scheme}--{spec.name}" - base_parameters = {"scheme": scheme, "carrier": spec.name, "dependencies": dependencies} + pair_id = f"dwt-dct--{scheme}--{spec.name}--{size}px" + base_parameters = { + "scheme": scheme, + "carrier": spec.name, + "size": [size, size], + "dependencies": dependencies, + } + if carrier_metadata is not None: + base_parameters["source"] = dict(carrier_metadata[spec.name]) rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--clean", pair_id=pair_id, @@ -327,17 +372,17 @@ def _append_dwt_rows( path=clean_path, reference=clean_path, source_revision=source_revision, - transform_name="synthetic-carrier", - transform_revision=RECIPE_VERSION, + transform_name=clean_transform_name, + transform_revision=clean_transform_revision, parameters=base_parameters, seed=spec.seed, expected="not_detected", ) ) - marked = _embed_dwt_dct(clean, scheme) - marked_path = _save(marked, artifacts / f"{spec.name}--dwt-{scheme}--marked.png") + marked = embed_dwt_dct(clean, scheme) + marked_path = save_rgb_png(marked, artifacts / f"{spec.name}--{size}px--dwt-{scheme}--marked.png") rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--marked", pair_id=pair_id, @@ -355,12 +400,12 @@ def _append_dwt_rows( ) ) for attack in attacks: - attacked_path = _save( - _attack(marked, attack), - artifacts / f"{spec.name}--dwt-{scheme}--{attack}.png", + attacked_path = save_rgb_png( + attack_image(marked, attack), + artifacts / f"{spec.name}--{size}px--dwt-{scheme}--{attack}.png", ) rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--{attack}", pair_id=pair_id, @@ -371,15 +416,15 @@ def _append_dwt_rows( reference=clean_path, source_revision=source_revision, transform_name=f"dwt-dct-embed-then-{attack}", - transform_revision=RECIPE_VERSION, - parameters=base_parameters | _attack_parameters(attack), + transform_revision=attack_transform_revision, + parameters=base_parameters | attack_parameters(attack), seed=spec.seed, expected="unresolved", ) ) -def _append_trustmark_rows( +def append_trustmark_rows( rows: list[dict[str, Any]], *, root: Path, @@ -388,21 +433,30 @@ def _append_trustmark_rows( attacks: Sequence[AttackName], source_revision: str, dependencies: dict[str, str], + size: int, + carrier_metadata: Mapping[str, Mapping[str, Any]] | None = None, + clean_transform_name: str = "synthetic-carrier", + clean_transform_revision: str = RECIPE_VERSION, + attack_transform_revision: str = RECIPE_VERSION, ) -> None: - runtime = _trustmark_runtime() + """Append matched TrustMark cases for prepared carrier images.""" + runtime = trustmark_runtime() for spec, clean in carriers: - clean_path = artifacts / f"{spec.name}--clean.png" + clean_path = artifacts / f"{spec.name}--{size}px--clean.png" if not clean_path.exists(): - _save(clean, clean_path) - pair_id = f"trustmark--p-schema1--{spec.name}" + save_rgb_png(clean, clean_path) + pair_id = f"trustmark--p-schema1--{spec.name}--{size}px" base_parameters = { "variant": "P", "schema": 1, "carrier": spec.name, + "size": [size, size], "dependencies": dependencies, } + if carrier_metadata is not None: + base_parameters["source"] = dict(carrier_metadata[spec.name]) rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--clean", pair_id=pair_id, @@ -412,8 +466,8 @@ def _append_trustmark_rows( path=clean_path, reference=clean_path, source_revision=source_revision, - transform_name="synthetic-carrier", - transform_revision=RECIPE_VERSION, + transform_name=clean_transform_name, + transform_revision=clean_transform_revision, parameters=base_parameters, seed=spec.seed, expected="not_detected", @@ -421,9 +475,9 @@ def _append_trustmark_rows( ) payload = format(spec.seed, "061b") marked = runtime.encode(clean, payload, MODE="binary") - marked_path = _save(marked, artifacts / f"{spec.name}--trustmark-p--marked.png") + marked_path = save_rgb_png(marked, artifacts / f"{spec.name}--{size}px--trustmark-p--marked.png") rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--marked", pair_id=pair_id, @@ -441,12 +495,12 @@ def _append_trustmark_rows( ) ) for attack in attacks: - attacked_path = _save( - _attack(marked, attack), - artifacts / f"{spec.name}--trustmark-p--{attack}.png", + attacked_path = save_rgb_png( + attack_image(marked, attack), + artifacts / f"{spec.name}--{size}px--trustmark-p--{attack}.png", ) rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--{attack}", pair_id=pair_id, @@ -457,18 +511,18 @@ def _append_trustmark_rows( reference=clean_path, source_revision=source_revision, transform_name=f"trustmark-embed-then-{attack}", - transform_revision=RECIPE_VERSION, - parameters=base_parameters | _attack_parameters(attack), + transform_revision=attack_transform_revision, + parameters=base_parameters | attack_parameters(attack), seed=spec.seed, expected="unresolved", ) ) - removed_path = _save( + removed_path = save_rgb_png( runtime.remove_watermark(marked), - artifacts / f"{spec.name}--trustmark-p--removed.png", + artifacts / f"{spec.name}--{size}px--trustmark-p--removed.png", ) rows.append( - _row( + benchmark_row( root=root, case_id=f"{pair_id}--removed", pair_id=pair_id, @@ -494,31 +548,23 @@ def build_cohort( carriers: Sequence[CarrierSpec] = tuple(CarrierSpec(name, seed) for name, seed in DEFAULT_CARRIERS), dwt_schemes: Sequence[str] = DEFAULT_DWT_SCHEMES, attacks: Sequence[AttackName] = DEFAULT_ATTACKS, + size: int = 512, ) -> Path: """Build and validate a new local cohort directory, returning its manifest.""" if output_dir.exists(): raise FileExistsError(f"output directory already exists: {output_dir}") - unknown_adapters = sorted(set(adapters) - set(_ADAPTERS)) - if not adapters or unknown_adapters: - detail = ", ".join(unknown_adapters) if unknown_adapters else "none selected" - raise ValueError(f"invalid adapters: {detail}") - unknown_schemes = sorted(set(dwt_schemes) - _DWT_SCHEMES) - if unknown_schemes: - raise ValueError(f"unknown DWT-DCT schemes: {', '.join(unknown_schemes)}") - unknown_attacks = sorted(set(attacks) - _ATTACKS) - if unknown_attacks: - raise ValueError(f"unknown attacks: {', '.join(unknown_attacks)}") + validate_cohort_options(adapters, dwt_schemes, attacks, size) output_dir.mkdir(parents=True) artifacts = output_dir / "artifacts" artifacts.mkdir() rows: list[dict[str, Any]] = [] source_revision = _source_revision() - dependencies = _dependencies() - carrier_images = tuple((spec, _carrier(spec)) for spec in carriers) + dependencies = benchmark_dependencies() + carrier_images = tuple((spec, _carrier(spec, size=size)) for spec in carriers) if "dwt-dct" in adapters: - _append_dwt_rows( + append_dwt_rows( rows, root=output_dir, artifacts=artifacts, @@ -527,6 +573,7 @@ def build_cohort( attacks=attacks, source_revision=source_revision, dependencies=dependencies, + size=size, ) _append_hard_negative( rows, @@ -535,9 +582,10 @@ def build_cohort( adapter="dwt-dct", source_revision=source_revision, dependencies=dependencies, + size=size, ) if "trustmark" in adapters: - _append_trustmark_rows( + append_trustmark_rows( rows, root=output_dir, artifacts=artifacts, @@ -545,6 +593,7 @@ def build_cohort( attacks=attacks, source_revision=source_revision, dependencies=dependencies, + size=size, ) _append_hard_negative( rows, @@ -553,6 +602,7 @@ def build_cohort( adapter="trustmark", source_revision=source_revision, dependencies=dependencies, + size=size, ) manifest = output_dir / "manifest.jsonl" @@ -564,6 +614,12 @@ def build_cohort( def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-dir", type=Path, required=True, help="new local cohort directory") + parser.add_argument( + "--size", + type=int, + default=512, + help="square carrier side, at least 256 and divisible by 16 (default: 512)", + ) parser.add_argument( "--adapter", action="append", @@ -574,7 +630,11 @@ def main() -> None: args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") try: - manifest = build_cohort(args.output_dir, adapters=tuple(args.adapters or _ADAPTERS)) + manifest = build_cohort( + args.output_dir, + adapters=tuple(args.adapters or _ADAPTERS), + size=args.size, + ) except (FileExistsError, ImportError, OSError, RuntimeError, ValueError) as exc: parser.error(str(exc)) log.info("Wrote validated benchmark cohort to %s", manifest) diff --git a/scripts/watermark_benchmark_real_cohort.py b/scripts/watermark_benchmark_real_cohort.py new file mode 100755 index 0000000..0ea47dd --- /dev/null +++ b/scripts/watermark_benchmark_real_cohort.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Build a publication-cleared, content-stratified real-image cohort. + +The default source is the repository's publication-cleared, prompt-matched +OpenAI/Meta image matrix. Source files are hash-checked, deduplicated by content +digest, and standardized to square RGB PNG carriers before any watermark is +embedded. Derived artifacts stay under a new local output directory. + +This is a development-only builder. It calls no provider API or provenance +oracle and does not download a corpus. + + uv run python scripts/watermark_benchmark_real_cohort.py \ + --output-dir .local-eval/watermark-benchmark-real-v1 +""" + +from __future__ import annotations + +import argparse +import hashlib +import logging +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from PIL import Image, ImageOps + +if TYPE_CHECKING: + from collections.abc import Sequence + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "scripts")) + +import engine_selection_manifest as manifest_helpers # noqa: E402 +import watermark_benchmark_cohort as cohort_helpers # noqa: E402 +from engine_selection_manifest import ( # noqa: E402 + REUSE_BASIS as SOURCE_REUSE_BASIS, +) +from engine_selection_manifest import ( # noqa: E402 + ContentFixture as RealCarrier, +) +from engine_selection_manifest import ( # noqa: E402 + load_content_manifest as load_real_carriers, +) +from watermark_benchmark import load_manifest, sha256_file, write_jsonl # noqa: E402 +from watermark_benchmark_cohort import ( # noqa: E402 + DEFAULT_ATTACKS, + DEFAULT_DWT_SCHEMES, + AdapterName, + AttackName, + CarrierSpec, + append_dwt_rows, + append_trustmark_rows, + benchmark_dependencies, + validate_cohort_options, +) + +log = logging.getLogger(__name__) + +RECIPE_VERSION = "watermark-benchmark-real-cohort-v1" +DEFAULT_SOURCE_MANIFEST = REPO / "data" / "evaluations" / "engine-selection" / "content-manifest.csv" +REUSE_BASIS = SOURCE_REUSE_BASIS +DEFAULT_ADAPTERS: tuple[AdapterName, ...] = ("dwt-dct", "trustmark") +_ADAPTERS = frozenset(DEFAULT_ADAPTERS) + + +def _standardize(carrier: RealCarrier, *, size: int) -> Image.Image: + with Image.open(carrier.path) as source: + return ImageOps.fit( + source.convert("RGB"), + (size, size), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) + + +def _source_metadata( + carrier: RealCarrier, + *, + manifest_sha256: str, + helper_sha256: str, + manifest_helper_sha256: str, + size: int, +) -> dict[str, Any]: + return { + "manifest_sha256": manifest_sha256, + "manifest_helper_sha256": manifest_helper_sha256, + "cohort_helper_sha256": helper_sha256, + "source_sha256": carrier.sha256, + "source_payload_sha256": carrier.source_payload_sha256, + "source_pair_id": carrier.pair_id, + "provider": carrier.provider, + "model": carrier.model, + "content_stratum": carrier.content_stratum, + "source_size": [carrier.width, carrier.height], + "source_format": "PNG", + "reuse_basis": carrier.reuse_basis, + "prompt_sha256": hashlib.sha256(carrier.prompt.encode()).hexdigest(), + "standardization": { + "mode": "center-crop-cover", + "size": [size, size], + "resampling": "lanczos", + "output_mode": "RGB", + "output_format": "PNG", + }, + } + + +def build_real_cohort( + output_dir: Path, + *, + source_manifest: Path = DEFAULT_SOURCE_MANIFEST, + adapters: Sequence[AdapterName] = DEFAULT_ADAPTERS, + dwt_schemes: Sequence[str] = DEFAULT_DWT_SCHEMES, + attacks: Sequence[AttackName] = DEFAULT_ATTACKS, + size: int = 512, +) -> Path: + """Build and validate a new real-image cohort directory.""" + if output_dir.exists(): + raise FileExistsError(f"output directory already exists: {output_dir}") + validate_cohort_options(adapters, dwt_schemes, attacks, size) + + carriers = load_real_carriers(source_manifest) + too_small = [carrier.name for carrier in carriers if min(carrier.width, carrier.height) < size] + if too_small: + raise ValueError(f"source images cannot be upscaled to {size}px: {', '.join(too_small)}") + + output_dir.mkdir(parents=True) + artifacts = output_dir / "artifacts" + artifacts.mkdir() + source_manifest = source_manifest.resolve() + manifest_sha256 = sha256_file(source_manifest) + recipe_revision = f"{RECIPE_VERSION}@sha256:{sha256_file(Path(__file__))}" + helper_sha256 = sha256_file(Path(cohort_helpers.__file__)) + manifest_helper_sha256 = sha256_file(Path(manifest_helpers.__file__)) + helper_revision = f"watermark-benchmark-cohort-helpers@sha256:{helper_sha256}" + manifest_helper_revision = f"engine-selection-manifest-helper@sha256:{manifest_helper_sha256}" + source_revision = ( + f"engine-selection-content@sha256:{manifest_sha256}+{recipe_revision}+{helper_revision}" + f"+{manifest_helper_revision}" + ) + dependencies = benchmark_dependencies() + prepared: list[tuple[CarrierSpec, Image.Image]] = [] + metadata: dict[str, dict[str, Any]] = {} + for carrier in carriers: + image = _standardize(carrier, size=size) + spec = CarrierSpec(carrier.name, carrier.seed) + prepared.append((spec, image)) + metadata[carrier.name] = _source_metadata( + carrier, + manifest_sha256=manifest_sha256, + helper_sha256=helper_sha256, + manifest_helper_sha256=manifest_helper_sha256, + size=size, + ) + + rows: list[dict[str, Any]] = [] + if "dwt-dct" in adapters: + append_dwt_rows( + rows, + root=output_dir, + artifacts=artifacts, + carriers=prepared, + schemes=dwt_schemes, + attacks=attacks, + source_revision=source_revision, + dependencies=dependencies, + size=size, + carrier_metadata=metadata, + clean_transform_name="real-carrier-standardize", + clean_transform_revision=recipe_revision, + attack_transform_revision=helper_revision, + ) + if "trustmark" in adapters: + append_trustmark_rows( + rows, + root=output_dir, + artifacts=artifacts, + carriers=prepared, + attacks=attacks, + source_revision=source_revision, + dependencies=dependencies, + size=size, + carrier_metadata=metadata, + clean_transform_name="real-carrier-standardize", + clean_transform_revision=recipe_revision, + attack_transform_revision=helper_revision, + ) + + manifest = output_dir / "manifest.jsonl" + write_jsonl(manifest, rows) + load_manifest(manifest) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True, help="new local cohort directory") + parser.add_argument( + "--source-manifest", + type=Path, + default=DEFAULT_SOURCE_MANIFEST, + help="publication-cleared source CSV (default: tracked engine-selection content manifest)", + ) + parser.add_argument( + "--size", + type=int, + default=512, + help="square carrier side, at least 256 and divisible by 16 (default: 512)", + ) + parser.add_argument( + "--adapter", + action="append", + choices=tuple(sorted(_ADAPTERS)), + dest="adapters", + help="adapter to include; repeat to select both (default: both)", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + try: + manifest = build_real_cohort( + args.output_dir, + source_manifest=args.source_manifest, + adapters=tuple(args.adapters or DEFAULT_ADAPTERS), + size=args.size, + ) + except (FileNotFoundError, FileExistsError, ImportError, OSError, RuntimeError, ValueError) as exc: + parser.error(str(exc)) + log.info("Wrote validated real-image benchmark cohort to %s", manifest) + + +if __name__ == "__main__": + main() diff --git a/scripts/watermark_benchmark_report.py b/scripts/watermark_benchmark_report.py index 8586e13..86363a8 100755 --- a/scripts/watermark_benchmark_report.py +++ b/scripts/watermark_benchmark_report.py @@ -179,7 +179,8 @@ def _case_identity(record: ResultRecord) -> tuple[object, ...]: ) -def _nearest_rank(ordered: Sequence[float], quantile: float) -> float: +def nearest_rank(ordered: Sequence[float], quantile: float) -> float: + """Return a nearest-rank percentile from values sorted in ascending order.""" if not ordered: raise ValueError("cannot calculate a percentile without measurements") return ordered[max(0, math.ceil(quantile * len(ordered)) - 1)] @@ -191,12 +192,34 @@ def _timing_stats(adapter: str, phase: TimingPhase, values: list[float]) -> dict "adapter": adapter, "phase": phase, "n": len(ordered), - "p50_ms": _nearest_rank(ordered, 0.50), - "p90_ms": _nearest_rank(ordered, 0.90), + "p50_ms": nearest_rank(ordered, 0.50), + "p90_ms": nearest_rank(ordered, 0.90), "max_ms": ordered[-1], } +def _artifact_status_summary(group: Sequence[ResultRecord]) -> dict[str, int]: + artifacts: dict[str, set[str]] = defaultdict(set) + for record in group: + row = record.data + artifact = cast("dict[str, Any]", row["artifact"]) + detection = cast("dict[str, Any]", row["detection"]) + artifacts[str(artifact["sha256"])].add(str(detection["status"])) + counts: dict[str, int] = dict.fromkeys(DETECTOR_STATUSES, 0) + unstable = 0 + for statuses in artifacts.values(): + if len(statuses) != 1: + unstable += 1 + continue + counts[next(iter(statuses))] += 1 + return { + "unique_artifacts": len(artifacts), + "observations": len(group), + **counts, + "unstable": unstable, + } + + def _detection_groups(records: Sequence[ResultRecord], *, include_transform: bool) -> list[dict[str, Any]]: grouped: dict[tuple[str, ...], list[ResultRecord]] = defaultdict(list) for record in records: @@ -209,37 +232,124 @@ def _detection_groups(records: Sequence[ResultRecord], *, include_transform: boo summaries: list[dict[str, Any]] = [] for identity, group in sorted(grouped.items()): - artifacts: dict[str, set[str]] = defaultdict(set) - for record in group: - row = record.data - artifact = cast("dict[str, Any]", row["artifact"]) - detection = cast("dict[str, Any]", row["detection"]) - artifacts[str(artifact["sha256"])].add(str(detection["status"])) - counts: dict[str, int] = dict.fromkeys(DETECTOR_STATUSES, 0) - unstable = 0 - for statuses in artifacts.values(): - if len(statuses) != 1: - unstable += 1 - continue - counts[next(iter(statuses))] += 1 summary: dict[str, Any] = { "adapter": identity[0], "state": identity[1], } if include_transform: summary["transform"] = identity[2] - summary.update( - { - "unique_artifacts": len(artifacts), - "observations": len(group), - **counts, - "unstable": unstable, - } - ) + summary.update(_artifact_status_summary(group)) summaries.append(summary) return summaries +def _detection_groups_by_source_field( + records: Sequence[ResultRecord], + *, + field: str, +) -> list[dict[str, Any]]: + grouped: dict[tuple[str, str, str], list[ResultRecord]] = defaultdict(list) + for record in records: + row = record.data + transform = cast("dict[str, Any]", row["transform"]) + parameters = cast("dict[str, Any]", transform["parameters"]) + source = parameters.get("source") + if not isinstance(source, dict): + continue + value = cast("dict[str, object]", source).get(field) + if not isinstance(value, str) or not value: + continue + grouped[(str(row["adapter"]), str(row["state"]), value)].append(record) + + summaries: list[dict[str, Any]] = [] + for (adapter, state, value), group in sorted(grouped.items()): + summaries.append( + { + "adapter": adapter, + "state": state, + field: value, + **_artifact_status_summary(group), + } + ) + return summaries + + +def _watermark_profile(parameters: Mapping[str, Any]) -> str | None: + scheme = parameters.get("scheme") + if isinstance(scheme, str) and scheme: + return scheme + variant = parameters.get("variant") + schema = parameters.get("schema") + if isinstance(variant, str) and variant and isinstance(schema, int) and not isinstance(schema, bool): + return f"{variant}/schema-{schema}" + return None + + +def _detection_groups_by_profile(records: Sequence[ResultRecord]) -> list[dict[str, Any]]: + grouped: dict[tuple[str, str, str], list[ResultRecord]] = defaultdict(list) + for record in records: + row = record.data + transform = cast("dict[str, Any]", row["transform"]) + parameters = cast("dict[str, Any]", transform["parameters"]) + profile = _watermark_profile(parameters) + if profile is not None: + grouped[(str(row["adapter"]), str(row["state"]), profile)].append(record) + return [ + { + "adapter": adapter, + "state": state, + "profile": profile, + **_artifact_status_summary(group), + } + for (adapter, state, profile), group in sorted(grouped.items()) + ] + + +def _attack_transition_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]: + marked: dict[tuple[str, str], set[str]] = defaultdict(set) + attacked: dict[tuple[str, str, str], set[str]] = defaultdict(set) + for record in records: + row = record.data + state = str(row["state"]) + if state not in ("marked", "attacked"): + continue + adapter = str(row["adapter"]) + pair_id = str(row["pair_id"]) + detection = cast("dict[str, Any]", row["detection"]) + status = str(detection["status"]) + if state == "marked": + marked[(adapter, pair_id)].add(status) + else: + transform = cast("dict[str, Any]", row["transform"]) + attacked[(adapter, pair_id, str(transform["name"]))].add(status) + + grouped: dict[tuple[str, str], dict[str, int]] = defaultdict( + lambda: {"retained": 0, "lost": 0, "gained": 0, "still_not_detected": 0, "unmeasured": 0} + ) + for (adapter, pair_id, transform), attacked_statuses in attacked.items(): + counts = grouped[(adapter, transform)] + marked_statuses = marked.get((adapter, pair_id), set()) + if len(marked_statuses) != 1 or len(attacked_statuses) != 1: + counts["unmeasured"] += 1 + continue + before = next(iter(marked_statuses)) + after = next(iter(attacked_statuses)) + transition = { + ("detected", "detected"): "retained", + ("detected", "not_detected"): "lost", + ("not_detected", "detected"): "gained", + ("not_detected", "not_detected"): "still_not_detected", + }.get((before, after)) + if transition is None: + counts["unmeasured"] += 1 + else: + counts[transition] += 1 + return [ + {"adapter": adapter, "transform": transform, "pairs": sum(counts.values()), **counts} + for (adapter, transform), counts in sorted(grouped.items()) + ] + + def _removal_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]: grouped: dict[str, list[ResultRecord]] = defaultdict(list) for record in records: @@ -323,7 +433,7 @@ def _fidelity_groups(records: Sequence[ResultRecord]) -> list[dict[str, Any]]: **counts, "identical": identical, "finite_psnr": len(psnr_values), - "p50_psnr_db": _nearest_rank(sorted(psnr_values), 0.50) if psnr_values else None, + "p50_psnr_db": nearest_rank(sorted(psnr_values), 0.50) if psnr_values else None, } ) return summaries @@ -392,6 +502,10 @@ def summarize(records: Sequence[ResultRecord]) -> dict[str, Any]: "inputs": inputs, "detection": _detection_groups(records, include_transform=False), "detection_by_transform": _detection_groups(records, include_transform=True), + "detection_by_profile": _detection_groups_by_profile(records), + "attack_transitions": _attack_transition_groups(records), + "detection_by_source_provider": _detection_groups_by_source_field(records, field="provider"), + "detection_by_content_stratum": _detection_groups_by_source_field(records, field="content_stratum"), "removal": _removal_groups(records), "fidelity": _fidelity_groups(records), "timing": timing, @@ -407,18 +521,27 @@ def summarize(records: Sequence[ResultRecord]) -> dict[str, Any]: } -def _cell(value: object) -> str: +def markdown_cell(value: object, *, missing: str = "-") -> str: + """Render one escaped Markdown table cell.""" if value is None: - return "-" + return missing if isinstance(value, float): return f"{value:.2f}" return str(value).replace("|", "\\|").replace("\n", " ") -def _table(columns: Sequence[tuple[str, str]], rows: Sequence[Mapping[str, Any]]) -> list[str]: +def markdown_table( + columns: Sequence[tuple[str, str]], + rows: Sequence[Mapping[str, Any]], + *, + missing: str = "-", +) -> list[str]: + """Render a Markdown table from named columns and mapping rows.""" header = "| " + " | ".join(label for _, label in columns) + " |" separator = "| " + " | ".join("---" for _ in columns) + " |" - body = ["| " + " | ".join(_cell(row.get(key)) for key, _ in columns) + " |" for row in rows] + body = [ + "| " + " | ".join(markdown_cell(row.get(key), missing=missing) for key, _ in columns) + " |" for row in rows + ] return [header, separator, *body] @@ -436,7 +559,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench "", "## Detection by state", "", - *_table( + *markdown_table( ( ("adapter", "Adapter"), ("state", "State"), @@ -453,7 +576,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench "", "## Detection by transform", "", - *_table( + *markdown_table( ( ("adapter", "Adapter"), ("state", "State"), @@ -468,13 +591,97 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench cast("list[dict[str, Any]]", summary["detection_by_transform"]), ), "", - "## Post-removal observation", - "", ] + by_profile = cast("list[dict[str, Any]]", summary["detection_by_profile"]) + if by_profile: + lines.extend( + [ + "## Detection by watermark profile", + "", + *markdown_table( + ( + ("adapter", "Adapter"), + ("state", "State"), + ("profile", "Profile"), + ("unique_artifacts", "Unique artifacts"), + ("detected", "Detected"), + ("not_detected", "Not detected"), + ("unavailable", "Unavailable"), + ("error", "Error"), + ("unstable", "Unstable"), + ), + by_profile, + ), + "", + ] + ) + attack_transitions = cast("list[dict[str, Any]]", summary["attack_transitions"]) + if attack_transitions: + lines.extend( + [ + "## Attack transitions from the matched marked case", + "", + *markdown_table( + ( + ("adapter", "Adapter"), + ("transform", "Attack transform"), + ("pairs", "Pairs"), + ("retained", "Detected -> detected"), + ("lost", "Detected -> not detected"), + ("gained", "Not detected -> detected"), + ("still_not_detected", "Not detected -> not detected"), + ("unmeasured", "Unmeasured"), + ), + attack_transitions, + ), + "", + ( + "A gained detection is a threshold-crossing observation after the transform, not evidence " + "that the attack added a watermark." + ), + "", + ] + ) + source_group_columns = ( + ("adapter", "Adapter"), + ("state", "State"), + ("unique_artifacts", "Unique artifacts"), + ("detected", "Detected"), + ("not_detected", "Not detected"), + ("unavailable", "Unavailable"), + ("error", "Error"), + ("unstable", "Unstable"), + ) + by_provider = cast("list[dict[str, Any]]", summary["detection_by_source_provider"]) + if by_provider: + lines.extend( + [ + "## Detection by source provider", + "", + *markdown_table( + (*source_group_columns[:2], ("provider", "Provider"), *source_group_columns[2:]), by_provider + ), + "", + ] + ) + by_stratum = cast("list[dict[str, Any]]", summary["detection_by_content_stratum"]) + if by_stratum: + lines.extend( + [ + "## Detection by content stratum", + "", + *markdown_table( + (*source_group_columns[:2], ("content_stratum", "Content stratum"), *source_group_columns[2:]), + by_stratum, + ), + "", + ] + ) + lines.extend(["## Post-removal observation", ""]) removal = cast("list[dict[str, Any]]", summary["removal"]) if removal: lines.extend( - _table( + markdown_table( ( ("adapter", "Adapter"), ("unique_artifacts", "Unique artifacts"), @@ -493,7 +700,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench timing = cast("list[dict[str, Any]]", summary["timing"]) if timing: lines.extend( - _table( + markdown_table( ( ("adapter", "Adapter"), ("phase", "Phase"), @@ -518,7 +725,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench "", "## Fidelity", "", - *_table( + *markdown_table( ( ("adapter", "Adapter"), ("state", "State"), @@ -549,7 +756,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench lines.extend( [ "", - *_table( + *markdown_table( (("case_id", "Unstable case"), ("adapter", "Adapter"), ("statuses", "Statuses")), [row | {"statuses": ", ".join(row["statuses"])} for row in unstable], ), @@ -559,7 +766,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench lines.extend( [ "", - *_table( + *markdown_table( (("case_id", "Expected-result mismatch"), ("adapter", "Adapter"), ("status", "Observed")), mismatches, ), @@ -569,7 +776,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench lines.extend( [ "", - *_table( + *markdown_table( ( ("case_id", "Non-result case"), ("adapter", "Adapter"), @@ -586,7 +793,7 @@ def render_markdown(summary: Mapping[str, Any], *, title: str = "Watermark bench "", "## Inputs", "", - *_table( + *markdown_table( (("path", "Result file"), ("sha256", "SHA-256"), ("observations", "Observations")), cast("list[dict[str, Any]]", summary["inputs"]), ), diff --git a/scripts/watermark_benchmark_resources.py b/scripts/watermark_benchmark_resources.py new file mode 100755 index 0000000..3f065f4 --- /dev/null +++ b/scripts/watermark_benchmark_resources.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Measure cold-process cost for watermark benchmark manifests. + +Each repetition runs ``watermark_benchmark.py`` in a fresh Python process and +records parent-observed wall time plus the child's absolute peak RSS. Detector +results remain ordinary benchmark JSONL files and can be aggregated with +``watermark_benchmark_report.py``. No provider API or provenance oracle is used. + + uv run python scripts/watermark_benchmark_resources.py \ + .local-eval/cohort-256/manifest.jsonl \ + .local-eval/cohort-512/manifest.jsonl \ + --repeat 3 --output-dir .local-eval/resource-profile +""" + +from __future__ import annotations + +import argparse +import json +import logging +import math +import os +import platform +import subprocess +import sys +import tempfile +import time +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) + +from watermark_benchmark import ( # noqa: E402 + DETECTOR_STATUSES, + load_manifest, + parse_strict_json, + run_benchmark, + sha256_file, + write_jsonl, +) +from watermark_benchmark_report import markdown_table, nearest_rank, write_report # noqa: E402 + +log = logging.getLogger(__name__) + +RESOURCE_SCHEMA_VERSION = 1 + + +def _peak_rss_mib() -> float | None: + """Return this process's absolute peak RSS in MiB on known Unix ABIs.""" + try: + import resource + except ImportError: + return None + + peak = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + if sys.platform == "darwin": + return peak / (1024 * 1024) + if sys.platform.startswith("linux"): + return peak / 1024 + return None + + +def _worker(manifest: Path, result: Path, metrics: Path) -> int: + counts = run_benchmark(manifest, result) + write_jsonl( + metrics, + [ + { + "peak_rss_mib": _peak_rss_mib(), + "status_counts": {status: counts[status] for status in DETECTOR_STATUSES}, + "worker_pid": os.getpid(), + } + ], + ) + return 0 + + +def _read_worker_metrics(path: Path) -> dict[str, Any]: + lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + if len(lines) != 1: + raise ValueError(f"{path}: worker metrics must contain exactly one row") + raw = parse_strict_json(lines[0]) + if not isinstance(raw, dict): + raise ValueError(f"{path}: worker metrics fields are invalid") + metrics = cast("dict[str, object]", raw) + if set(metrics) != {"peak_rss_mib", "status_counts", "worker_pid"}: + raise ValueError(f"{path}: worker metrics fields are invalid") + peak = metrics["peak_rss_mib"] + if peak is not None and ( + not isinstance(peak, (int, float)) or isinstance(peak, bool) or not math.isfinite(peak) or peak < 0 + ): + raise ValueError(f"{path}: peak_rss_mib must be a finite non-negative number or null") + raw_counts = metrics["status_counts"] + if not isinstance(raw_counts, dict): + raise ValueError(f"{path}: status_counts fields are invalid") + counts = cast("dict[str, object]", raw_counts) + if set(counts) != set(DETECTOR_STATUSES): + raise ValueError(f"{path}: status_counts fields are invalid") + if any(not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in counts.values()): + raise ValueError(f"{path}: status_counts values must be non-negative integers") + worker_pid = metrics["worker_pid"] + if not isinstance(worker_pid, int) or isinstance(worker_pid, bool) or worker_pid < 1: + raise ValueError(f"{path}: worker_pid must be a positive integer") + return { + "peak_rss_mib": peak, + "status_counts": {status: cast("int", counts[status]) for status in DETECTOR_STATUSES}, + "worker_pid": worker_pid, + } + + +def _manifest_geometry(manifest: Path) -> dict[str, int | float]: + from PIL import Image + + cases = load_manifest(manifest) + artifacts = {case.sha256: case.path for case in cases} + dimensions: list[tuple[int, int]] = [] + for path in artifacts.values(): + with Image.open(path) as image: + dimensions.append(image.size) + max_width, max_height = max(dimensions, key=lambda size: size[0] * size[1]) + return { + "case_count": len(cases), + "unique_artifacts": len(artifacts), + "max_width": max_width, + "max_height": max_height, + "max_megapixels": max_width * max_height / 1_000_000, + } + + +def _run_child(manifest: Path, result: Path, metrics: Path) -> tuple[float, dict[str, Any]]: + command = [ + sys.executable, + str(Path(__file__).resolve()), + str(manifest), + "--_worker", + "--_result", + str(result), + "--_metrics", + str(metrics), + ] + started_ns = time.perf_counter_ns() + completed = subprocess.run(command, capture_output=True, text=True, check=False) # noqa: S603 + elapsed_ms = (time.perf_counter_ns() - started_ns) / 1e6 + if completed.returncode != 0: + raise RuntimeError( + f"benchmark worker failed for {manifest} with exit {completed.returncode}; " + f"stdout={completed.stdout!r}; stderr={completed.stderr!r}" + ) + return elapsed_ms, _read_worker_metrics(metrics) + + +def profile_manifests(manifests: Sequence[Path], output_dir: Path, *, repeat: int) -> list[dict[str, Any]]: + """Profile each manifest in fresh child processes and write their results.""" + if not manifests: + raise ValueError("at least one manifest is required") + if repeat < 1: + raise ValueError("repeat must be at least 1") + if output_dir.exists(): + raise FileExistsError(f"output directory already exists: {output_dir}") + + resolved = [manifest.resolve() for manifest in manifests] + if len(set(resolved)) != len(resolved): + raise ValueError("duplicate manifest") + manifest_profiles = [(manifest, _manifest_geometry(manifest), sha256_file(manifest)) for manifest in resolved] + output_dir.mkdir(parents=True) + + rows: list[dict[str, Any]] = [] + profiler_source_sha256 = sha256_file(Path(__file__)) + for manifest_index, (manifest, geometry, manifest_digest) in enumerate(manifest_profiles, 1): + for run_index in range(1, repeat + 1): + result = output_dir / f"manifest-{manifest_index:02d}-run-{run_index:02d}.jsonl" + with tempfile.TemporaryDirectory(prefix="watermark-resource-") as temporary: + metrics_path = Path(temporary) / "metrics.jsonl" + wall_ms, worker_metrics = _run_child(manifest, result, metrics_path) + counts = cast("dict[str, int]", worker_metrics["status_counts"]) + if sum(counts.values()) != geometry["case_count"]: + raise ValueError(f"worker status count does not match manifest case count: {manifest}") + rows.append( + { + "schema_version": RESOURCE_SCHEMA_VERSION, + "profiler_source_sha256": profiler_source_sha256, + "manifest": {"path": str(manifest), "sha256": manifest_digest}, + "run_index": run_index, + **geometry, + "process": { + "wall_ms": wall_ms, + "peak_rss_mib": worker_metrics["peak_rss_mib"], + "platform": sys.platform, + "machine": platform.machine(), + "python": sys.version.split()[0], + "worker_pid": worker_metrics["worker_pid"], + }, + "status_counts": counts, + "result": {"path": str(result.resolve()), "sha256": sha256_file(result)}, + } + ) + log.info( + "Profiled %s run %s/%s: %.2f ms, peak RSS %s MiB", + manifest, + run_index, + repeat, + wall_ms, + worker_metrics["peak_rss_mib"], + ) + + write_jsonl(output_dir / "resources.jsonl", rows) + write_report(output_dir / "resources.md", render_resource_report(summarize_resources(rows))) + return rows + + +def _metric_stats(values: Sequence[float]) -> dict[str, float]: + ordered = sorted(values) + return { + "p50": nearest_rank(ordered, 0.50), + "p90": nearest_rank(ordered, 0.90), + "max": ordered[-1], + } + + +def summarize_resources(rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Aggregate process metrics by exact manifest digest.""" + grouped: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for row in rows: + manifest = cast("Mapping[str, Any]", row["manifest"]) + grouped[str(manifest["sha256"])].append(row) + + summaries: list[dict[str, Any]] = [] + for digest, group in grouped.items(): + first = group[0] + process_rows = [cast("Mapping[str, Any]", row["process"]) for row in group] + wall = _metric_stats([float(row["wall_ms"]) for row in process_rows]) + rss_values = [float(row["peak_rss_mib"]) for row in process_rows if row["peak_rss_mib"] is not None] + rss = _metric_stats(rss_values) if rss_values else None + status_signatures = {json.dumps(row["status_counts"], sort_keys=True) for row in group} + manifest = cast("Mapping[str, Any]", first["manifest"]) + summaries.append( + { + "manifest": str(manifest["path"]), + "cohort": Path(str(manifest["path"])).parent.name, + "sha256": digest, + "profiler_source_sha256": first["profiler_source_sha256"], + "runs": len(group), + "cases": first["case_count"], + "unique_artifacts": first["unique_artifacts"], + "max_geometry": f"{first['max_width']}x{first['max_height']}", + "max_megapixels": first["max_megapixels"], + "wall_p50_ms": wall["p50"], + "wall_p90_ms": wall["p90"], + "wall_max_ms": wall["max"], + "rss_p50_mib": rss["p50"] if rss is not None else None, + "rss_p90_mib": rss["p90"] if rss is not None else None, + "rss_max_mib": rss["max"] if rss is not None else None, + "status_counts_stable": len(status_signatures) == 1, + } + ) + return summaries + + +def render_resource_report(summaries: Sequence[Mapping[str, Any]]) -> str: + """Render an absolute process-cost report.""" + columns = ( + ("cohort", "Cohort"), + ("max_geometry", "Max geometry"), + ("max_megapixels", "Max MP"), + ("cases", "Cases"), + ("runs", "Runs"), + ("wall_p50_ms", "Wall p50 ms"), + ("wall_p90_ms", "Wall p90 ms"), + ("wall_max_ms", "Wall max ms"), + ("rss_p50_mib", "Peak RSS p50 MiB"), + ("rss_p90_mib", "Peak RSS p90 MiB"), + ("rss_max_mib", "Peak RSS max MiB"), + ("status_counts_stable", "Statuses stable"), + ) + lines = [ + "# Watermark benchmark resource profile", + "", + ( + "Each run is a fresh Python process. Wall time includes interpreter startup, imports, image decoding, " + "detector calls, fidelity metrics, and result writing. Peak RSS is the process's absolute high-water " + "mark, not incremental memory attributable only to validation." + ), + "", + *markdown_table(columns, summaries, missing="unavailable"), + ] + lines.extend( + [ + "", + "Detector verdicts remain in the sibling `manifest-XX-run-YY.jsonl` files. Aggregate them with " + "`watermark_benchmark_report.py`; this table answers only the process-cost question.", + "", + "Profiler source SHA-256: " + ", ".join(sorted({str(row["profiler_source_sha256"]) for row in summaries})), + "", + ] + ) + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifests", nargs="+", type=Path, help="one or more strict benchmark manifests") + parser.add_argument("--output-dir", type=Path, help="new directory for result JSONL and resource reports") + parser.add_argument("--repeat", type=int, default=3, help="fresh processes per manifest (default: 3)") + parser.add_argument("--_worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--_result", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--_metrics", type=Path, help=argparse.SUPPRESS) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + try: + if args._worker: + if len(args.manifests) != 1 or args._result is None or args._metrics is None: + raise ValueError("worker mode requires one manifest, --_result, and --_metrics") + return _worker(args.manifests[0], args._result, args._metrics) + if args.output_dir is None: + raise ValueError("--output-dir is required") + if args._result is not None or args._metrics is not None: + raise ValueError("internal worker outputs require --_worker") + profile_manifests(args.manifests, args.output_dir, repeat=args.repeat) + except (FileExistsError, FileNotFoundError, OSError, RuntimeError, ValueError) as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/remove_ai_watermarks/trustmark_detector.py b/src/remove_ai_watermarks/trustmark_detector.py index 375c11d..e811397 100644 --- a/src/remove_ai_watermarks/trustmark_detector.py +++ b/src/remove_ai_watermarks/trustmark_detector.py @@ -28,6 +28,8 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from pathlib import Path + from PIL import Image + logger = logging.getLogger(__name__) # Adobe ships Variant P in production (com.adobe.trustmark.P). @@ -66,9 +68,43 @@ def _decoder() -> Any: _REENCODE_QUALITY = 95 +def _detect_cover(cover: Image.Image, source: object) -> str | None: + """Decode one RGB cover after availability and image loading are resolved.""" + try: + decoder = _decoder() + wm_secret, wm_present, wm_schema = decoder.decode(cover, "binary") + if not wm_present: + return None + if wm_schema not in _SUPPORTED_SCHEMAS: + logger.debug( + "TrustMark decode for %s used weak schema %s; treating as false positive", + source, + wm_schema, + ) + return None + if not _survives_reencode(decoder, cover, wm_secret, wm_schema): + logger.debug("TrustMark decode for %s did not survive re-encode; treating as false positive", source) + return None + except Exception as exc: # model download / decode failure + logger.debug("TrustMark decode failed for %s: %s", source, exc) + return None + return f"Adobe TrustMark (variant {_MODEL_TYPE}, schema {wm_schema})" + + +def detect_trustmark_image(image: Image.Image, *, source: object = "decoded image") -> str | None: + """Detect a durable TrustMark in an already decoded Pillow image.""" + if not is_available(): + return None + try: + cover = image.convert("RGB") + except Exception as exc: + logger.debug("TrustMark image conversion failed for %s: %s", source, exc) + return None + return _detect_cover(cover, source) + + def detect_trustmark(image_path: Path) -> str | None: - """Return a TrustMark scheme note if a *durable* TrustMark watermark is - decoded, else None. + """Return a TrustMark scheme note if a *durable* TrustMark watermark is decoded, else None. Returns e.g. ``"Adobe TrustMark (variant P, schema 0)"`` when the decoder reports the watermark present AND it survives a mild JPEG re-encode, or None @@ -93,24 +129,10 @@ def detect_trustmark(image_path: Path) -> str | None: with Image.open(image_path) as img: cover = img.convert("RGB") - decoder = _decoder() - wm_secret, wm_present, wm_schema = decoder.decode(cover, "binary") - if not wm_present: - return None - if wm_schema not in _SUPPORTED_SCHEMAS: - logger.debug( - "TrustMark decode for %s used weak schema %s; treating as false positive", - image_path, - wm_schema, - ) - return None - if not _survives_reencode(decoder, cover, wm_secret, wm_schema): - logger.debug("TrustMark decode for %s did not survive re-encode; treating as false positive", image_path) - return None - except Exception as exc: # model download / decode failure / unreadable image + except Exception as exc: # unreadable image logger.debug("TrustMark decode failed for %s: %s", image_path, exc) return None - return f"Adobe TrustMark (variant {_MODEL_TYPE}, schema {wm_schema})" + return _detect_cover(cover, image_path) def _survives_reencode(decoder: Any, cover: Any, payload: str, schema: int) -> bool: diff --git a/src/remove_ai_watermarks/video.py b/src/remove_ai_watermarks/video.py index cb4e90c..8127d88 100644 --- a/src/remove_ai_watermarks/video.py +++ b/src/remove_ai_watermarks/video.py @@ -8,7 +8,7 @@ The visible pixel path reuses the image package's shared fill backends. from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Literal @@ -64,15 +64,26 @@ class VideoAudioStatus: watermark_status: Literal["unverified"] = "unverified" +class _VideoAudioBoundary: + """Derive the audio disposition from whether an output was written.""" + + @property + def audio(self) -> VideoAudioStatus: + """Return the non-claiming audio status for this result.""" + action: Literal["copied_if_present", "not_written"] = ( + "copied_if_present" if getattr(self, "output", None) is not None else "not_written" + ) + return VideoAudioStatus(stream_action=action) + + @dataclass(frozen=True) -class VideoMetadataResult: +class VideoMetadataResult(_VideoAudioBoundary): """Result of a verified metadata strip that preserves source audio.""" source: Path output: Path detected: dict[str, str] remaining: dict[str, str] - audio: VideoAudioStatus = field(default_factory=VideoAudioStatus, init=False) @dataclass(frozen=True) @@ -92,7 +103,7 @@ class VideoProvenanceReport: @dataclass(frozen=True) -class VideoVisibleResult: +class VideoVisibleResult(_VideoAudioBoundary): """Result of visible AI-watermark removal with unverified copied audio.""" source: Path @@ -102,15 +113,10 @@ class VideoVisibleResult: detected_frames: int removed_frames: int remaining_metadata: dict[str, str] - audio: VideoAudioStatus = field(default_factory=VideoAudioStatus, init=False) - - def __post_init__(self) -> None: - if self.output is None: - object.__setattr__(self, "audio", VideoAudioStatus(stream_action="not_written")) @dataclass(frozen=True) -class VideoInvisibleResult: +class VideoInvisibleResult(_VideoAudioBoundary): """Result of applying the calibrated VAE profile to video pixels only.""" source: Path @@ -118,7 +124,6 @@ class VideoInvisibleResult: noise_std: float metrics: RegenerationMetrics remaining_metadata: dict[str, str] - audio: VideoAudioStatus = field(default_factory=VideoAudioStatus, init=False) @property def visual_invisible_action(self) -> Literal["regenerated"]: @@ -151,7 +156,7 @@ class VideoInvisibleResult: @dataclass(frozen=True) -class VideoAllResult: +class VideoAllResult(_VideoAudioBoundary): """Result of visible, metadata, and optional video-pixel processing.""" source: Path @@ -163,7 +168,6 @@ class VideoAllResult: detected_metadata: dict[str, str] remaining_metadata: dict[str, str] invisible_removed: bool - audio: VideoAudioStatus = field(default_factory=VideoAudioStatus, init=False) @property def visual_invisible_action(self) -> Literal["not_run", "regenerated"]: @@ -172,7 +176,7 @@ class VideoAllResult: @dataclass(frozen=True) -class VideoBatchItem: +class VideoBatchItem(_VideoAudioBoundary): """Outcome for one source in a video batch, including the audio boundary.""" source: Path @@ -182,11 +186,6 @@ class VideoBatchItem: visible_mark: str | None invisible_removed: bool error: str | None = None - audio: VideoAudioStatus = field(default_factory=VideoAudioStatus, init=False) - - def __post_init__(self) -> None: - if self.output is None: - object.__setattr__(self, "audio", VideoAudioStatus(stream_action="not_written")) @property def visual_invisible_action(self) -> Literal["not_run", "regenerated"]: diff --git a/tests/test_script_help.py b/tests/test_script_help.py index a0989ed..d74995b 100644 --- a/tests/test_script_help.py +++ b/tests/test_script_help.py @@ -21,7 +21,9 @@ ROOT = Path(__file__).resolve().parents[1] "registered_mark_calibrate.py", "watermark_benchmark.py", "watermark_benchmark_cohort.py", + "watermark_benchmark_real_cohort.py", "watermark_benchmark_report.py", + "watermark_benchmark_resources.py", ], ) def test_script_help_exits_cleanly(script: str) -> None: diff --git a/tests/test_trustmark_detector.py b/tests/test_trustmark_detector.py index 4083f26..20bf171 100644 --- a/tests/test_trustmark_detector.py +++ b/tests/test_trustmark_detector.py @@ -12,10 +12,11 @@ import hashlib from pathlib import Path import pytest +from PIL import Image from remove_ai_watermarks import trustmark_detector from remove_ai_watermarks.identify import identify -from remove_ai_watermarks.trustmark_detector import detect_trustmark, is_available +from remove_ai_watermarks.trustmark_detector import detect_trustmark, detect_trustmark_image, is_available _OFFICIAL_FIXTURE = ( Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance" / "adobe-trustmark-p.png" @@ -75,6 +76,15 @@ class TestFalsePositiveGate: assert result == "Adobe TrustMark (variant P, schema 2)" assert decoder.calls == 2 # original + re-encode + def test_already_decoded_image_uses_the_same_gate(self, monkeypatch): + decoder = _FakeDecoder(("secret", True, 2), ("secret", True, 2)) + self._patch_decoder(monkeypatch, decoder) + + result = detect_trustmark_image(Image.new("RGB", (32, 32)), source="benchmark fixture") + + assert result == "Adobe TrustMark (variant P, schema 2)" + assert decoder.calls == 2 + def test_false_positive_collapsing_on_reencode_is_dropped(self, monkeypatch, tmp_clean_png: Path): # Present on the original, absent after re-encode -> content-noise FP. decoder = _FakeDecoder(("01", True, 2), ("", False, -1)) diff --git a/tests/test_watermark_benchmark.py b/tests/test_watermark_benchmark.py index 692066e..9fbd5c2 100644 --- a/tests/test_watermark_benchmark.py +++ b/tests/test_watermark_benchmark.py @@ -248,6 +248,31 @@ def test_identical_pair_marks_psnr_as_unbounded_not_zero(tmp_path: Path) -> None } +def test_identical_artifact_and_reference_are_decoded_once(tmp_path: Path) -> None: + artifact = _image(tmp_path / "same.png", 90) + case = MODULE.load_manifest( + _write_manifest( + tmp_path / "manifest.jsonl", + [_row(artifact, reference=artifact, state="clean", expected="not_detected")], + ) + )[0] + calls: list[Path] = [] + + def decode(path: Path) -> Any: + calls.append(path) + return np.full((32, 40, 3), 90, dtype=np.uint8) + + MODULE.evaluate_case( + case, + adapters={"fake": _FakeAdapter(status="not_detected", label=None)}, + repository={"commit": "abc123", "dirty": False}, + artifact_decoder=decode, + reference_decoder=lambda _path: pytest.fail("identical reference was decoded again"), + ) + + assert calls == [artifact.resolve()] + + def test_missing_reference_is_explicitly_unmeasured(tmp_path: Path) -> None: artifact = _image(tmp_path / "marked.png", 100) manifest = _write_manifest(tmp_path / "manifest.jsonl", [_row(artifact)]) @@ -372,6 +397,34 @@ def test_builtin_adapters_name_their_exact_source_files() -> None: assert adapters["trustmark"].source_file.name == "trustmark_detector.py" +def test_run_benchmark_caches_only_recurring_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + recurring = _image(tmp_path / "recurring.png", 90) + unique = _image(tmp_path / "unique.png", 100) + manifest = _write_manifest( + tmp_path / "manifest.jsonl", + [ + _row(recurring, case_id="case-1", state="clean", expected="not_detected"), + _row(recurring, case_id="case-2", state="clean", expected="not_detected"), + _row(unique, case_id="case-3", state="clean", expected="not_detected"), + ], + ) + real_decode = MODULE._decode_image + calls: list[Path] = [] + + def decode(path: Path) -> Any: + calls.append(path) + return real_decode(path) + + monkeypatch.setattr(MODULE, "_decode_image", decode) + monkeypatch.setattr(MODULE, "default_adapters", lambda: {"fake": _FakeAdapter("not_detected", None)}) + monkeypatch.setattr(MODULE, "repository_state", lambda: {"commit": "abc123", "dirty": False}) + + MODULE.run_benchmark(manifest, tmp_path / "results.jsonl") + + assert calls.count(recurring.resolve()) == 1 + assert calls.count(unique.resolve()) == 1 + + def test_run_benchmark_writes_official_trustmark_result(tmp_path: Path) -> None: if not MODULE.default_adapters()["trustmark"].available: pytest.skip("trustmark not installed") diff --git a/tests/test_watermark_benchmark_cohort.py b/tests/test_watermark_benchmark_cohort.py index af8c23f..d38921d 100644 --- a/tests/test_watermark_benchmark_cohort.py +++ b/tests/test_watermark_benchmark_cohort.py @@ -7,6 +7,7 @@ import sys from pathlib import Path import pytest +from PIL import Image SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "watermark_benchmark_cohort.py" SPEC = importlib.util.spec_from_file_location("watermark_benchmark_cohort", SCRIPT) @@ -51,6 +52,43 @@ def test_cohort_refuses_an_existing_output_directory(tmp_path: Path) -> None: MODULE.build_cohort(output, adapters=("dwt-dct",)) +def test_cohort_size_is_explicit_in_artifacts_and_case_identity(tmp_path: Path) -> None: + output = tmp_path / "cohort" + + manifest = MODULE.build_cohort( + output, + adapters=("dwt-dct",), + carriers=(MODULE.CarrierSpec("texture", 7),), + dwt_schemes=("sdxl",), + attacks=("resize-75",), + size=256, + ) + rows = MODULE.load_manifest(manifest) + + assert all("256px" in row.case_id for row in rows) + assert all(row.transform.parameters["size"] == [256, 256] for row in rows) + for row in rows: + with Image.open(row.path) as image: + assert image.size == (256, 256) + + +def test_resize_attack_scales_relative_to_the_carrier() -> None: + image = MODULE._carrier(MODULE.CarrierSpec("texture", 7), size=256) + + observed = MODULE.attack_image(image, "resize-75") + expected = image.resize((192, 192), Image.Resampling.LANCZOS).resize((256, 256), Image.Resampling.LANCZOS) + old_fixed_size = image.resize((384, 384), Image.Resampling.LANCZOS).resize((256, 256), Image.Resampling.LANCZOS) + + assert observed.tobytes() == expected.tobytes() + assert observed.tobytes() != old_fixed_size.tobytes() + + +@pytest.mark.parametrize("size", [255, 258]) +def test_cohort_rejects_unsupported_size(tmp_path: Path, size: int) -> None: + with pytest.raises(ValueError, match="at least 256 and divisible by 16"): + MODULE.build_cohort(tmp_path / f"cohort-{size}", adapters=("dwt-dct",), size=size) + + def test_trustmark_cohort_includes_removed_observation(tmp_path: Path) -> None: if not MODULE.trustmark_available(): pytest.skip("trustmark not installed") diff --git a/tests/test_watermark_benchmark_real_cohort.py b/tests/test_watermark_benchmark_real_cohort.py new file mode 100644 index 0000000..7247aca --- /dev/null +++ b/tests/test_watermark_benchmark_real_cohort.py @@ -0,0 +1,131 @@ +"""Contracts for the publication-cleared real-image cohort builder.""" + +from __future__ import annotations + +import csv +import hashlib +import importlib.util +import sys +from pathlib import Path + +import pytest +from PIL import Image + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "watermark_benchmark_real_cohort.py" +SPEC = importlib.util.spec_from_file_location("watermark_benchmark_real_cohort", SCRIPT) +assert SPEC is not None +assert SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + +FIELDS = ( + "pair_id", + "provider", + "model", + "file", + "width", + "height", + "content_stratum", + "sha256", + "source_payload_sha256", + "reuse_basis", + "prompt", +) + + +def _source_row(root: Path, provider: str, color: tuple[int, int, int]) -> dict[str, str]: + path = root / f"{provider}.png" + Image.new("RGB", (320, 256), color).save(path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return { + "pair_id": "00", + "provider": provider, + "model": f"{provider}-model", + "file": path.name, + "width": "320", + "height": "256", + "content_stratum": "landscape", + "sha256": digest, + "source_payload_sha256": digest, + "reuse_basis": MODULE.REUSE_BASIS, + "prompt": "a test landscape", + } + + +def _write_source_manifest(root: Path, rows: list[dict[str, str]]) -> Path: + manifest = root / "sources.csv" + with manifest.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + return manifest + + +def test_real_cohort_records_source_rights_strata_and_standardization(tmp_path: Path) -> None: + source_manifest = _write_source_manifest( + tmp_path, + [ + _source_row(tmp_path, "alpha", (20, 80, 140)), + _source_row(tmp_path, "beta", (160, 90, 30)), + ], + ) + + manifest = MODULE.build_real_cohort( + tmp_path / "cohort", + source_manifest=source_manifest, + adapters=("dwt-dct",), + dwt_schemes=("sdxl",), + attacks=(), + size=256, + ) + rows = MODULE.load_manifest(manifest) + + assert len(rows) == 4 + assert {(row.arm, row.state) for row in rows} == { + ("matched_negative", "clean"), + ("positive", "marked"), + } + assert {row.transform.parameters["source"]["provider"] for row in rows} == {"alpha", "beta"} + assert {row.transform.parameters["source"]["content_stratum"] for row in rows} == {"landscape"} + assert {row.transform.parameters["source"]["reuse_basis"] for row in rows} == {MODULE.REUSE_BASIS} + assert all(len(row.transform.parameters["source"]["cohort_helper_sha256"]) == 64 for row in rows) + assert all(len(row.transform.parameters["source"]["manifest_helper_sha256"]) == 64 for row in rows) + assert all(row.source_revision.startswith("engine-selection-content@sha256:") for row in rows) + for row in rows: + with Image.open(row.path) as image: + assert image.size == (256, 256) + + +def test_real_cohort_rejects_duplicate_pixels_before_creating_output(tmp_path: Path) -> None: + first = _source_row(tmp_path, "alpha", (20, 80, 140)) + second = _source_row(tmp_path, "beta", (20, 80, 140)) + source_manifest = _write_source_manifest(tmp_path, [first, second]) + output = tmp_path / "cohort" + + with pytest.raises(ValueError, match="duplicate source sha256"): + MODULE.build_real_cohort(output, source_manifest=source_manifest, adapters=("dwt-dct",), size=256) + + assert not output.exists() + + +def test_real_cohort_requires_explicit_public_reuse_basis(tmp_path: Path) -> None: + rows = [ + _source_row(tmp_path, "alpha", (20, 80, 140)), + _source_row(tmp_path, "beta", (160, 90, 30)), + ] + rows[1]["reuse_basis"] = "unspecified" + source_manifest = _write_source_manifest(tmp_path, rows) + + with pytest.raises(ValueError, match="unsupported reuse_basis"): + MODULE.load_real_carriers(source_manifest) + + +def test_real_cohort_requires_provider_balance(tmp_path: Path) -> None: + source_manifest = _write_source_manifest( + tmp_path, + [_source_row(tmp_path, "alpha", (20, 80, 140))], + ) + + with pytest.raises(ValueError, match="at least two providers"): + MODULE.load_real_carriers(source_manifest) diff --git a/tests/test_watermark_benchmark_report.py b/tests/test_watermark_benchmark_report.py index 2acd439..2a85e8d 100644 --- a/tests/test_watermark_benchmark_report.py +++ b/tests/test_watermark_benchmark_report.py @@ -56,6 +56,7 @@ def _row( elapsed_ms: float | None = 2.0, transform: str = "embed", expected: str = "detected", + transform_parameters: dict[str, Any] | None = None, ) -> dict[str, Any]: digest = BENCHMARK.sha256_file(artifact) case = BENCHMARK.BenchmarkCase( @@ -70,7 +71,11 @@ def _row( reference_path=artifact, reference_sha256=digest, source_revision="cohort@abc", - transform=BENCHMARK.Transform(name=transform, revision="recipe-v1", parameters={}), + transform=BENCHMARK.Transform( + name=transform, + revision="recipe-v1", + parameters=transform_parameters or {}, + ), seed=7, expected=expected, ) @@ -196,6 +201,92 @@ def test_summary_rejects_case_identity_drift_between_runs(tmp_path: Path) -> Non MODULE.summarize(MODULE.load_results([first, second])) +def test_summary_preserves_real_cohort_provider_and_content_strata(tmp_path: Path) -> None: + first_artifact = _image(tmp_path / "first.png", 10) + second_artifact = _image(tmp_path / "second.png", 20) + first = _row( + first_artifact, + "first", + transform_parameters={ + "scheme": "sdxl", + "source": {"provider": "openai", "content_stratum": "portrait"}, + }, + ) + second = _row( + second_artifact, + "second", + status="not_detected", + transform_parameters={ + "scheme": "sdxl", + "source": {"provider": "meta", "content_stratum": "portrait"}, + }, + ) + source = _write(tmp_path / "run.jsonl", [first, second]) + + summary = MODULE.summarize(MODULE.load_results([source])) + + openai = _group( + summary, + "detection_by_source_provider", + adapter="dwt-dct", + state="marked", + provider="openai", + ) + portrait = _group( + summary, + "detection_by_content_stratum", + adapter="dwt-dct", + state="marked", + content_stratum="portrait", + ) + profile = _group( + summary, + "detection_by_profile", + adapter="dwt-dct", + state="marked", + profile="sdxl", + ) + assert openai["detected"] == 1 + assert portrait["unique_artifacts"] == 2 + assert portrait["detected"] == 1 + assert portrait["not_detected"] == 1 + assert profile["unique_artifacts"] == 2 + + +def test_summary_reports_attack_transitions_from_each_marked_pair(tmp_path: Path) -> None: + marked_artifact = _image(tmp_path / "marked.png", 10) + attacked_artifact = _image(tmp_path / "attacked.png", 20) + marked = _row(marked_artifact, "pair--marked", status="not_detected") + attacked = _row( + attacked_artifact, + "pair--jpeg", + state="attacked", + status="detected", + transform="embed-then-jpeg", + expected="unresolved", + ) + source = _write(tmp_path / "run.jsonl", [marked, attacked]) + + summary = MODULE.summarize(MODULE.load_results([source])) + + transitions = _group( + summary, + "attack_transitions", + adapter="dwt-dct", + transform="embed-then-jpeg", + ) + assert transitions == { + "adapter": "dwt-dct", + "transform": "embed-then-jpeg", + "pairs": 1, + "retained": 0, + "lost": 0, + "gained": 1, + "still_not_detected": 0, + "unmeasured": 0, + } + + def test_loader_names_invalid_json_location(tmp_path: Path) -> None: path = tmp_path / "broken.jsonl" artifact = _image(tmp_path / "valid.png", 10) diff --git a/tests/test_watermark_benchmark_resources.py b/tests/test_watermark_benchmark_resources.py new file mode 100644 index 0000000..06250d2 --- /dev/null +++ b/tests/test_watermark_benchmark_resources.py @@ -0,0 +1,113 @@ +"""Contracts for the watermark benchmark process-resource profiler.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "watermark_benchmark_resources.py" +SPEC = importlib.util.spec_from_file_location("watermark_benchmark_resources", SCRIPT) +assert SPEC is not None +assert SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) +BENCHMARK = sys.modules["watermark_benchmark"] + + +def _manifest(path: Path) -> Path: + artifact = path.parent / "clean.png" + Image.fromarray(np.full((256, 256, 3), 127, dtype=np.uint8), "RGB").save(artifact) + row = { + "schema_version": 1, + "case_id": "dwt-dct--resource--256px--clean", + "pair_id": "dwt-dct--resource--256px", + "media_type": "image", + "adapter": "dwt-dct", + "arm": "hard_negative", + "state": "clean", + "path": artifact.name, + "sha256": BENCHMARK.sha256_file(artifact), + "reference_path": None, + "reference_sha256": None, + "source_revision": "resource-test@v1", + "transform": { + "name": "synthetic-carrier", + "revision": "resource-test-v1", + "parameters": {"size": [256, 256]}, + }, + "seed": 7, + "expected": "not_detected", + } + path.write_text(json.dumps(row) + "\n", encoding="utf-8") + return path + + +def test_profiler_runs_benchmark_in_a_fresh_process(tmp_path: Path) -> None: + manifest = _manifest(tmp_path / "manifest.jsonl") + output = tmp_path / "profile" + + rows = MODULE.profile_manifests([manifest], output, repeat=1) + + assert len(rows) == 1 + assert rows[0]["case_count"] == 1 + assert rows[0]["max_width"] == 256 + assert rows[0]["max_height"] == 256 + assert rows[0]["process"]["wall_ms"] > 0 + assert rows[0]["profiler_source_sha256"] == BENCHMARK.sha256_file(SCRIPT) + assert rows[0]["process"]["worker_pid"] != os.getpid() + if sys.platform in ("darwin", "linux"): + assert rows[0]["process"]["peak_rss_mib"] > 0 + assert sum(rows[0]["status_counts"].values()) == 1 + assert (output / "manifest-01-run-01.jsonl").is_file() + assert (output / "resources.jsonl").is_file() + report = (output / "resources.md").read_text(encoding="utf-8") + assert "absolute high-water mark" in report + assert "256x256" in report + + +def test_profiler_rejects_invalid_repeat_before_creating_output(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="repeat must be at least 1"): + MODULE.profile_manifests([tmp_path / "missing.jsonl"], tmp_path / "profile", repeat=0) + + assert not (tmp_path / "profile").exists() + + +def test_profiler_validates_manifests_before_creating_output(tmp_path: Path) -> None: + output = tmp_path / "profile" + + with pytest.raises(FileNotFoundError): + MODULE.profile_manifests([tmp_path / "missing.jsonl"], output, repeat=1) + + assert not output.exists() + + +def test_resource_summary_preserves_requested_manifest_order() -> None: + def row(name: str, digest: str, width: int) -> dict[str, object]: + return { + "manifest": {"path": name, "sha256": digest}, + "profiler_source_sha256": "a" * 64, + "case_count": 1, + "unique_artifacts": 1, + "max_width": width, + "max_height": width, + "max_megapixels": width * width / 1_000_000, + "process": {"wall_ms": 1.0, "peak_rss_mib": 2.0}, + "status_counts": {"detected": 1}, + } + + summaries = MODULE.summarize_resources( + [ + row("small", "f" * 64, 256), + row("large", "0" * 64, 1024), + ] + ) + + assert [summary["manifest"] for summary in summaries] == ["small", "large"]