diff --git a/docs/verification-plan.md b/docs/verification-plan.md index 1053daa..5f49f96 100644 --- a/docs/verification-plan.md +++ b/docs/verification-plan.md @@ -147,6 +147,13 @@ INSIDE the footprint (whole-frame PSNR sits near 60 dB whatever the backend does the MEDIAN (a fill that reproduces a flat background exactly scores PSNR=inf, and one inf makes a mean inf -- the first run reported "+inf" for every flat bucket). +`tests/test_visible_fill_quality.py` is the bounded CI smoke layer: every registered +image mark is constructed on two deterministic backgrounds and sizes, then checked for +stamp coverage, one-fill detector clearance, outside-mask pixel equality, and conservative +regional PSNR/SSIM floors with the cheap OpenCV backend. The full local harness keeps the +backend comparison and texture slicing out of CI, where model downloads and a large clean +corpus would make the gate expensive and nondeterministic. + #### Local fill-quality protocol Construct marked images from clean local references, compare each backend against the known original inside the affected footprint, and keep the report under `.local-eval/`. diff --git a/scripts/detector_response.py b/scripts/detector_response.py index 9645c4a..5e1aac2 100644 --- a/scripts/detector_response.py +++ b/scripts/detector_response.py @@ -78,12 +78,12 @@ from typing import Any import numpy as np -sys.path.insert(0, str(Path(__file__).parent.parent)) -sys.path.insert(0, str(Path(__file__).parent)) - -from fill_quality import SLOT_STAMPABLE, STAMPABLE, clean_sources, stamp_any, texture_of - REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) +sys.path.insert(0, str(REPO / "scripts")) + +from fill_quality import SLOT_STAMPABLE, STAMPABLE, clean_sources, stamp_any, texture_of # noqa: E402 + OUT = REPO / ".local-eval" / "detector-response.jsonl" # 1.0 = the geometry/opacity the engine's own constants assume. The sweep reaches below diff --git a/scripts/fill_quality.py b/scripts/fill_quality.py index 69cda76..bfe7632 100644 --- a/scripts/fill_quality.py +++ b/scripts/fill_quality.py @@ -6,10 +6,10 @@ THE PROBLEM THIS SOLVES state a preference order for `--backend auto` that rests on no measurement. THE CONSTRUCTION - Take a VERIFIED-CLEAN corpus image, stamp a known mark onto it using the mark's own - captured alpha map (the forward model the reverse-alpha work established: - `stamped = (1-a)*bg + a*white`), then remove it and compare against the TRUE original. - The original is the answer key that reality never provides. + Take a VERIFIED-CLEAN corpus image, stamp a known mark onto it using the mark's bundled + alpha map or repository-owned synthetic silhouette (`stamped = (1-a)*bg + a*white`), + then remove it and compare against the TRUE original. The original is the answer key + that reality never provides. TWO MEASUREMENT DECISIONS THAT MAKE OR BREAK THIS * Score INSIDE the footprint only. The fill touches a tiny corner, so whole-frame PSNR @@ -44,17 +44,28 @@ from typing import Any import cv2 import numpy as np -sys.path.insert(0, str(Path(__file__).parent.parent)) -sys.path.insert(0, str(Path(__file__).parent)) - -from invisible_quality_audit import _ssim # reuse, do not reimplement a third SSIM - REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "src")) +sys.path.insert(0, str(REPO / "scripts")) + +from invisible_quality_audit import _ssim # noqa: E402 -- path selects this worktree; reuse shared SSIM + CORPUS = REPO / ".local-eval" / "originals" OUT = REPO / ".local-eval" / "fill-quality.jsonl" # Text marks: a bundled alpha PNG plus the engine's own corner geometry. -STAMPABLE = ("doubao", "jimeng", "kling", "samsung") +STAMPABLE = ( + "doubao", + "jimeng", + "qwen", + "kling", + "yuanbao", + "samsung", + "runninghub", + "baidu", + "liblib", + "microsoft", +) # Marks with no bundled alpha asset. Both expose a default footprint via # `footprint_mask(force=True)`, so they are stamped by fitting their own alpha source # into that slot: Gemini's alpha is derived from its background captures, the pill's diff --git a/scripts/render_visible_examples.py b/scripts/render_visible_examples.py index d4fa65a..7a97c3a 100644 --- a/scripts/render_visible_examples.py +++ b/scripts/render_visible_examples.py @@ -16,11 +16,14 @@ from __future__ import annotations import sys from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING import cv2 import numpy as np +if TYPE_CHECKING: + from collections.abc import Callable + _ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(_ROOT / "src")) @@ -39,12 +42,17 @@ _STRENGTH: dict[str, int] = { } _DEFAULT_STRENGTH = 238 -# Base geometry: one canonical size per mark where the engine's size modes -# matter (qwen's big mode), otherwise a plain 3:2 landscape frame. +# Base geometry: one canonical size per mark where the engine's size modes or +# calibrated orientation matter, otherwise a plain 3:2 landscape frame. # Samsung keeps the larger base: its overlay is faint (peak alpha ~0.38) and # the real marks live on ~2958px phone photos -- at 1536 the example falls to 0.39, # just under the engine's 0.40 gate. -_SIZE: dict[str, tuple[int, int]] = {"qwen": (1536, 1536), "liblib": (1152, 1536), "samsung": (2048, 1536)} +_SIZE: dict[str, tuple[int, int]] = { + "qwen": (1536, 1536), + "liblib": (1152, 1536), + "samsung": (2048, 1536), + "jimeng_pill": (1152, 1536), +} def base_photo(w: int, h: int, seed: int = 7) -> np.ndarray: @@ -80,11 +88,10 @@ def _composite_light(base: np.ndarray, alpha: np.ndarray, x: int, y: int, streng return out -def _text_mark_example(key: str) -> np.ndarray: +def _text_mark_example(key: str, base: np.ndarray) -> np.ndarray: engine = wr._engine(key) # the generator drives the engine's own config cfg = engine.config - w, h = _SIZE.get(key, (1536, 1152)) - base = base_photo(w, h) + h, w = base.shape[:2] if key == "microsoft": # Opaque white pill with dark text/sparkle holes, at the measured inset. at = _glyph_asset("microsoft_alpha.png") @@ -94,11 +101,10 @@ def _text_mark_example(key: str) -> np.ndarray: pad = int(0.010 * long_side) pill = cv2.resize(at, (pw, ph)) x, y = w - pad - pw, pad - roi = base[y : y + ph, x : x + pw].astype(np.float32) + out = base.copy() bright = (pill > 0.6)[:, :, None] - roi = np.where(bright, 245.0, 46.0) - base[y : y + ph, x : x + pw] = roi.astype(np.uint8) - return base + out[y : y + ph, x : x + pw] = np.where(bright, 245.0, 46.0).astype(np.uint8) + return out base_dim = {"short": min(w, h), "width": w, "long": max(w, h)}[cfg.scale_basis] # Size the glyph ON a ladder rung: the continuous front ends sweep only the # configured rungs, and a glyph sized between rungs collapses the NCC (the @@ -125,11 +131,10 @@ def _text_mark_example(key: str) -> np.ndarray: return _composite_light(base, alpha, x, y, _STRENGTH.get(key, _DEFAULT_STRENGTH)) -def _gemini_example() -> np.ndarray: +def _gemini_example(base: np.ndarray) -> np.ndarray: from remove_ai_watermarks.gemini_engine import GeminiEngine, get_watermark_config, get_watermark_size - w, h = 1536, 1152 - base = base_photo(w, h) + h, w = base.shape[:2] eng = GeminiEngine() size = get_watermark_size(w, h) alpha = eng.get_alpha_map(size) @@ -138,9 +143,8 @@ def _gemini_example() -> np.ndarray: return _composite_light(base, alpha.astype(np.float32), x, y, 255) -def _pill_example() -> np.ndarray: - w, h = 1152, 1536 # the measured pill cohort is 3:4 portrait - base = base_photo(w, h) +def _pill_example(base: np.ndarray) -> np.ndarray: + h, w = base.shape[:2] at = _glyph_asset("jimeng_pill.png") pw = max(24, int(0.161 * w)) ph = max(8, int(pw * at.shape[0] / at.shape[1])) @@ -149,13 +153,33 @@ def _pill_example() -> np.ndarray: return _composite_light(base, alpha, x, y, 232) -_BUILDERS: dict[str, Any] = {"gemini": _gemini_example, "jimeng_pill": _pill_example} +_BUILDERS: dict[str, Callable[[np.ndarray], np.ndarray]] = { + "gemini": _gemini_example, + "jimeng_pill": _pill_example, +} -def build(key: str) -> np.ndarray: - if key in _BUILDERS: - return _BUILDERS[key]() - return _text_mark_example(key) +def build( + key: str, + *, + size: tuple[int, int] | None = None, + seed: int = 7, +) -> np.ndarray: + return build_pair(key, size=size, seed=seed)[1] + + +def build_pair( + key: str, + *, + size: tuple[int, int] | None = None, + seed: int = 7, +) -> tuple[np.ndarray, np.ndarray]: + """Return the clean synthetic source and the matching marked image.""" + size = size or _SIZE.get(key, (1536, 1152)) + clean = base_photo(*size, seed=seed) + builder = _BUILDERS.get(key) + marked = builder(clean) if builder is not None else _text_mark_example(key, clean) + return clean, marked # ── Video mark examples ────────────────────────────────────────────────────── diff --git a/tests/test_visible_fill_quality.py b/tests/test_visible_fill_quality.py new file mode 100644 index 0000000..0fc5143 --- /dev/null +++ b/tests/test_visible_fill_quality.py @@ -0,0 +1,86 @@ +"""Cheap constructed-reference regressions for every visible image mark.""" + +from __future__ import annotations + +import cv2 +import numpy as np +import pytest + +from remove_ai_watermarks import watermark_registry as wr +from scripts.fill_quality import SLOT_STAMPABLE, STAMPABLE, psnr, stamp_any +from scripts.invisible_quality_audit import _ssim +from scripts.render_visible_examples import build_pair + +_IMAGE_KEYS = tuple(mark.key for mark in wr.known_marks()) + +# The second case changes both the generated background and the geometry. Samsung's +# faint overlay needs the larger width its calibrated detector covers. +_ALTERNATE_SIZE = { + "gemini": (1280, 960), + "doubao": (1280, 960), + "jimeng": (1280, 960), + "qwen": (1280, 1280), + "kling": (1280, 960), + "yuanbao": (1280, 960), + "samsung": (2304, 1728), + "runninghub": (1280, 960), + "baidu": (1280, 960), + "liblib": (960, 1280), + "microsoft": (1280, 960), + "jimeng_pill": (960, 1280), +} + +_MIN_MASK_COVERAGE = 0.94 +_MIN_FILLED_PSNR = 25.0 +_MIN_FILLED_SSIM = 0.90 + + +def _score_box(clean: np.ndarray, filled: np.ndarray, changed: np.ndarray) -> tuple[float, float]: + ys, xs = np.nonzero(changed) + assert ys.size > 0 + pad = 8 + y0, y1 = max(0, int(ys.min()) - pad), min(clean.shape[0], int(ys.max()) + 1 + pad) + x0, x1 = max(0, int(xs.min()) - pad), min(clean.shape[1], int(xs.max()) + 1 + pad) + truth = clean[y0:y1, x0:x1] + output = filled[y0:y1, x0:x1] + return psnr(output, truth), _ssim( + cv2.cvtColor(output, cv2.COLOR_BGR2GRAY), + cv2.cvtColor(truth, cv2.COLOR_BGR2GRAY), + ) + + +def test_full_quality_harness_covers_every_registered_image_mark() -> None: + assert {*STAMPABLE, *SLOT_STAMPABLE} == set(_IMAGE_KEYS) + for key in _IMAGE_KEYS: + clean, _marked = build_pair(key) + assert stamp_any(clean, key) is not None, key + + +@pytest.mark.parametrize("key", _IMAGE_KEYS) +@pytest.mark.parametrize("case", ["canonical", "alternate"]) +def test_cv2_mask_quality_smoke(key: str, case: str) -> None: + """One fill covers the stamp, clears detection, and preserves local quality.""" + size = None if case == "canonical" else _ALTERNATE_SIZE[key] + seed = 7 if case == "canonical" else 23 + clean, marked = build_pair(key, size=size, seed=seed) + changed = np.any(marked != clean, axis=2) + assert np.any(changed), key + + mark = wr.get_mark(key) + before = mark.detect(marked, provenance=False) + assert before.detected, f"{key}/{case}: confidence {before.confidence:.3f}" + localization = mark.localize(marked, force=False, detection=before) + assert localization.mask is not None, f"{key}/{case}: no mask" + + coverage = float(np.count_nonzero(localization.mask[changed])) / float(np.count_nonzero(changed)) + assert coverage >= _MIN_MASK_COVERAGE, f"{key}/{case}: mask coverage {coverage:.3f}" + + filled, region = mark.remove(marked, backend="cv2", detection=before) + assert region is not None, key + assert np.array_equal(filled[localization.mask == 0], marked[localization.mask == 0]), key + + after = mark.detect(filled, provenance=False) + assert not after.detected, f"{key}/{case}: residual confidence {after.confidence:.3f}" + filled_psnr, filled_ssim = _score_box(clean, filled, changed) + assert filled_psnr >= _MIN_FILLED_PSNR, f"{key}/{case}: PSNR {filled_psnr:.2f} dB" + assert filled_ssim >= _MIN_FILLED_SSIM, f"{key}/{case}: SSIM {filled_ssim:.4f}"