From 9c67a7aaccd987ba08fd746fd4b0b99ce47a8cc7 Mon Sep 17 00:00:00 2001 From: Kenneth Estanislao Date: Mon, 18 May 2026 01:36:24 +0800 Subject: [PATCH] fixed poisson blend --- modules/processors/frame/face_swapper.py | 184 +++++++++++++++++++---- 1 file changed, 156 insertions(+), 28 deletions(-) diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index 6f43896..b664fde 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional +from typing import Any, List, Optional, Tuple import cv2 import insightface import logging @@ -29,6 +29,149 @@ NAME = "DLC.FACE-SWAPPER" PREVIOUS_FRAME_RESULT = None # Stores the final processed frame from the previous step # --- END: Added for Interpolation --- +# --- Poisson blend (ported from deep-live-cam-gumroad-edition) --- +# Root-cause fix for the "wobble": the blend mask is NOT built from the +# independently-detected 106-pt landmarks (they jitter sub-pixel every frame +# and seamlessClone is hyper-sensitive to its mask boundary). Instead it is +# derived from the swap's OWN affine transform (M) + the swapped pixels +# (bgr_fake), so the mask is locked exactly to where the swapped face was +# placed — no independent jitter source, no EMA, no lag. The mask is cached +# when the face is nearly still so an identical array is reused (zero wobble). +_ELLIPTICAL_MASK_CACHE: dict = {} +_poisson_cached_mask: Optional[np.ndarray] = None +_poisson_cached_key: Optional[tuple] = None + + +def _create_elliptical_mask(size: Tuple[int, int]) -> np.ndarray: + """Fixed, heavily-blurred elliptical mask in aligned-face space. + + Geometry-based (not content-adaptive) and cached by size — identical + every frame for the same model input size, so it contributes no jitter. + """ + global _ELLIPTICAL_MASK_CACHE + if size in _ELLIPTICAL_MASK_CACHE: + return _ELLIPTICAL_MASK_CACHE[size] + h, w = size + center = (w // 2, h // 2) + axes = (int(w * 0.44), int(h * 0.44)) + mask = np.zeros((h, w), dtype=np.float32) + cv2.ellipse(mask, center, axes, 0, 0, 360, 1, -1) + if h * w < 65536: + mask = cv2.GaussianBlur(mask, (31, 31), 12) + else: + mask = gpu_gaussian_blur(mask, (31, 31), 12) + _ELLIPTICAL_MASK_CACHE[size] = mask + return mask + + +def _apply_poisson_blend(swapped_frame: Frame, original_frame: Frame, + target_face: Face, affine_matrix: np.ndarray = None, + bgr_fake: np.ndarray = None) -> Frame: + """Poisson-blend the swapped face onto the original frame. + + Preferred path derives the blend mask from the swap's inverse affine so + it tracks the swapped face exactly per-frame (no landmark jitter, no + smoothing). Falls back to a cached bbox-ellipse if the affine is absent. + Writes only the blended ellipse back so other faces are preserved. + """ + global _poisson_cached_mask, _poisson_cached_key + try: + # ---- Preferred: blend ONLY the genuinely-swapped region ---- + # Use the exact paste-back mask (warped elliptical mask), eroded so + # the Poisson seam sits on solidly-swapped pixels only. + if affine_matrix is not None and bgr_fake is not None: + try: + h, w = swapped_frame.shape[:2] + fh, fw = bgr_fake.shape[:2] + inv = cv2.invertAffineTransform(affine_matrix) + corners = np.array([[0, 0, 1], [fw, 0, 1], [fw, fh, 1], [0, fh, 1]], + dtype=np.float32) + t = corners @ inv.T + px1 = max(0, int(np.floor(t[:, 0].min()))) + py1 = max(0, int(np.floor(t[:, 1].min()))) + px2 = min(w, int(np.ceil(t[:, 0].max()))) + py2 = min(h, int(np.ceil(t[:, 1].max()))) + rw, rh = px2 - px1, py2 - py1 + if rw > 8 and rh > 8: + roi_aff = inv.copy() + roi_aff[0, 2] -= px1 + roi_aff[1, 2] -= py1 + fm = _create_elliptical_mask((fh, fw)) + mroi = cv2.warpAffine(fm, roi_aff, (rw, rh), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, borderValue=0) + bin_roi = np.where(mroi > 0.5, np.uint8(255), np.uint8(0)) + k = max(3, (min(rw, rh) // 20) | 1) + bin_roi = cv2.erode(bin_roi, + cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))) + bx, by, bw, bh = cv2.boundingRect(bin_roi) + if bw > 0 and bh > 0: + mx1, my1 = px1 + bx, py1 + by + mx2, my2 = mx1 + bw - 1, my1 + bh - 1 + # seamlessClone needs the cloned region off the border + if mx1 > 0 and my1 > 0 and mx2 < w - 1 and my2 < h - 1: + mask = np.zeros((h, w), dtype=np.uint8) + mask[py1:py2, px1:px2] = bin_roi + center = (mx1 + bw // 2, my1 + bh // 2) + blended = cv2.seamlessClone(swapped_frame, original_frame, + mask, center, cv2.NORMAL_CLONE) + np.copyto(swapped_frame[my1:my2 + 1, mx1:mx2 + 1], + blended[my1:my2 + 1, mx1:mx2 + 1], + where=mask[my1:my2 + 1, mx1:mx2 + 1, None].astype(bool)) + return swapped_frame + except Exception: + pass # fall through to the robust bbox-ellipse path below + # ---- Fallback: bbox-ellipse (defensive, cached when still) ---- + if not hasattr(target_face, 'bbox') or target_face.bbox is None: + return swapped_frame + x1, y1, x2, y2 = target_face.bbox.astype(int) + h, w = swapped_frame.shape[:2] + x1, y1 = (max(0, x1), max(0, y1)) + x2, y2 = (min(w, x2), min(h, y2)) + if x2 <= x1 or y2 <= y1 or x2 - x1 <= 10 or (y2 - y1 <= 10): + return swapped_frame + padding = int(min(x2 - x1, y2 - y1) * 0.1) + x1_p = max(0, x1 - padding) + y1_p = max(0, y1 - padding) + x2_p = min(w, x2 + padding) + y2_p = min(h, y2 + padding) + center_x = int(round((x1 + x2) / 2.0)) + center_y = int(round((y1 + y2) / 2.0)) + radius_x = max(1, int(round((x2_p - x1_p) / 2.0))) + radius_y = max(1, int(round((y2_p - y1_p) / 2.0))) + if not (0 <= center_x < w and 0 <= center_y < h): + return swapped_frame + center = (center_x, center_y) + if center_x - radius_x < 0 or center_x + radius_x >= w or center_y - radius_y < 0 or (center_y + radius_y >= h): + return swapped_frame + # Reuse cached mask when center/radius unchanged frame-to-frame + # (face nearly still) — saves the np.zeros + cv2.ellipse, and the + # identical array means literally zero wobble while still. + mask_key = (center_x, center_y, radius_x, radius_y, h, w) + if _poisson_cached_key == mask_key and _poisson_cached_mask is not None: + mask = _poisson_cached_mask + else: + mask = np.zeros((h, w), dtype=np.uint8) + cv2.ellipse(mask, center, (radius_x, radius_y), 0, 0, 360, 255, -1) + if np.sum(mask) == 0: + return swapped_frame + _poisson_cached_mask = mask + _poisson_cached_key = mask_key + blended = cv2.seamlessClone(swapped_frame, original_frame, mask, center, cv2.NORMAL_CLONE) + # Composite ONLY this face's ellipse back (ROI-bounded) so previously + # blended faces in multi-face mode are preserved. + rx0 = max(0, center_x - radius_x) + rx1 = min(w, center_x + radius_x + 1) + ry0 = max(0, center_y - radius_y) + ry1 = min(h, center_y + radius_y + 1) + roi_mask = mask[ry0:ry1, rx0:rx1] + np.copyto(swapped_frame[ry0:ry1, rx0:rx1], + blended[ry0:ry1, rx0:rx1], + where=roi_mask[:, :, None].astype(bool)) + return swapped_frame + except Exception: + return swapped_frame + # --- START: Mac M1-M5 Optimizations --- IS_APPLE_SILICON = platform.system() == 'Darwin' and platform.machine() == 'arm64' FRAME_CACHE = deque(maxlen=3) # Cache for frame reuse @@ -368,7 +511,12 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame: opacity = getattr(modules.globals, "opacity", 1.0) opacity = max(0.0, min(1.0, opacity)) mouth_mask_enabled = getattr(modules.globals, "mouth_mask", False) - needs_original = opacity < 1.0 or mouth_mask_enabled + poisson_blend_enabled = getattr(modules.globals, "poisson_blend", False) + # Poisson blend's seamlessClone needs the genuine pre-swap frame as its + # destination. Without this, original_frame aliases temp_frame, which + # _fast_paste_back mutates in place — so seamlessClone would blend the + # swapped face onto the already-swapped frame (no visible effect). + needs_original = opacity < 1.0 or mouth_mask_enabled or poisson_blend_enabled if needs_original: original_frame = temp_frame.copy() else: @@ -436,34 +584,14 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame: ) # --- Poisson Blending --- + # Mask derived from the swap's own affine (M) + swapped pixels (bgr_fake), + # so it tracks the swapped face exactly per-frame — no landmark jitter, + # no EMA, no lag. See _apply_poisson_blend. if getattr(modules.globals, "poisson_blend", False): - face_mask = create_face_mask(target_face, temp_frame) - if face_mask is not None: - # Find bounding box of the mask - y_indices, x_indices = np.where(face_mask > 0) - if len(x_indices) > 0 and len(y_indices) > 0: - x_min, x_max = np.min(x_indices), np.max(x_indices) - y_min, y_max = np.min(y_indices), np.max(y_indices) + swapped_frame = _apply_poisson_blend( + swapped_frame, original_frame, target_face, M, bgr_fake + ) - # Calculate center - center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2)) - - # Crop src and mask - src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1] - mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1] - - try: - # Use original_frame as destination to blend the swapped face onto it - swapped_frame = cv2.seamlessClone( - src_crop, - original_frame, - mask_crop, - center, - cv2.NORMAL_CLONE, - ) - except Exception as e: - print(f"Poisson blending failed: {e}") - # Apply opacity blend between the original frame and the swapped frame if opacity >= 1.0: return swapped_frame.astype(np.uint8)