facefusion/facefusion/vision.py

import math
from functools import lru_cache
from typing import Dict, List, Optional, Tuple

import cv2
import numpy
from cv2.typing import Size

from facefusion.common_helper import is_windows
from facefusion.ffprobe import extract_video_metadata
from facefusion.filesystem import get_file_extension, is_image, is_video
from facefusion.thread_helper import thread_lock, thread_semaphore
from facefusion.types import ColorMode, Duration, Fps, Mask, Orientation, Resolution, Scale, VisionFrame
from facefusion.video_manager import conditional_set_video_reader_position, create_video_sampler_process, get_video_reader, read_video_reader_frame, read_video_reader_window


def read_static_images(image_paths : List[str], color_mode : ColorMode = 'rgb') -> List[VisionFrame]:
	vision_frames = []

	if image_paths:
		for image_path in image_paths:
			vision_frames.append(read_static_image(image_path, color_mode))
	return vision_frames


@lru_cache(maxsize = 64)
def read_static_image(image_path : str, color_mode : ColorMode = 'rgb') -> Optional[VisionFrame]:
	return read_image(image_path, color_mode)


def read_image(image_path : str, color_mode : ColorMode = 'rgb') -> Optional[VisionFrame]:
	if is_image(image_path):
		flag = cv2.IMREAD_COLOR

		if color_mode == 'rgba':
			flag = cv2.IMREAD_UNCHANGED

		if is_windows():
			image_buffer = numpy.fromfile(image_path, dtype = numpy.uint8)
			return cv2.imdecode(image_buffer, flag)
		return cv2.imread(image_path, flag)
	return None


def write_image(image_path : str, vision_frame : VisionFrame) -> bool:
	if image_path:
		if is_windows():
			image_file_extension = get_file_extension(image_path)
			_, vision_frame = cv2.imencode(image_file_extension, vision_frame)
			vision_frame.tofile(image_path)
			return is_image(image_path)
		return cv2.imwrite(image_path, vision_frame)
	return False


def detect_image_resolution(image_path : str) -> Optional[Resolution]:
	if is_image(image_path):
		image = read_image(image_path)
		height, width = image.shape[:2]

		if width > 0 and height > 0:
			return width, height
	return None


def restrict_image_resolution(image_path : str, resolution : Resolution) -> Resolution:
	if is_image(image_path):
		image_resolution = detect_image_resolution(image_path)
		if image_resolution < resolution:
			return image_resolution
	return resolution


@lru_cache(maxsize = 64)
def read_static_video_frame(video_path : str, frame_number : int = 0) -> Optional[VisionFrame]:
	return read_video_frame(video_path, frame_number)


#todo: needs review - [decoding] [critical: medium] cv2 capture replaced by pooled ffmpeg reader, position clamped to the frame_total estimate
def read_video_frame(video_path : str, frame_number : int = 0) -> Optional[VisionFrame]:
	if is_video(video_path):
		video_reader = get_video_reader(video_path)

		if video_reader:
			video_frame_position = min(video_reader.get('frame_total'), frame_number)

			with thread_semaphore():
				conditional_set_video_reader_position(video_reader, video_frame_position)
				has_vision_frame, vision_frame = read_video_reader_frame(video_reader)

			if has_vision_frame:
				return vision_frame

	return None


#todo: needs review - [memory] [critical: high] all sampled frames are held in ram, long high resolution videos can exhaust memory
def sample_video_frames(video_path : str, frame_start : int, frame_end : int, frame_stride : int) -> List[VisionFrame]:
	vision_frames = []

	if is_video(video_path):
		width, height = extract_video_metadata(video_path).get('resolution')
		frame_size = width * height * 3
		sample_total = len([ frame_number for frame_number in range(frame_start, frame_end) if frame_number % frame_stride == 0 ])
		video_sampler = create_video_sampler_process(video_path, frame_start, frame_end, frame_stride)

		for _ in range(sample_total):
			frame_buffer = video_sampler.stdout.read(frame_size)

			if len(frame_buffer) == frame_size:
				vision_frames.append(numpy.frombuffer(frame_buffer, dtype = numpy.uint8).reshape(height, width, 3))

		video_sampler.stdout.close()
		video_sampler.wait()

	return vision_frames


@lru_cache(maxsize = 2)
def read_static_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> Dict[int, VisionFrame]:
	return read_video_chunk(video_path, chunk_number, chunk_size)


#todo: needs review - [decoding] [critical: medium] chunk end clamped by the ffprobe frame_total estimate instead of the exact cv2 count
def read_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> Dict[int, VisionFrame]:
	video_frame_chunk = {}

	if is_video(video_path) and chunk_number > -1:
		video_reader = get_video_reader(video_path)

		if video_reader:
			video_frame_position = chunk_number * chunk_size
			video_frame_end = video_frame_position + chunk_size

			if video_reader.get('frame_total') > 0:
				video_frame_end = min(video_frame_end, video_reader.get('frame_total'))

			with thread_semaphore():
				conditional_set_video_reader_position(video_reader, video_frame_position)

				for frame_number in range(video_frame_position, video_frame_end):
					has_vision_frame, vision_frame = read_video_reader_frame(video_reader)

					if has_vision_frame:
						video_frame_chunk[frame_number] = vision_frame

	return video_frame_chunk


#todo: needs review - [decoding] [critical: high] window read replaces the chunk cache, out of range frames fall back to empty vision frames
def select_video_frames(video_path : str, frame_number : int = 0, frame_offset : int = 2) -> List[VisionFrame]:
	vision_frames = []
	frame_start = frame_number - frame_offset
	frame_end = frame_number + frame_offset

	if is_video(video_path):
		with thread_lock():
			video_reader = get_video_reader(video_path)
			frame_buffer = read_video_reader_window(video_reader, frame_start, frame_end)

			for current_number in range(frame_start, frame_end + 1):
				vision_frame = create_empty_vision_frame()

				if current_number in frame_buffer:
					vision_frame = frame_buffer.get(current_number)

				vision_frames.append(vision_frame)

	return vision_frames


#todo: needs review - [probing] [critical: medium] frame total is duration times fps estimate, diverges from the real count on vfr sources
def count_video_frame_total(video_path : str) -> int:
	if is_video(video_path):
		return extract_video_metadata(video_path).get('frame_total')

	return 0


def predict_video_frame_total(video_path : str, fps : Fps, trim_frame_start : int, trim_frame_end : int) -> int:
	if is_video(video_path):
		video_fps = detect_video_fps(video_path)
		extract_frame_total = count_trim_frame_total(video_path, trim_frame_start, trim_frame_end) * fps / video_fps
		return math.floor(extract_frame_total)
	return 0


#todo: needs review - [probing] [critical: low] fps now from cached ffprobe metadata
def detect_video_fps(video_path : str) -> Optional[float]:
	if is_video(video_path):
		return extract_video_metadata(video_path).get('fps')

	return None


def restrict_video_fps(video_path : str, fps : Fps) -> Fps:
	if is_video(video_path):
		video_fps = detect_video_fps(video_path)
		if video_fps < fps:
			return video_fps
	return fps


def detect_video_duration(video_path : str) -> Duration:
	video_frame_total = count_video_frame_total(video_path)
	video_fps = detect_video_fps(video_path)

	if video_frame_total and video_fps:
		return video_frame_total / video_fps
	return 0


def count_trim_frame_total(video_path : str, trim_frame_start : Optional[int], trim_frame_end : Optional[int]) -> int:
	trim_frame_start, trim_frame_end = restrict_trim_frame(video_path, trim_frame_start, trim_frame_end)

	return trim_frame_end - trim_frame_start


def restrict_trim_frame(video_path : str, trim_frame_start : Optional[int], trim_frame_end : Optional[int]) -> Tuple[int, int]:
	video_frame_total = count_video_frame_total(video_path)

	if isinstance(trim_frame_start, int):
		trim_frame_start = max(0, min(trim_frame_start, video_frame_total))
	if isinstance(trim_frame_end, int):
		trim_frame_end = max(0, min(trim_frame_end, video_frame_total))

	if isinstance(trim_frame_start, int) and isinstance(trim_frame_end, int):
		return trim_frame_start, trim_frame_end
	if isinstance(trim_frame_start, int):
		return trim_frame_start, video_frame_total
	if isinstance(trim_frame_end, int):
		return 0, trim_frame_end

	return 0, video_frame_total


#todo: needs review - [probing] [critical: low] resolution now from cached ffprobe metadata
def detect_video_resolution(video_path : str) -> Optional[Resolution]:
	if is_video(video_path):
		return extract_video_metadata(video_path).get('resolution')

	return None


def restrict_video_resolution(video_path : str, resolution : Resolution) -> Resolution:
	if is_video(video_path):
		video_resolution = detect_video_resolution(video_path)
		if video_resolution < resolution:
			return video_resolution
	return resolution


def scale_resolution(resolution : Resolution, scale : Scale) -> Resolution:
	resolution = (int(resolution[0] * scale), int(resolution[1] * scale))
	resolution = normalize_resolution(resolution)
	return resolution


def normalize_resolution(resolution : Tuple[float, float]) -> Resolution:
	width, height = resolution

	if width > 0 and height > 0:
		normalize_width = round(width / 2) * 2
		normalize_height = round(height / 2) * 2
		return normalize_width, normalize_height
	return 0, 0


def pack_resolution(resolution : Resolution) -> str:
	width, height = normalize_resolution(resolution)
	return str(width) + 'x' + str(height)


def unpack_resolution(resolution : str) -> Resolution:
	width, height = map(int, resolution.split('x'))
	return width, height


def detect_frame_orientation(vision_frame : VisionFrame) -> Orientation:
	height, width = vision_frame.shape[:2]

	if width > height:
		return 'landscape'
	return 'portrait'


def restrict_frame(vision_frame : VisionFrame, resolution : Resolution) -> VisionFrame:
	height, width = vision_frame.shape[:2]
	restrict_width, restrict_height = resolution

	if height > restrict_height or width > restrict_width:
		scale = min(restrict_height / height, restrict_width / width)
		new_width = int(width * scale)
		new_height = int(height * scale)
		return cv2.resize(vision_frame, (new_width, new_height))
	return vision_frame


def fit_contain_frame(vision_frame : VisionFrame, resolution : Resolution) -> VisionFrame:
	contain_width, contain_height = resolution
	height, width = vision_frame.shape[:2]
	scale = min(contain_height / height, contain_width / width)
	new_width = int(width * scale)
	new_height = int(height * scale)
	start_x = max(0, (contain_width - new_width) // 2)
	start_y = max(0, (contain_height - new_height) // 2)
	end_x = max(0, contain_width - new_width - start_x)
	end_y = max(0, contain_height - new_height - start_y)
	temp_vision_frame = cv2.resize(vision_frame, (new_width, new_height))
	temp_vision_frame = numpy.pad(temp_vision_frame, ((start_y, end_y), (start_x, end_x), (0, 0)))
	return temp_vision_frame


def fit_cover_frame(vision_frame : VisionFrame, resolution : Resolution) -> VisionFrame:
	cover_width, cover_height = resolution
	height, width = vision_frame.shape[:2]
	scale = max(cover_width / width, cover_height / height)
	new_width = int(width * scale)
	new_height = int(height * scale)
	start_x = max(0, (new_width - cover_width) // 2)
	start_y = max(0, (new_height - cover_height) // 2)
	end_x = min(new_width, start_x + cover_width)
	end_y = min(new_height, start_y + cover_height)
	temp_vision_frame = cv2.resize(vision_frame, (new_width, new_height))
	temp_vision_frame = temp_vision_frame[start_y:end_y, start_x:end_x]
	return temp_vision_frame


def obscure_frame(vision_frame : VisionFrame) -> VisionFrame:
	return cv2.GaussianBlur(vision_frame, (99, 99), 0)


def blend_frame(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame, blend_factor : float) -> VisionFrame:
	blend_vision_frame = cv2.addWeighted(source_vision_frame, 1 - blend_factor, target_vision_frame, blend_factor, 0)
	return blend_vision_frame


def conditional_match_frame_color(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame) -> VisionFrame:
	histogram_factor = calculate_histogram_difference(source_vision_frame, target_vision_frame)
	target_vision_frame = blend_frame(target_vision_frame, match_frame_color(source_vision_frame, target_vision_frame), histogram_factor)
	return target_vision_frame


def match_frame_color(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame) -> VisionFrame:
	color_difference_sizes = numpy.linspace(16, target_vision_frame.shape[0], 3, endpoint = False)

	for color_difference_size in color_difference_sizes:
		source_vision_frame = equalize_frame_color(source_vision_frame, target_vision_frame, normalize_resolution((color_difference_size, color_difference_size)))
	target_vision_frame = equalize_frame_color(source_vision_frame, target_vision_frame, target_vision_frame.shape[:2][::-1])
	return target_vision_frame


def equalize_frame_color(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame, size : Size) -> VisionFrame:
	source_frame_resize = cv2.resize(source_vision_frame, size, interpolation = cv2.INTER_AREA).astype(numpy.float32)
	target_frame_resize = cv2.resize(target_vision_frame, size, interpolation = cv2.INTER_AREA).astype(numpy.float32)
	color_difference_vision_frame = numpy.subtract(source_frame_resize, target_frame_resize)
	color_difference_vision_frame = cv2.resize(color_difference_vision_frame, target_vision_frame.shape[:2][::-1], interpolation = cv2.INTER_CUBIC)
	target_vision_frame = numpy.add(target_vision_frame, color_difference_vision_frame).clip(0, 255).astype(numpy.uint8)
	return target_vision_frame


def calculate_histogram_difference(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame) -> float:
	histogram_source = cv2.calcHist([cv2.cvtColor(source_vision_frame, cv2.COLOR_BGR2HSV)], [ 0, 1 ], None, [ 50, 60 ], [ 0, 180, 0, 256 ])
	histogram_target = cv2.calcHist([cv2.cvtColor(target_vision_frame, cv2.COLOR_BGR2HSV)], [ 0, 1 ], None, [ 50, 60 ], [ 0, 180, 0, 256 ])
	histogram_difference = float(numpy.interp(cv2.compareHist(histogram_source, histogram_target, cv2.HISTCMP_CORREL), [ -1, 1 ], [ 0, 1 ]))
	return histogram_difference


def blend_vision_frames(source_vision_frame : VisionFrame, target_vision_frame : VisionFrame, blend_factor : float) -> VisionFrame:
	blend_vision_frame = cv2.addWeighted(source_vision_frame, 1 - blend_factor, target_vision_frame, blend_factor, 0)
	return blend_vision_frame


def create_empty_vision_frame() -> VisionFrame:
	return numpy.zeros((1, 1, 3)).astype(numpy.uint8)


def create_tile_frames(vision_frame : VisionFrame, size : Size) -> Tuple[List[VisionFrame], int, int]:
	tile_width = size[0] - 2 * size[2]
	pad_size_top = size[1] + size[2]
	pad_size_bottom = pad_size_top + tile_width - (vision_frame.shape[0] + 2 * size[1]) % tile_width
	pad_size_right = pad_size_top + tile_width - (vision_frame.shape[1] + 2 * size[1]) % tile_width
	pad_vision_frame = numpy.pad(vision_frame, ((pad_size_top, pad_size_bottom), (pad_size_top, pad_size_right), (0, 0)))
	pad_height, pad_width = pad_vision_frame.shape[:2]
	row_range = range(size[2], pad_height - size[2], tile_width)
	col_range = range(size[2], pad_width - size[2], tile_width)
	tile_vision_frames = []

	for row_vision_frame in row_range:
		top = row_vision_frame - size[2]
		bottom = row_vision_frame + size[2] + tile_width

		for column_vision_frame in col_range:
			left = column_vision_frame - size[2]
			right = column_vision_frame + size[2] + tile_width
			tile_vision_frames.append(pad_vision_frame[top:bottom, left:right, :])

	return tile_vision_frames, pad_width, pad_height


def merge_tile_frames(tile_vision_frames : List[VisionFrame], temp_width : int, temp_height : int, pad_width : int, pad_height : int, size : Size) -> VisionFrame:
	merge_vision_frame = numpy.zeros((pad_height, pad_width, 3)).astype(numpy.uint8)
	tile_width = tile_vision_frames[0].shape[1] - 2 * size[2]
	tiles_per_row = min(pad_width // tile_width, len(tile_vision_frames))

	for index, tile_vision_frame in enumerate(tile_vision_frames):
		tile_vision_frame = tile_vision_frame[size[2]:-size[2], size[2]:-size[2]]
		row_index = index // tiles_per_row
		col_index = index % tiles_per_row
		top = row_index * tile_vision_frame.shape[0]
		bottom = top + tile_vision_frame.shape[0]
		left = col_index * tile_vision_frame.shape[1]
		right = left + tile_vision_frame.shape[1]
		merge_vision_frame[top:bottom, left:right, :] = tile_vision_frame

	merge_vision_frame = merge_vision_frame[size[1] : size[1] + temp_height, size[1]: size[1] + temp_width, :]
	return merge_vision_frame


def extract_vision_mask(vision_frame : VisionFrame) -> Mask:
	if vision_frame.ndim == 3 and vision_frame.shape[2] == 4:
		return vision_frame[:, :, 3]
	return numpy.full(vision_frame.shape[:2], 255, dtype = numpy.uint8)


def merge_vision_mask(vision_frame : VisionFrame, vision_mask : Mask) -> VisionFrame:
	return numpy.dstack((vision_frame[:, :, :3], vision_mask))


def conditional_merge_vision_mask(vision_frame : VisionFrame, vision_mask : Mask) -> VisionFrame:
	if numpy.any(vision_mask < 255):
		return merge_vision_mask(vision_frame, vision_mask)
	return vision_frame