Next (#477)

* Add real_hatgan_x4 model

* Mark it as NEXT

* Force download to be executed and exit

* Fix frame per second interpolation

* 5 to 68 landmark (#456)

* changes

* changes

* Adjust model url

* Cleanup 5 to 68 landmark convertion

* Move everything to face analyser

* Introduce matrix only face helper

* Revert facefusion.ini

* Adjust limit due false positive analysis

* changes (#457)

* Use pixel format yuv422p to merge video

* Fix some code

* Minor cleanup

* Add gpen_bfr_1024 and gpen_bfr_2048

* Revert it back to yuv420p due compatibility issues

* Add debug back to ffmpeg

* Add debug back to ffmpeg

* Migrate to conda (#461)

* Migrate from venv to conda

* Migrate from venv to conda

* Message when conda is not activated

* Use release for every slider (#463)

* Use release event handler for every slider

* Move more sliders to release handler

* Move more sliders to release handler

* Add get_ui_components() to simplify code

* Revert some changes on frame slider

* Add the first iteration of a frame colorizer

* Support for the DDColor model

* Improve model file handling

* Improve model file handling part2

* Remove deoldify

* Remove deoldify

* Voice separator (#468)

* changes

* changes

* changes

* changes

* changes

* changes

* Rename audio extractor to voice extractor

* Cosmetic changes

* Cosmetic changes

* Fix fps lowering and boosting

* Fix fps lowering and boosting

* Fix fps lowering and boosting

* Some refactoring for audio.py and some astype() here and there (#470)

* Some refactoring for audio.py and some astype() here and there

* Fix lint

* Spacing

* Add mp3 to benchmark suite for lip syncer testing

* Improve naming

* Adjust chunk size

* Use higher quality

* Revert "Use higher quality"

This reverts commit d32f287572.

* Improve naming in ffmpeg.py

* Simplify code

* Better fps calculation

* Fix naming here and there

* Add back real esrgan x2

* Remove trailing comma

* Update wording and README

* Use semaphore to prevent frame colorizer memory issues

* Revert "Remove deoldify"

This reverts commit bd8034cbc7.

* Remove unused type from frame colorizer

* Adjust naming

* Add missing clear of model initializer

* Change nvenc preset mappping to support old FFMPEG 4

* Update onnxruntime to 1.17.1

* Fix lint

* Prepare 2.5.0

* Fix Gradio overrides

* Add Deoldify Artistic back

* Feat/audio refactoring (#476)

* Improve audio naming and variables

* Improve audio naming and variables

* Refactor voice extractor like crazy

* Refactor voice extractor like crazy

* Remove spaces

* Update the usage

---------

Co-authored-by: Harisreedhar <46858047+harisreedhar@users.noreply.github.com>

facefusion/processors/frame/choices.py

@@ -1,13 +1,15 @@
 from typing import List
 
 from facefusion.common_helper import create_int_range
-from facefusion.processors.frame.typings import FaceDebuggerItem, FaceEnhancerModel, FaceSwapperModel, FrameEnhancerModel, LipSyncerModel
+from facefusion.processors.frame.typings import FaceDebuggerItem, FaceEnhancerModel, FaceSwapperModel, FrameColorizerModel, FrameEnhancerModel, LipSyncerModel
 
-face_debugger_items : List[FaceDebuggerItem] = [ 'bounding-box', 'face-landmark-5', 'face-landmark-5/68', 'face-landmark-68', 'face-mask', 'face-detector-score', 'face-landmarker-score', 'age', 'gender' ]
-face_enhancer_models : List[FaceEnhancerModel] = [ 'codeformer', 'gfpgan_1.2', 'gfpgan_1.3', 'gfpgan_1.4', 'gpen_bfr_256', 'gpen_bfr_512', 'restoreformer_plus_plus' ]
+face_debugger_items : List[FaceDebuggerItem] = [ 'bounding-box', 'face-landmark-5', 'face-landmark-5/68', 'face-landmark-68', 'face-landmark-68/5', 'face-mask', 'face-detector-score', 'face-landmarker-score', 'age', 'gender' ]
+face_enhancer_models : List[FaceEnhancerModel] = [ 'codeformer', 'gfpgan_1.2', 'gfpgan_1.3', 'gfpgan_1.4', 'gpen_bfr_256', 'gpen_bfr_512', 'gpen_bfr_1024', 'gpen_bfr_2048', 'restoreformer_plus_plus' ]
 face_swapper_models : List[FaceSwapperModel] = [ 'blendswap_256', 'inswapper_128', 'inswapper_128_fp16', 'simswap_256', 'simswap_512_unofficial', 'uniface_256' ]
-frame_enhancer_models : List[FrameEnhancerModel] = [ 'lsdir_x4', 'nomos8k_sc_x4', 'real_esrgan_x4',  'real_esrgan_x4_fp16', 'span_kendata_x4' ]
+frame_colorizer_models : List[FrameColorizerModel] = [ 'ddcolor', 'ddcolor_artistic', 'deoldify_artistic' ]
+frame_enhancer_models : List[FrameEnhancerModel] = [ 'lsdir_x4', 'nomos8k_sc_x4', 'real_esrgan_x2', 'real_esrgan_x2_fp16', 'real_esrgan_x4', 'real_esrgan_x4_fp16', 'real_hatgan_x4', 'span_kendata_x4' ]
 lip_syncer_models : List[LipSyncerModel] = [ 'wav2lip_gan' ]
 
 face_enhancer_blend_range : List[int] = create_int_range(0, 100, 1)
+frame_colorizer_blend_range : List[int] = create_int_range(0, 100, 1)
 frame_enhancer_blend_range : List[int] = create_int_range(0, 100, 1)

facefusion/processors/frame/globals.py

@@ -1,11 +1,13 @@
 from typing import List, Optional
 
-from facefusion.processors.frame.typings import FaceDebuggerItem, FaceEnhancerModel, FaceSwapperModel, FrameEnhancerModel, LipSyncerModel
+from facefusion.processors.frame.typings import FaceDebuggerItem, FaceEnhancerModel, FaceSwapperModel, FrameColorizerModel, FrameEnhancerModel, LipSyncerModel
 
 face_debugger_items : Optional[List[FaceDebuggerItem]] = None
 face_enhancer_model : Optional[FaceEnhancerModel] = None
 face_enhancer_blend : Optional[int] = None
 face_swapper_model : Optional[FaceSwapperModel] = None
+frame_colorizer_model : Optional[FrameColorizerModel] = None
+frame_colorizer_blend : Optional[int] = None
 frame_enhancer_model : Optional[FrameEnhancerModel] = None
 frame_enhancer_blend : Optional[int] = None
 lip_syncer_model : Optional[LipSyncerModel] = None

facefusion/processors/frame/modules/face_debugger.py

@@ -11,7 +11,7 @@ from facefusion.face_masker import create_static_box_mask, create_occlusion_mask
 from facefusion.face_helper import warp_face_by_face_landmark_5, categorize_age, categorize_gender
 from facefusion.face_store import get_reference_faces
 from facefusion.content_analyser import clear_content_analyser
-from facefusion.typing import Face, VisionFrame, UpdateProcess, ProcessMode, QueuePayload
+from facefusion.typing import Face, VisionFrame, UpdateProgress, ProcessMode, QueuePayload
 from facefusion.vision import read_image, read_static_image, write_image
 from facefusion.processors.frame.typings import FaceDebuggerInputs
 from facefusion.processors.frame import globals as frame_processors_globals, choices as frame_processors_choices
@@ -74,6 +74,7 @@ def debug_face(target_face : Face, temp_vision_frame : VisionFrame) -> VisionFra
 	bounding_box = target_face.bounding_box.astype(numpy.int32)
 	temp_vision_frame = temp_vision_frame.copy()
 	has_face_landmark_5_fallback = numpy.array_equal(target_face.landmarks.get('5'), target_face.landmarks.get('5/68'))
+	has_face_landmark_68_fallback = numpy.array_equal(target_face.landmarks.get('68'), target_face.landmarks.get('68/5'))
 
 	if 'bounding-box' in frame_processors_globals.face_debugger_items:
 		cv2.rectangle(temp_vision_frame, (bounding_box[0], bounding_box[1]), (bounding_box[2], bounding_box[3]), primary_color, 2)
@@ -109,7 +110,11 @@ def debug_face(target_face : Face, temp_vision_frame : VisionFrame) -> VisionFra
 	if 'face-landmark-68' in frame_processors_globals.face_debugger_items and numpy.any(target_face.landmarks.get('68')):
 		face_landmark_68 = target_face.landmarks.get('68').astype(numpy.int32)
 		for index in range(face_landmark_68.shape[0]):
-			cv2.circle(temp_vision_frame, (face_landmark_68[index][0], face_landmark_68[index][1]), 3, secondary_color, -1)
+			cv2.circle(temp_vision_frame, (face_landmark_68[index][0], face_landmark_68[index][1]), 3, tertiary_color if has_face_landmark_68_fallback else secondary_color, -1)
+	if 'face-landmark-68/5' in frame_processors_globals.face_debugger_items and numpy.any(target_face.landmarks.get('68')):
+		face_landmark_68 = target_face.landmarks.get('68/5').astype(numpy.int32)
+		for index in range(face_landmark_68.shape[0]):
+			cv2.circle(temp_vision_frame, (face_landmark_68[index][0], face_landmark_68[index][1]), 3, primary_color, -1)
 	if bounding_box[3] - bounding_box[1] > 50 and bounding_box[2] - bounding_box[0] > 50:
 		top = bounding_box[1]
 		left = bounding_box[0] - 20
@@ -157,7 +162,7 @@ def process_frame(inputs : FaceDebuggerInputs) -> VisionFrame:
 	return target_vision_frame
 
 
-def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProcess) -> None:
+def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
 	reference_faces = get_reference_faces() if 'reference' in facefusion.globals.face_selector_mode else None
 
 	for queue_payload in process_manager.manage(queue_payloads):
@@ -169,7 +174,7 @@ def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload]
 			'target_vision_frame': target_vision_frame
 		})
 		write_image(target_vision_path, output_vision_frame)
-		update_progress()
+		update_progress(1)
 
 
 def process_image(source_paths : List[str], target_path : str, output_path : str) -> None:

facefusion/processors/frame/modules/face_enhancer.py

@@ -16,7 +16,7 @@ from facefusion.execution import apply_execution_provider_options
 from facefusion.content_analyser import clear_content_analyser
 from facefusion.face_store import get_reference_faces
 from facefusion.normalizer import normalize_output_path
-from facefusion.typing import Face, VisionFrame, UpdateProcess, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
+from facefusion.typing import Face, VisionFrame, UpdateProgress, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
 from facefusion.common_helper import create_metavar
 from facefusion.filesystem import is_file, is_image, is_video, resolve_relative_path
 from facefusion.download import conditional_download, is_download_done
@@ -73,6 +73,20 @@ MODELS : ModelSet =\
 		'template': 'ffhq_512',
 		'size': (512, 512)
 	},
+	'gpen_bfr_1024':
+	{
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/gpen_bfr_1024.onnx',
+		'path': resolve_relative_path('../.assets/models/gpen_bfr_1024.onnx'),
+		'template': 'ffhq_512',
+		'size': (1024, 1024)
+	},
+	'gpen_bfr_2048':
+	{
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/gpen_bfr_2048.onnx',
+		'path': resolve_relative_path('../.assets/models/gpen_bfr_2048.onnx'),
+		'template': 'ffhq_512',
+		'size': (2048, 2048)
+	},
 	'restoreformer_plus_plus':
 	{
 		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/restoreformer_plus_plus.onnx',
@@ -131,22 +145,25 @@ def apply_args(program : ArgumentParser) -> None:
 
 
 def pre_check() -> bool:
+	download_directory_path = resolve_relative_path('../.assets/models')
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download:
-		download_directory_path = resolve_relative_path('../.assets/models')
-		model_url = get_options('model').get('url')
 		process_manager.check()
 		conditional_download(download_directory_path, [ model_url ])
 		process_manager.end()
-	return True
+	return is_file(model_path)
 
 
 def post_check() -> bool:
 	model_url = get_options('model').get('url')
 	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download and not is_download_done(model_url, model_path):
 		logger.error(wording.get('model_download_not_done') + wording.get('exclamation_mark'), NAME)
 		return False
-	elif not is_file(model_path):
+	if not is_file(model_path):
 		logger.error(wording.get('model_file_not_present') + wording.get('exclamation_mark'), NAME)
 		return False
 	return True
@@ -202,7 +219,7 @@ def apply_enhance(crop_vision_frame : VisionFrame) -> VisionFrame:
 		if frame_processor_input.name == 'input':
 			frame_processor_inputs[frame_processor_input.name] = crop_vision_frame
 		if frame_processor_input.name == 'weight':
-			weight = numpy.array([ 1 ], dtype = numpy.double)
+			weight = numpy.array([ 1 ]).astype(numpy.double)
 			frame_processor_inputs[frame_processor_input.name] = weight
 	with THREAD_SEMAPHORE:
 		crop_vision_frame = frame_processor.run(None, frame_processor_inputs)[0][0]
@@ -256,7 +273,7 @@ def process_frame(inputs : FaceEnhancerInputs) -> VisionFrame:
 	return target_vision_frame
 
 
-def process_frames(source_path : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProcess) -> None:
+def process_frames(source_path : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
 	reference_faces = get_reference_faces() if 'reference' in facefusion.globals.face_selector_mode else None
 
 	for queue_payload in process_manager.manage(queue_payloads):
@@ -268,7 +285,7 @@ def process_frames(source_path : List[str], queue_payloads : List[QueuePayload],
 			'target_vision_frame': target_vision_frame
 		})
 		write_image(target_vision_path, output_vision_frame)
-		update_progress()
+		update_progress(1)
 
 
 def process_image(source_path : str, target_path : str, output_path : str) -> None:

facefusion/processors/frame/modules/face_swapper.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Literal, Optional
 from argparse import ArgumentParser
 from time import sleep
+import platform
 import threading
 import numpy
 import onnx
@@ -15,10 +16,9 @@ from facefusion.face_analyser import get_one_face, get_average_face, get_many_fa
 from facefusion.face_masker import create_static_box_mask, create_occlusion_mask, create_region_mask, clear_face_occluder, clear_face_parser
 from facefusion.face_helper import warp_face_by_face_landmark_5, paste_back
 from facefusion.face_store import get_reference_faces
-from facefusion.common_helper import extract_major_version
 from facefusion.content_analyser import clear_content_analyser
 from facefusion.normalizer import normalize_output_path
-from facefusion.typing import Face, Embedding, VisionFrame, UpdateProcess, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
+from facefusion.typing import Face, Embedding, VisionFrame, UpdateProgress, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
 from facefusion.filesystem import is_file, is_image, has_image, is_video, filter_image_paths, resolve_relative_path
 from facefusion.download import conditional_download, is_download_done
 from facefusion.vision import read_image, read_static_image, read_static_images, write_image
@@ -27,7 +27,7 @@ from facefusion.processors.frame import globals as frame_processors_globals
 from facefusion.processors.frame import choices as frame_processors_choices
 
 FRAME_PROCESSOR = None
-MODEL_MATRIX = None
+MODEL_INITIALIZER = None
 THREAD_LOCK : threading.Lock = threading.Lock()
 NAME = __name__.upper()
 MODELS : ModelSet =\
@@ -114,23 +114,23 @@ def clear_frame_processor() -> None:
 	FRAME_PROCESSOR = None
 
 
-def get_model_matrix() -> Any:
-	global MODEL_MATRIX
+def get_model_initializer() -> Any:
+	global MODEL_INITIALIZER
 
 	with THREAD_LOCK:
 		while process_manager.is_checking():
 			sleep(0.5)
-		if MODEL_MATRIX is None:
+		if MODEL_INITIALIZER is None:
 			model_path = get_options('model').get('path')
 			model = onnx.load(model_path)
-			MODEL_MATRIX = numpy_helper.to_array(model.graph.initializer[-1])
-	return MODEL_MATRIX
+			MODEL_INITIALIZER = numpy_helper.to_array(model.graph.initializer[-1])
+	return MODEL_INITIALIZER
 
 
-def clear_model_matrix() -> None:
-	global MODEL_MATRIX
+def clear_model_initializer() -> None:
+	global MODEL_INITIALIZER
 
-	MODEL_MATRIX = None
+	MODEL_INITIALIZER = None
 
 
 def get_options(key : Literal['model']) -> Any:
@@ -151,8 +151,7 @@ def set_options(key : Literal['model'], value : Any) -> None:
 
 
 def register_args(program : ArgumentParser) -> None:
-	onnxruntime_version = extract_major_version(onnxruntime.__version__)
-	if onnxruntime_version > (1, 16):
+	if platform.system().lower() == 'darwin':
 		face_swapper_model_fallback = 'inswapper_128'
 	else:
 		face_swapper_model_fallback = 'inswapper_128_fp16'
@@ -173,22 +172,25 @@ def apply_args(program : ArgumentParser) -> None:
 
 
 def pre_check() -> bool:
+	download_directory_path = resolve_relative_path('../.assets/models')
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download:
-		download_directory_path = resolve_relative_path('../.assets/models')
-		model_url = get_options('model').get('url')
 		process_manager.check()
 		conditional_download(download_directory_path, [ model_url ])
 		process_manager.end()
-	return True
+	return is_file(model_path)
 
 
 def post_check() -> bool:
 	model_url = get_options('model').get('url')
 	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download and not is_download_done(model_url, model_path):
 		logger.error(wording.get('model_download_not_done') + wording.get('exclamation_mark'), NAME)
 		return False
-	elif not is_file(model_path):
+	if not is_file(model_path):
 		logger.error(wording.get('model_file_not_present') + wording.get('exclamation_mark'), NAME)
 		return False
 	return True
@@ -216,8 +218,8 @@ def pre_process(mode : ProcessMode) -> bool:
 def post_process() -> None:
 	read_static_image.cache_clear()
 	if facefusion.globals.video_memory_strategy == 'strict' or facefusion.globals.video_memory_strategy == 'moderate':
+		clear_model_initializer()
 		clear_frame_processor()
-		clear_model_matrix()
 	if facefusion.globals.video_memory_strategy == 'strict':
 		clear_face_analyser()
 		clear_content_analyser()
@@ -281,9 +283,9 @@ def prepare_source_frame(source_face : Face) -> VisionFrame:
 def prepare_source_embedding(source_face : Face) -> Embedding:
 	model_type = get_options('model').get('type')
 	if model_type == 'inswapper':
-		model_matrix = get_model_matrix()
+		model_initializer = get_model_initializer()
 		source_embedding = source_face.embedding.reshape((1, -1))
-		source_embedding = numpy.dot(source_embedding, model_matrix) / numpy.linalg.norm(source_embedding)
+		source_embedding = numpy.dot(source_embedding, model_initializer) / numpy.linalg.norm(source_embedding)
 	else:
 		source_embedding = source_face.normed_embedding.reshape(1, -1)
 	return source_embedding
@@ -332,7 +334,7 @@ def process_frame(inputs : FaceSwapperInputs) -> VisionFrame:
 	return target_vision_frame
 
 
-def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProcess) -> None:
+def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
 	reference_faces = get_reference_faces() if 'reference' in facefusion.globals.face_selector_mode else None
 	source_frames = read_static_images(source_paths)
 	source_face = get_average_face(source_frames)
@@ -347,7 +349,7 @@ def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload]
 			'target_vision_frame': target_vision_frame
 		})
 		write_image(target_vision_path, output_vision_frame)
-		update_progress()
+		update_progress(1)
 
 
 def process_image(source_paths : List[str], target_path : str, output_path : str) -> None:

facefusion/processors/frame/modules/frame_colorizer.py

@@ -0,0 +1,232 @@
+from typing import Any, List, Literal, Optional
+from argparse import ArgumentParser
+from time import sleep
+import threading
+import cv2
+import numpy
+import onnxruntime
+
+import facefusion.globals
+import facefusion.processors.frame.core as frame_processors
+from facefusion import config, process_manager, logger, wording
+from facefusion.face_analyser import clear_face_analyser
+from facefusion.content_analyser import clear_content_analyser
+from facefusion.execution import apply_execution_provider_options
+from facefusion.normalizer import normalize_output_path
+from facefusion.typing import Face, VisionFrame, UpdateProgress, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
+from facefusion.common_helper import create_metavar
+from facefusion.filesystem import is_file, resolve_relative_path, is_image, is_video
+from facefusion.download import conditional_download, is_download_done
+from facefusion.vision import read_image, read_static_image, write_image
+from facefusion.processors.frame.typings import FrameColorizerInputs
+from facefusion.processors.frame import globals as frame_processors_globals
+from facefusion.processors.frame import choices as frame_processors_choices
+
+FRAME_PROCESSOR = None
+THREAD_LOCK : threading.Lock = threading.Lock()
+THREAD_SEMAPHORE : threading.Semaphore = threading.Semaphore()
+NAME = __name__.upper()
+MODELS : ModelSet =\
+{
+	'ddcolor':
+	{
+		'type': 'ddcolor',
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/ddcolor.onnx',
+		'path': resolve_relative_path('../.assets/models/ddcolor.onnx'),
+		'size': (512, 512)
+	},
+	'ddcolor_artistic':
+	{
+		'type': 'ddcolor',
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/ddcolor_artistic.onnx',
+		'path': resolve_relative_path('../.assets/models/ddcolor_artistic.onnx'),
+		'size': (512, 512)
+	},
+	'deoldify_artistic':
+	{
+		'type': 'deoldify',
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/deoldify_artistic.onnx',
+		'path': resolve_relative_path('../.assets/models/deoldify_artistic.onnx'),
+		'size': (512, 512)
+	}
+}
+OPTIONS : Optional[OptionsWithModel] = None
+
+
+def get_frame_processor() -> Any:
+	global FRAME_PROCESSOR
+
+	with THREAD_LOCK:
+		while process_manager.is_checking():
+			sleep(0.5)
+		if FRAME_PROCESSOR is None:
+			model_path = get_options('model').get('path')
+			FRAME_PROCESSOR = onnxruntime.InferenceSession(model_path, providers = apply_execution_provider_options(facefusion.globals.execution_providers))
+	return FRAME_PROCESSOR
+
+
+def clear_frame_processor() -> None:
+	global FRAME_PROCESSOR
+
+	FRAME_PROCESSOR = None
+
+
+def get_options(key : Literal['model']) -> Any:
+	global OPTIONS
+
+	if OPTIONS is None:
+		OPTIONS =\
+		{
+			'model': MODELS[frame_processors_globals.frame_colorizer_model]
+		}
+	return OPTIONS.get(key)
+
+
+def set_options(key : Literal['model'], value : Any) -> None:
+	global OPTIONS
+
+	OPTIONS[key] = value
+
+
+def register_args(program : ArgumentParser) -> None:
+	program.add_argument('--frame-colorizer-model', help = wording.get('help.frame_colorizer_model'), default = config.get_str_value('frame_processors.frame_colorizer_model', 'ddcolor'), choices = frame_processors_choices.frame_colorizer_models)
+	program.add_argument('--frame-colorizer-blend', help = wording.get('help.frame_colorizer_blend'), type = int, default = config.get_int_value('frame_processors.frame_colorizer_blend', '100'), choices = frame_processors_choices.frame_colorizer_blend_range, metavar = create_metavar(frame_processors_choices.frame_colorizer_blend_range))
+
+
+def apply_args(program : ArgumentParser) -> None:
+	args = program.parse_args()
+	frame_processors_globals.frame_colorizer_model = args.frame_colorizer_model
+	frame_processors_globals.frame_colorizer_blend = args.frame_colorizer_blend
+
+
+def pre_check() -> bool:
+	download_directory_path = resolve_relative_path('../.assets/models')
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
+	if not facefusion.globals.skip_download:
+		process_manager.check()
+		conditional_download(download_directory_path, [ model_url ])
+		process_manager.end()
+	return is_file(model_path)
+
+
+def post_check() -> bool:
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
+	if not facefusion.globals.skip_download and not is_download_done(model_url, model_path):
+		logger.error(wording.get('model_download_not_done') + wording.get('exclamation_mark'), NAME)
+		return False
+	if not is_file(model_path):
+		logger.error(wording.get('model_file_not_present') + wording.get('exclamation_mark'), NAME)
+		return False
+	return True
+
+
+def pre_process(mode : ProcessMode) -> bool:
+	if mode in [ 'output', 'preview' ] and not is_image(facefusion.globals.target_path) and not is_video(facefusion.globals.target_path):
+		logger.error(wording.get('select_image_or_video_target') + wording.get('exclamation_mark'), NAME)
+		return False
+	if mode == 'output' and not normalize_output_path(facefusion.globals.target_path, facefusion.globals.output_path):
+		logger.error(wording.get('select_file_or_directory_output') + wording.get('exclamation_mark'), NAME)
+		return False
+	return True
+
+
+def post_process() -> None:
+	read_static_image.cache_clear()
+	if facefusion.globals.video_memory_strategy == 'strict' or facefusion.globals.video_memory_strategy == 'moderate':
+		clear_frame_processor()
+	if facefusion.globals.video_memory_strategy == 'strict':
+		clear_face_analyser()
+		clear_content_analyser()
+
+
+def colorize_frame(temp_vision_frame : VisionFrame) -> VisionFrame:
+	frame_processor = get_frame_processor()
+	prepare_vision_frame = prepare_temp_frame(temp_vision_frame)
+	with THREAD_SEMAPHORE:
+		color_vision_frame = frame_processor.run(None,
+		{
+			frame_processor.get_inputs()[0].name: prepare_vision_frame
+		})[0][0]
+	color_vision_frame = merge_color_frame(temp_vision_frame, color_vision_frame)
+	color_vision_frame = blend_frame(temp_vision_frame, color_vision_frame)
+	return color_vision_frame
+
+
+def prepare_temp_frame(temp_vision_frame : VisionFrame) -> VisionFrame:
+	model_size = get_options('model').get('size')
+	model_type = get_options('model').get('type')
+	temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_BGR2GRAY)
+	temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_GRAY2RGB)
+	if model_type == 'ddcolor':
+		temp_vision_frame = (temp_vision_frame / 255.0).astype(numpy.float32)
+		temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_RGB2LAB)[:, :, :1]
+		temp_vision_frame = numpy.concatenate((temp_vision_frame, numpy.zeros_like(temp_vision_frame), numpy.zeros_like(temp_vision_frame)), axis = -1)
+		temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_LAB2RGB)
+	temp_vision_frame = cv2.resize(temp_vision_frame, model_size)
+	temp_vision_frame = temp_vision_frame.transpose((2, 0, 1))
+	temp_vision_frame = numpy.expand_dims(temp_vision_frame, axis = 0).astype(numpy.float32)
+	return temp_vision_frame
+
+
+def merge_color_frame(temp_vision_frame : VisionFrame, color_vision_frame : VisionFrame) -> VisionFrame:
+	model_type = get_options('model').get('type')
+	color_vision_frame = color_vision_frame.transpose(1, 2, 0)
+	color_vision_frame = cv2.resize(color_vision_frame, (temp_vision_frame.shape[1], temp_vision_frame.shape[0]))
+	if model_type == 'ddcolor':
+		temp_vision_frame = (temp_vision_frame / 255.0).astype(numpy.float32)
+		temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_BGR2LAB)[:, :, :1]
+		color_vision_frame = numpy.concatenate((temp_vision_frame, color_vision_frame), axis = -1)
+		color_vision_frame = cv2.cvtColor(color_vision_frame, cv2.COLOR_LAB2BGR)
+		color_vision_frame = (color_vision_frame * 255.0).round().astype(numpy.uint8)
+	if model_type == 'deoldify':
+		temp_blue_channel, _, _ = cv2.split(temp_vision_frame)
+		color_vision_frame = cv2.cvtColor(color_vision_frame, cv2.COLOR_BGR2RGB).astype(numpy.uint8)
+		color_vision_frame = cv2.cvtColor(color_vision_frame, cv2.COLOR_BGR2LAB)
+		_, color_green_channel, color_red_channel = cv2.split(color_vision_frame)
+		color_vision_frame = cv2.merge((temp_blue_channel, color_green_channel, color_red_channel))
+		color_vision_frame = cv2.cvtColor(color_vision_frame, cv2.COLOR_LAB2BGR)
+	return color_vision_frame
+
+
+def blend_frame(temp_vision_frame : VisionFrame, paste_vision_frame : VisionFrame) -> VisionFrame:
+	frame_colorizer_blend = 1 - (frame_processors_globals.frame_colorizer_blend / 100)
+	temp_vision_frame = cv2.addWeighted(temp_vision_frame, frame_colorizer_blend, paste_vision_frame, 1 - frame_colorizer_blend, 0)
+	return temp_vision_frame
+
+
+def get_reference_frame(source_face : Face, target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
+	pass
+
+
+def process_frame(inputs : FrameColorizerInputs) -> VisionFrame:
+	target_vision_frame = inputs.get('target_vision_frame')
+	return colorize_frame(target_vision_frame)
+
+
+def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
+	for queue_payload in process_manager.manage(queue_payloads):
+		target_vision_path = queue_payload['frame_path']
+		target_vision_frame = read_image(target_vision_path)
+		output_vision_frame = process_frame(
+		{
+			'target_vision_frame': target_vision_frame
+		})
+		write_image(target_vision_path, output_vision_frame)
+		update_progress(1)
+
+
+def process_image(source_paths : List[str], target_path : str, output_path : str) -> None:
+	target_vision_frame = read_static_image(target_path)
+	output_vision_frame = process_frame(
+	{
+		'target_vision_frame': target_vision_frame
+	})
+	write_image(output_path, output_vision_frame)
+
+
+def process_video(source_paths : List[str], temp_frame_paths : List[str]) -> None:
+	frame_processors.multi_process_frames(None, temp_frame_paths, process_frames)

facefusion/processors/frame/modules/frame_enhancer.py

@@ -13,7 +13,7 @@ from facefusion.face_analyser import clear_face_analyser
 from facefusion.content_analyser import clear_content_analyser
 from facefusion.execution import apply_execution_provider_options
 from facefusion.normalizer import normalize_output_path
-from facefusion.typing import Face, VisionFrame, UpdateProcess, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
+from facefusion.typing import Face, VisionFrame, UpdateProgress, ProcessMode, ModelSet, OptionsWithModel, QueuePayload
 from facefusion.common_helper import create_metavar
 from facefusion.filesystem import is_file, resolve_relative_path, is_image, is_video
 from facefusion.download import conditional_download, is_download_done
@@ -41,6 +41,20 @@ MODELS : ModelSet =\
 		'size': (128, 8, 2),
 		'scale': 4
 	},
+	'real_esrgan_x2':
+	{
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/real_esrgan_x2.onnx',
+		'path': resolve_relative_path('../.assets/models/real_esrgan_x2.onnx'),
+		'size': (128, 8, 2),
+		'scale': 2
+	},
+	'real_esrgan_x2_fp16':
+	{
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/real_esrgan_x2_fp16.onnx',
+		'path': resolve_relative_path('../.assets/models/real_esrgan_x2_fp16.onnx'),
+		'size': (128, 8, 2),
+		'scale': 2
+	},
 	'real_esrgan_x4':
 	{
 		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/real_esrgan_x4.onnx',
@@ -55,6 +69,13 @@ MODELS : ModelSet =\
 		'size': (128, 8, 2),
 		'scale': 4
 	},
+	'real_hatgan_x4':
+	{
+		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/real_hatgan_x4.onnx',
+		'path': resolve_relative_path('../.assets/models/real_hatgan_x4.onnx'),
+		'size': (256, 8, 2),
+		'scale': 4
+	},
 	'span_kendata_x4':
 	{
 		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/span_kendata_x4.onnx',
@@ -113,22 +134,25 @@ def apply_args(program : ArgumentParser) -> None:
 
 
 def pre_check() -> bool:
+	download_directory_path = resolve_relative_path('../.assets/models')
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download:
-		download_directory_path = resolve_relative_path('../.assets/models')
-		model_url = get_options('model').get('url')
 		process_manager.check()
 		conditional_download(download_directory_path, [ model_url ])
 		process_manager.end()
-	return True
+	return is_file(model_path)
 
 
 def post_check() -> bool:
 	model_url = get_options('model').get('url')
 	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download and not is_download_done(model_url, model_path):
 		logger.error(wording.get('model_download_not_done') + wording.get('exclamation_mark'), NAME)
 		return False
-	elif not is_file(model_path):
+	if not is_file(model_path):
 		logger.error(wording.get('model_file_not_present') + wording.get('exclamation_mark'), NAME)
 		return False
 	return True
@@ -172,7 +196,7 @@ def enhance_frame(temp_vision_frame : VisionFrame) -> VisionFrame:
 
 
 def prepare_tile_frame(vision_tile_frame : VisionFrame) -> VisionFrame:
-	vision_tile_frame = numpy.expand_dims(vision_tile_frame[:,:,::-1], axis = 0)
+	vision_tile_frame = numpy.expand_dims(vision_tile_frame[:, :, ::-1], axis = 0)
 	vision_tile_frame = vision_tile_frame.transpose(0, 3, 1, 2)
 	vision_tile_frame = vision_tile_frame.astype(numpy.float32) / 255
 	return vision_tile_frame
@@ -180,14 +204,14 @@ def prepare_tile_frame(vision_tile_frame : VisionFrame) -> VisionFrame:
 
 def normalize_tile_frame(vision_tile_frame : VisionFrame) -> VisionFrame:
 	vision_tile_frame = vision_tile_frame.transpose(0, 2, 3, 1).squeeze(0) * 255
-	vision_tile_frame = vision_tile_frame.clip(0, 255).astype(numpy.uint8)[:,:,::-1]
+	vision_tile_frame = vision_tile_frame.clip(0, 255).astype(numpy.uint8)[:, :, ::-1]
 	return vision_tile_frame
 
 
-def blend_frame(temp_vision_frame : VisionFrame, paste_vision_frame : VisionFrame) -> VisionFrame:
+def blend_frame(temp_vision_frame : VisionFrame, merge_vision_frame : VisionFrame) -> VisionFrame:
 	frame_enhancer_blend = 1 - (frame_processors_globals.frame_enhancer_blend / 100)
-	temp_vision_frame = cv2.resize(temp_vision_frame, (paste_vision_frame.shape[1], paste_vision_frame.shape[0]))
-	temp_vision_frame = cv2.addWeighted(temp_vision_frame, frame_enhancer_blend, paste_vision_frame, 1 - frame_enhancer_blend, 0)
+	temp_vision_frame = cv2.resize(temp_vision_frame, (merge_vision_frame.shape[1], merge_vision_frame.shape[0]))
+	temp_vision_frame = cv2.addWeighted(temp_vision_frame, frame_enhancer_blend, merge_vision_frame, 1 - frame_enhancer_blend, 0)
 	return temp_vision_frame
 
 
@@ -200,7 +224,7 @@ def process_frame(inputs : FrameEnhancerInputs) -> VisionFrame:
 	return enhance_frame(target_vision_frame)
 
 
-def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProcess) -> None:
+def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
 	for queue_payload in process_manager.manage(queue_payloads):
 		target_vision_path = queue_payload['frame_path']
 		target_vision_frame = read_image(target_vision_path)
@@ -209,7 +233,7 @@ def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload]
 			'target_vision_frame': target_vision_frame
 		})
 		write_image(target_vision_path, output_vision_frame)
-		update_progress()
+		update_progress(1)
 
 
 def process_image(source_paths : List[str], target_path : str, output_path : str) -> None:

facefusion/processors/frame/modules/lip_syncer.py

@@ -16,19 +16,19 @@ from facefusion.face_helper import warp_face_by_face_landmark_5, warp_face_by_bo
 from facefusion.face_store import get_reference_faces
 from facefusion.content_analyser import clear_content_analyser
 from facefusion.normalizer import normalize_output_path
-from facefusion.typing import Face, VisionFrame, UpdateProcess, ProcessMode, ModelSet, OptionsWithModel, AudioFrame, QueuePayload
+from facefusion.typing import Face, VisionFrame, UpdateProgress, ProcessMode, ModelSet, OptionsWithModel, AudioFrame, QueuePayload
 from facefusion.filesystem import is_file, has_audio, resolve_relative_path
 from facefusion.download import conditional_download, is_download_done
-from facefusion.audio import read_static_audio, get_audio_frame, create_empty_audio_frame
+from facefusion.audio import read_static_voice, get_voice_frame, create_empty_audio_frame
 from facefusion.filesystem import is_image, is_video, filter_audio_paths
 from facefusion.common_helper import get_first
-from facefusion.vision import read_image, write_image, read_static_image
+from facefusion.vision import read_image, read_static_image, write_image, restrict_video_fps
 from facefusion.processors.frame.typings import LipSyncerInputs
+from facefusion.voice_extractor import clear_voice_extractor
 from facefusion.processors.frame import globals as frame_processors_globals
 from facefusion.processors.frame import choices as frame_processors_choices
 
 FRAME_PROCESSOR = None
-MODEL_MATRIX = None
 THREAD_LOCK : threading.Lock = threading.Lock()
 NAME = __name__.upper()
 MODELS : ModelSet =\
@@ -36,7 +36,7 @@ MODELS : ModelSet =\
 	'wav2lip_gan':
 	{
 		'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models/wav2lip_gan.onnx',
-		'path': resolve_relative_path('../.assets/models/wav2lip_gan.onnx'),
+		'path': resolve_relative_path('../.assets/models/wav2lip_gan.onnx')
 	}
 }
 OPTIONS : Optional[OptionsWithModel] = None
@@ -87,22 +87,25 @@ def apply_args(program : ArgumentParser) -> None:
 
 
 def pre_check() -> bool:
+	download_directory_path = resolve_relative_path('../.assets/models')
+	model_url = get_options('model').get('url')
+	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download:
-		download_directory_path = resolve_relative_path('../.assets/models')
-		model_url = get_options('model').get('url')
 		process_manager.check()
 		conditional_download(download_directory_path, [ model_url ])
 		process_manager.end()
-	return True
+	return is_file(model_path)
 
 
 def post_check() -> bool:
 	model_url = get_options('model').get('url')
 	model_path = get_options('model').get('path')
+
 	if not facefusion.globals.skip_download and not is_download_done(model_url, model_path):
 		logger.error(wording.get('model_download_not_done') + wording.get('exclamation_mark'), NAME)
 		return False
-	elif not is_file(model_path):
+	if not is_file(model_path):
 		logger.error(wording.get('model_file_not_present') + wording.get('exclamation_mark'), NAME)
 		return False
 	return True
@@ -123,7 +126,7 @@ def pre_process(mode : ProcessMode) -> bool:
 
 def post_process() -> None:
 	read_static_image.cache_clear()
-	read_static_audio.cache_clear()
+	read_static_voice.cache_clear()
 	if facefusion.globals.video_memory_strategy == 'strict' or facefusion.globals.video_memory_strategy == 'moderate':
 		clear_frame_processor()
 	if facefusion.globals.video_memory_strategy == 'strict':
@@ -131,6 +134,7 @@ def post_process() -> None:
 		clear_content_analyser()
 		clear_face_occluder()
 		clear_face_parser()
+		clear_voice_extractor()
 
 
 def sync_lip(target_face : Face, temp_audio_frame : AudioFrame, temp_vision_frame : VisionFrame) -> VisionFrame:
@@ -138,14 +142,11 @@ def sync_lip(target_face : Face, temp_audio_frame : AudioFrame, temp_vision_fram
 	crop_mask_list = []
 	temp_audio_frame = prepare_audio_frame(temp_audio_frame)
 	crop_vision_frame, affine_matrix = warp_face_by_face_landmark_5(temp_vision_frame, target_face.landmarks.get('5/68'), 'ffhq_512', (512, 512))
-	if numpy.any(target_face.landmarks.get('68')):
-		face_landmark_68 = cv2.transform(target_face.landmarks.get('68').reshape(1, -1, 2), affine_matrix).reshape(-1, 2)
-		bounding_box = create_bounding_box_from_face_landmark_68(face_landmark_68)
-		bounding_box[1] -= numpy.abs(bounding_box[3] - bounding_box[1]) * 0.125
-		mouth_mask = create_mouth_mask(face_landmark_68)
-		crop_mask_list.append(mouth_mask)
-	else:
-		bounding_box = target_face.bounding_box
+	face_landmark_68 = cv2.transform(target_face.landmarks.get('68').reshape(1, -1, 2), affine_matrix).reshape(-1, 2)
+	bounding_box = create_bounding_box_from_face_landmark_68(face_landmark_68)
+	bounding_box[1] -= numpy.abs(bounding_box[3] - bounding_box[1]) * 0.125
+	mouth_mask = create_mouth_mask(face_landmark_68)
+	crop_mask_list.append(mouth_mask)
 	box_mask = create_static_box_mask(crop_vision_frame.shape[:2][::-1], facefusion.globals.face_mask_blur, facefusion.globals.face_mask_padding)
 	crop_mask_list.append(box_mask)
 
@@ -216,14 +217,15 @@ def process_frame(inputs : LipSyncerInputs) -> VisionFrame:
 	return target_vision_frame
 
 
-def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProcess) -> None:
+def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
 	reference_faces = get_reference_faces() if 'reference' in facefusion.globals.face_selector_mode else None
 	source_audio_path = get_first(filter_audio_paths(source_paths))
+	temp_video_fps = restrict_video_fps(facefusion.globals.target_path, facefusion.globals.output_video_fps)
 
 	for queue_payload in process_manager.manage(queue_payloads):
 		frame_number = queue_payload['frame_number']
 		target_vision_path = queue_payload['frame_path']
-		source_audio_frame = get_audio_frame(source_audio_path, facefusion.globals.output_video_fps, frame_number)
+		source_audio_frame = get_voice_frame(source_audio_path, temp_video_fps, frame_number)
 		if not numpy.any(source_audio_frame):
 			source_audio_frame = create_empty_audio_frame()
 		target_vision_frame = read_image(target_vision_path)
@@ -234,7 +236,7 @@ def process_frames(source_paths : List[str], queue_payloads : List[QueuePayload]
 			'target_vision_frame': target_vision_frame
 		})
 		write_image(target_vision_path, output_vision_frame)
-		update_progress()
+		update_progress(1)
 
 
 def process_image(source_paths : List[str], target_path : str, output_path : str) -> None:

facefusion/processors/frame/typings.py

@@ -2,10 +2,11 @@ from typing import Literal, TypedDict
 
 from facefusion.typing import Face, FaceSet, AudioFrame, VisionFrame
 
-FaceDebuggerItem = Literal['bounding-box', 'face-landmark-5', 'face-landmark-5/68', 'face-landmark-68', 'face-mask', 'face-detector-score', 'face-landmarker-score', 'age', 'gender']
-FaceEnhancerModel = Literal['codeformer', 'gfpgan_1.2', 'gfpgan_1.3', 'gfpgan_1.4', 'gpen_bfr_256', 'gpen_bfr_512', 'restoreformer_plus_plus']
+FaceDebuggerItem = Literal['bounding-box', 'face-landmark-5', 'face-landmark-5/68', 'face-landmark-68', 'face-landmark-68/5', 'face-mask', 'face-detector-score', 'face-landmarker-score', 'age', 'gender']
+FaceEnhancerModel = Literal['codeformer', 'gfpgan_1.2', 'gfpgan_1.3', 'gfpgan_1.4', 'gpen_bfr_256', 'gpen_bfr_512', 'gpen_bfr_1024', 'gpen_bfr_2048', 'restoreformer_plus_plus']
 FaceSwapperModel = Literal['blendswap_256', 'inswapper_128', 'inswapper_128_fp16', 'simswap_256', 'simswap_512_unofficial', 'uniface_256']
-FrameEnhancerModel = Literal['lsdir_x4', 'nomos8k_sc_x4', 'real_esrgan_x4',  'real_esrgan_x4_fp16', 'span_kendata_x4']
+FrameColorizerModel = Literal['ddcolor', 'ddcolor_artistic', 'deoldify_artistic']
+FrameEnhancerModel = Literal['lsdir_x4', 'nomos8k_sc_x4', 'real_esrgan_x2', 'real_esrgan_x2_fp16', 'real_esrgan_x4', 'real_esrgan_x4_fp16', 'real_hatgan_x4', 'span_kendata_x4']
 LipSyncerModel = Literal['wav2lip_gan']
 
 FaceDebuggerInputs = TypedDict('FaceDebuggerInputs',
@@ -24,6 +25,10 @@ FaceSwapperInputs = TypedDict('FaceSwapperInputs',
 	'source_face' : Face,
 	'target_vision_frame' : VisionFrame
 })
+FrameColorizerInputs = TypedDict('FrameColorizerInputs',
+{
+	'target_vision_frame' : VisionFrame
+})
 FrameEnhancerInputs = TypedDict('FrameEnhancerInputs',
 {
 	'target_vision_frame' : VisionFrame