vibe code to recent v4 changes

refactor api

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+patch = subprocess

.github/workflows/ci.yml

@@ -35,6 +35,7 @@ jobs:
      python-version: '3.12'
   - run: python install.py --onnxruntime default --skip-conda
   - run: pip install pytest
+  - run: pip install httpx
   - run: pytest
  report:
   needs: test
@@ -52,6 +53,7 @@ jobs:
   - run: pip install coveralls
   - run: pip install pytest
   - run: pip install pytest-cov
+  - run: pip install httpx
   - run: pytest tests --cov facefusion
   - run: coveralls --service github
     env:

.gitignore

@@ -1,6 +1,7 @@
 __pycache__
 .assets
+.claude
 .caches
-.jobs
 .idea
+.jobs
 .vscode

README.md

@@ -8,12 +8,6 @@ FaceFusion
 ![License](https://img.shields.io/badge/license-OpenRAIL--AS-green)
 
 
-Preview
--------
-
-![Preview](https://raw.githubusercontent.com/facefusion/facefusion/master/.github/preview.png?sanitize=true)
-
-
 Installation
 ------------
 
@@ -34,10 +28,10 @@ options:
 
 commands:
     run                                           run the program
-    headless-run                                  run the program in headless mode
     batch-run                                     run the program in batch mode
     force-download                                force automate downloads and exit
     benchmark                                     benchmark the program
+    api                                           start the API server
     job-list                                      list jobs by status
     job-create                                    create a drafted job
     job-submit                                    submit a drafted job to become a queued job

facefusion.ini

@@ -1,3 +1,6 @@
+[workflow]
+workflow =
+
 [paths]
 temp_path =
 jobs_path =
@@ -13,6 +16,7 @@ output_pattern =
 [face_detector]
 face_detector_model =
 face_detector_size =
+face_detector_margin =
 face_detector_angles =
 face_detector_score =
 
@@ -40,32 +44,37 @@ face_mask_regions =
 face_mask_blur =
 face_mask_padding =
 
+[voice_extractor]
+voice_extractor_model =
+
 [frame_extraction]
 trim_frame_start =
 trim_frame_end =
 temp_frame_format =
-keep_temp =
 
 [output_creation]
 output_image_quality =
-output_image_resolution =
+output_image_scale =
 output_audio_encoder =
 output_audio_quality =
 output_audio_volume =
 output_video_encoder =
 output_video_preset =
 output_video_quality =
-output_video_resolution =
+output_video_scale =
 output_video_fps =
 
 [processors]
 processors =
 age_modifier_model =
 age_modifier_direction =
+background_remover_model =
+background_remover_color =
 deep_swapper_model =
 deep_swapper_morph =
 expression_restorer_model =
 expression_restorer_factor =
+expression_restorer_areas =
 face_debugger_items =
 face_editor_model =
 face_editor_eyebrow_direction =
@@ -87,6 +96,7 @@ face_enhancer_blend =
 face_enhancer_weight =
 face_swapper_model =
 face_swapper_pixel_boost =
+face_swapper_weight =
 frame_colorizer_model =
 frame_colorizer_size =
 frame_colorizer_blend =
@@ -95,28 +105,26 @@ frame_enhancer_blend =
 lip_syncer_model =
 lip_syncer_weight =
 
-[uis]
-open_browser =
-ui_layouts =
-ui_workflow =
-
 [download]
 download_providers =
 download_scope =
 
 [benchmark]
+benchmark_mode =
 benchmark_resolutions =
 benchmark_cycle_count =
 
+[api]
+api_host =
+api_port =
+
 [execution]
-execution_device_id =
+execution_device_ids =
 execution_providers =
 execution_thread_count =
-execution_queue_count =
 
 [memory]
 video_memory_strategy =
-system_memory_limit =
 
 [misc]
 log_level =

facefusion/apis/api_helper.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+from starlette.datastructures import Headers
+from starlette.types import Scope
+
+
+def get_sec_websocket_protocol(scope : Scope) -> Optional[str]:
+	protocol_header = Headers(scope = scope).get('Sec-WebSocket-Protocol')
+
+	if protocol_header:
+		protocol, _, _ = protocol_header.partition(',')
+		return protocol.strip()
+
+	return None
+

facefusion/apis/asset_helper.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from facefusion.audio import detect_audio_duration
+from facefusion.filesystem import is_audio, is_image, is_video
+from facefusion.types import AudioMetadata, ImageMetadata, MediaType, VideoMetadata
+from facefusion.vision import count_video_frame_total, detect_image_resolution, detect_video_duration, detect_video_fps, detect_video_resolution
+
+
+def extract_audio_metadata(file_path : str) -> AudioMetadata:
+	metadata : AudioMetadata =\
+	{
+		'duration': detect_audio_duration(file_path),
+		'sample_rate': 0,
+		'frame_total': 0,
+		'channels': 0,
+		'format': ''
+	}
+	return metadata
+
+
+def extract_image_metadata(file_path : str) -> ImageMetadata:
+	resolution = detect_image_resolution(file_path)
+	metadata : ImageMetadata =\
+	{
+		'resolution': resolution if resolution else (0, 0)
+	}
+	return metadata
+
+
+def extract_video_metadata(file_path : str) -> VideoMetadata:
+	resolution = detect_video_resolution(file_path)
+	fps = detect_video_fps(file_path)
+	metadata : VideoMetadata =\
+	{
+		'duration': detect_video_duration(file_path),
+		'frame_total': count_video_frame_total(file_path),
+		'fps': fps if fps else 0.0,
+		'resolution': resolution if resolution else (0, 0)
+	}
+	return metadata
+
+
+def detect_media_type(file_path : str) -> Optional[MediaType]:
+	if is_audio(file_path):
+		return 'audio'
+	if is_image(file_path):
+		return 'image'
+	if is_video(file_path):
+		return 'video'
+	return None

facefusion/apis/asset_store.py

@@ -0,0 +1,92 @@
+import os
+import uuid
+from datetime import datetime, timedelta
+from typing import List, Optional, cast
+
+from facefusion.apis.asset_helper import detect_media_type, extract_audio_metadata, extract_image_metadata, extract_video_metadata
+from facefusion.filesystem import get_file_format, get_file_name
+from facefusion.types import AssetId, AssetSet, AssetStore, AssetType, AudioAsset, AudioFormat, ImageAsset, ImageFormat, SessionId, VideoAsset, VideoFormat
+
+ASSET_STORE : AssetStore = {}
+
+
+def create_asset(session_id : SessionId, asset_type : AssetType, asset_path : str) -> Optional[AudioAsset | ImageAsset | VideoAsset]:
+	asset_id = str(uuid.uuid4())
+	asset_name = get_file_name(asset_path)
+	asset_format = get_file_format(asset_path)
+	asset_size = os.path.getsize(asset_path)
+	media_type = detect_media_type(asset_path)
+	created_at = datetime.now()
+	expires_at = created_at + timedelta(hours = 2)
+
+	if session_id not in ASSET_STORE:
+		ASSET_STORE[session_id] = {}
+
+	if media_type == 'audio':
+		ASSET_STORE[session_id][asset_id] = cast(AudioAsset,
+		{
+			'id': asset_id,
+			'created_at': created_at,
+			'expires_at': expires_at,
+			'type': asset_type,
+			'media': media_type,
+			'name': asset_name,
+			'format': cast(AudioFormat, asset_format),
+			'size': asset_size,
+			'path': asset_path,
+			'metadata': extract_audio_metadata(asset_path)
+		})
+
+	if media_type == 'image':
+		ASSET_STORE[session_id][asset_id] = cast(ImageAsset,
+		{
+			'id': asset_id,
+			'created_at': created_at,
+			'expires_at': expires_at,
+			'type': asset_type,
+			'media': media_type,
+			'name': asset_name,
+			'format': cast(ImageFormat, asset_format),
+			'size': asset_size,
+			'path': asset_path,
+			'metadata': extract_image_metadata(asset_path)
+		})
+
+	if media_type == 'video':
+		ASSET_STORE[session_id][asset_id] = cast(VideoAsset,
+		{
+			'id': asset_id,
+			'created_at': created_at,
+			'expires_at': expires_at,
+			'type': asset_type,
+			'media': media_type,
+			'name': asset_name,
+			'format': cast(VideoFormat, asset_format),
+			'size': asset_size,
+			'path': asset_path,
+			'metadata': extract_video_metadata(asset_path)
+		})
+
+	return ASSET_STORE[session_id].get(asset_id)
+
+
+def get_assets(session_id : SessionId) -> Optional[AssetSet]:
+	return ASSET_STORE.get(session_id)
+
+
+def get_asset(session_id : SessionId, asset_id : AssetId) -> Optional[AudioAsset | ImageAsset | VideoAsset]:
+	if session_id in ASSET_STORE:
+		return ASSET_STORE.get(session_id).get(asset_id)
+	return None
+
+
+def delete_assets(session_id : SessionId, asset_ids : List[AssetId]) -> None:
+	if session_id in ASSET_STORE:
+		for asset_id in asset_ids:
+			if asset_id in ASSET_STORE.get(session_id):
+				del ASSET_STORE[session_id][asset_id]
+	return None
+
+
+def clear() -> None:
+	ASSET_STORE.clear()

facefusion/apis/choices.py

@@ -0,0 +1,162 @@
+from typing import Any, Dict
+
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.status import HTTP_200_OK
+
+import facefusion.choices
+from facefusion.execution import get_available_execution_providers
+from facefusion.ffmpeg import get_available_encoder_set
+from facefusion.processors.modules.face_debugger import choices as face_debugger_choices
+from facefusion.processors.modules.face_enhancer import choices as face_enhancer_choices
+from facefusion.processors.modules.face_swapper import choices as face_swapper_choices
+from facefusion.processors.modules.frame_enhancer import choices as frame_enhancer_choices
+
+
+async def get_choices(request : Request) -> JSONResponse:
+	available_execution_providers = get_available_execution_providers()
+	available_encoder_set = get_available_encoder_set()
+
+	choices_data : Dict[str, Any] =\
+	{
+		'face_detector_models': facefusion.choices.face_detector_models,
+		'face_detector_set': facefusion.choices.face_detector_set,
+		'face_landmarker_models': facefusion.choices.face_landmarker_models,
+		'face_selector_modes': facefusion.choices.face_selector_modes,
+		'face_selector_orders': facefusion.choices.face_selector_orders,
+		'face_selector_genders': facefusion.choices.face_selector_genders,
+		'face_selector_races': facefusion.choices.face_selector_races,
+		'face_occluder_models': facefusion.choices.face_occluder_models,
+		'face_parser_models': facefusion.choices.face_parser_models,
+		'face_mask_types': facefusion.choices.face_mask_types,
+		'face_mask_areas': facefusion.choices.face_mask_areas,
+		'face_mask_regions': facefusion.choices.face_mask_regions,
+		'voice_extractor_models': facefusion.choices.voice_extractor_models,
+		'workflows': facefusion.choices.workflows,
+		'audio_formats': facefusion.choices.audio_formats,
+		'image_formats': facefusion.choices.image_formats,
+		'video_formats': facefusion.choices.video_formats,
+		'temp_frame_formats': facefusion.choices.temp_frame_formats,
+		'output_audio_encoders': available_encoder_set.get('audio'),
+		'output_video_encoders': available_encoder_set.get('video'),
+		'output_video_presets': facefusion.choices.output_video_presets,
+		'execution_providers': available_execution_providers,
+		'video_memory_strategies': facefusion.choices.video_memory_strategies,
+		'log_levels': facefusion.choices.log_levels,
+		'face_swapper_models': face_swapper_choices.face_swapper_models,
+		'face_swapper_set': face_swapper_choices.face_swapper_set,
+		'face_enhancer_models': face_enhancer_choices.face_enhancer_models,
+		'frame_enhancer_models': frame_enhancer_choices.frame_enhancer_models,
+		'face_debugger_items': face_debugger_choices.face_debugger_items,
+		'face_detector_angles': list(facefusion.choices.face_detector_angles),
+		'face_detector_score_range':
+		{
+			'min': min(facefusion.choices.face_detector_score_range),
+			'max': max(facefusion.choices.face_detector_score_range),
+			'step': facefusion.choices.face_detector_score_range[1] - facefusion.choices.face_detector_score_range[0]
+		},
+		'face_landmarker_score_range':
+		{
+			'min': min(facefusion.choices.face_landmarker_score_range),
+			'max': max(facefusion.choices.face_landmarker_score_range),
+			'step': facefusion.choices.face_landmarker_score_range[1] - facefusion.choices.face_landmarker_score_range[0]
+		},
+		'face_mask_blur_range':
+		{
+			'min': min(facefusion.choices.face_mask_blur_range),
+			'max': max(facefusion.choices.face_mask_blur_range),
+			'step': facefusion.choices.face_mask_blur_range[1] - facefusion.choices.face_mask_blur_range[0]
+		},
+		'face_mask_padding_range':
+		{
+			'min': min(facefusion.choices.face_mask_padding_range),
+			'max': max(facefusion.choices.face_mask_padding_range),
+			'step': 1
+		},
+		'face_selector_age_range':
+		{
+			'min': min(facefusion.choices.face_selector_age_range),
+			'max': max(facefusion.choices.face_selector_age_range),
+			'step': 1
+		},
+		'reference_face_distance_range':
+		{
+			'min': min(facefusion.choices.reference_face_distance_range),
+			'max': max(facefusion.choices.reference_face_distance_range),
+			'step': facefusion.choices.reference_face_distance_range[1] - facefusion.choices.reference_face_distance_range[0]
+		},
+		'output_image_quality_range':
+		{
+			'min': min(facefusion.choices.output_image_quality_range),
+			'max': max(facefusion.choices.output_image_quality_range),
+			'step': 1
+		},
+		'output_image_scale_range':
+		{
+			'min': min(facefusion.choices.output_image_scale_range),
+			'max': max(facefusion.choices.output_image_scale_range),
+			'step': facefusion.choices.output_image_scale_range[1] - facefusion.choices.output_image_scale_range[0]
+		},
+		'output_audio_quality_range':
+		{
+			'min': min(facefusion.choices.output_audio_quality_range),
+			'max': max(facefusion.choices.output_audio_quality_range),
+			'step': 1
+		},
+		'output_audio_volume_range':
+		{
+			'min': min(facefusion.choices.output_audio_volume_range),
+			'max': max(facefusion.choices.output_audio_volume_range),
+			'step': 1
+		},
+		'output_video_quality_range':
+		{
+			'min': min(facefusion.choices.output_video_quality_range),
+			'max': max(facefusion.choices.output_video_quality_range),
+			'step': 1
+		},
+		'output_video_scale_range':
+		{
+			'min': min(facefusion.choices.output_video_scale_range),
+			'max': max(facefusion.choices.output_video_scale_range),
+			'step': facefusion.choices.output_video_scale_range[1] - facefusion.choices.output_video_scale_range[0]
+		},
+		'execution_thread_count_range':
+		{
+			'min': min(facefusion.choices.execution_thread_count_range),
+			'max': max(facefusion.choices.execution_thread_count_range),
+			'step': 1
+		},
+		'face_detector_margin_range':
+		{
+			'min': min(facefusion.choices.face_detector_margin_range),
+			'max': max(facefusion.choices.face_detector_margin_range),
+			'step': 1
+		},
+		'face_swapper_weight_range':
+		{
+			'min': min(face_swapper_choices.face_swapper_weight_range),
+			'max': max(face_swapper_choices.face_swapper_weight_range),
+			'step': face_swapper_choices.face_swapper_weight_range[1] - face_swapper_choices.face_swapper_weight_range[0]
+		},
+		'face_enhancer_blend_range':
+		{
+			'min': min(face_enhancer_choices.face_enhancer_blend_range),
+			'max': max(face_enhancer_choices.face_enhancer_blend_range),
+			'step': 1
+		},
+		'face_enhancer_weight_range':
+		{
+			'min': min(face_enhancer_choices.face_enhancer_weight_range),
+			'max': max(face_enhancer_choices.face_enhancer_weight_range),
+			'step': face_enhancer_choices.face_enhancer_weight_range[1] - face_enhancer_choices.face_enhancer_weight_range[0]
+		},
+		'frame_enhancer_blend_range':
+		{
+			'min': min(frame_enhancer_choices.frame_enhancer_blend_range),
+			'max': max(frame_enhancer_choices.frame_enhancer_blend_range),
+			'step': 1
+		}
+	}
+
+	return JSONResponse(choices_data, status_code = HTTP_200_OK)

facefusion/apis/core.py

@@ -0,0 +1,47 @@
+from starlette.applications import Starlette
+from starlette.middleware import Middleware
+from starlette.middleware.cors import CORSMiddleware
+from starlette.routing import Route, WebSocketRoute
+
+from facefusion.apis.choices import get_choices
+from facefusion.apis.endpoints.assets import delete_asset, delete_assets, get_asset, get_assets, upload_asset
+from facefusion.apis.endpoints.ping import websocket_ping
+from facefusion.apis.endpoints.session import create_session, create_session_guard, destroy_session, get_session, refresh_session
+from facefusion.apis.endpoints.state import get_state, set_state
+from facefusion.apis.endpoints.stream import delete_stream, post_stream, websocket_stream
+from facefusion.apis.metrics import websocket_metrics
+from facefusion.apis.remote import remote
+from facefusion.apis.timeline import get_timeline
+from facefusion.apis.version import create_version_guard
+
+
+def create_api() -> Starlette:
+	version_guard = Middleware(create_version_guard)
+	session_guard = Middleware(create_session_guard)
+	routes =\
+	[
+		Route('/session', create_session, methods = [ 'POST' ], middleware = [ version_guard ]),
+		Route('/session', get_session, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/session', refresh_session, methods = [ 'PUT' ], middleware = [ version_guard ]),
+		Route('/session', destroy_session, methods = [ 'DELETE' ], middleware = [ version_guard, session_guard ]),
+		Route('/state', get_state, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/state', set_state, methods = [ 'PUT' ], middleware = [ version_guard, session_guard ]),
+		Route('/assets', get_assets, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/assets', upload_asset, methods = [ 'POST' ], middleware = [ version_guard, session_guard ]),
+		Route('/assets/{asset_id}', get_asset, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/assets/{asset_id}', delete_asset, methods = [ 'DELETE' ], middleware = [ version_guard, session_guard ]),
+		Route('/assets', delete_assets, methods = [ 'DELETE' ], middleware = [ version_guard, session_guard ]),
+		Route('/choices', get_choices, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/remote', remote, methods = [ 'POST' ], middleware = [ version_guard, session_guard ]),
+		Route('/timeline/{count:int}', get_timeline, methods = [ 'GET' ], middleware = [ version_guard, session_guard ]),
+		Route('/stream', post_stream, methods = [ 'POST' ], middleware = [ version_guard, session_guard ]),
+		Route('/stream', delete_stream, methods = [ 'DELETE' ], middleware = [ version_guard, session_guard ]),
+		WebSocketRoute('/stream', websocket_stream, middleware = [ version_guard, session_guard ]),
+		WebSocketRoute('/metrics', websocket_metrics, middleware = [ version_guard, session_guard ]),
+		WebSocketRoute('/ping', websocket_ping, middleware = [ version_guard, session_guard ])
+	]
+
+	api = Starlette(routes = routes)
+	api.add_middleware(CORSMiddleware, allow_origins = [ '*' ], allow_methods = [ '*' ], allow_headers = [ '*' ])
+
+	return api

facefusion/apis/endpoints/assets.py

@@ -0,0 +1,163 @@
+import tempfile
+from typing import Any, Dict, List, Optional
+
+from starlette.datastructures import UploadFile
+from starlette.requests import Request
+from starlette.responses import FileResponse, JSONResponse, Response
+from starlette.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND
+
+from facefusion import session_manager
+from facefusion.apis import asset_store
+from facefusion.apis.asset_helper import detect_media_type
+from facefusion.apis.endpoints.session import extract_access_token
+from facefusion.filesystem import get_file_extension, remove_file
+from facefusion.types import AudioAsset, ImageAsset, VideoAsset
+
+
+def translate_asset(asset : AudioAsset | ImageAsset | VideoAsset) -> Optional[Dict[str, Any]]:
+	return\
+	{
+		'id': asset.get('id'),
+		'created_at': asset.get('created_at').isoformat(),
+		'expires_at': asset.get('expires_at').isoformat(),
+		'type': asset.get('type'),
+		'media_type': asset.get('media'),
+		'filename': asset.get('name'),
+		'format': asset.get('format'),
+		'size': asset.get('size'),
+		'metadata': asset.get('metadata')
+	}
+
+
+async def upload_asset(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	asset_type = request.query_params.get('type')
+
+	if session_id and asset_type in [ 'source', 'target' ]:
+		form = await request.form()
+		upload_files = form.getlist('file')
+		asset_paths = await save_asset_files(upload_files) # type: ignore[arg-type]
+
+		if asset_paths:
+			asset_ids : List[str] = []
+
+			for asset_path in asset_paths:
+				asset = asset_store.create_asset(session_id, asset_type, asset_path) # type: ignore[arg-type]
+				asset_id = asset.get('id')
+
+				if asset_id:
+					asset_ids.append(asset_id)
+
+			if asset_ids:
+				if asset_type == 'target':
+					return JSONResponse(
+					{
+						'asset_id': asset_ids[0]
+					}, status_code = HTTP_201_CREATED)
+
+				return JSONResponse(
+				{
+					'asset_ids': asset_ids
+				}, status_code = HTTP_201_CREATED)
+
+	return Response(status_code = HTTP_400_BAD_REQUEST)
+
+
+async def save_asset_files(upload_files : List[UploadFile]) -> List[str]:
+	asset_paths : List[str] = []
+
+	for upload_file in upload_files:
+		upload_file_extension = get_file_extension(upload_file.filename)
+
+		with tempfile.NamedTemporaryFile(suffix = upload_file_extension, delete = False) as temp_file:
+
+			while upload_chunk := await upload_file.read(1024):
+				temp_file.write(upload_chunk)
+
+			temp_file.flush()
+
+			media_type = detect_media_type(temp_file.name)
+
+			if media_type:
+				asset_paths.append(temp_file.name)
+			else:
+				remove_file(temp_file.name)
+
+	return asset_paths
+
+
+async def get_assets(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	asset_type = request.query_params.get('type')
+
+	if session_id:
+		asset_set = asset_store.get_assets(session_id)
+		assets = []
+
+		if asset_set:
+			for asset in asset_set.values():
+				if not asset_type or asset.get('type') == asset_type:
+					assets.append(translate_asset(asset))
+
+		return JSONResponse(
+		{
+			'assets': assets,
+			'count': len(assets)
+		}, status_code = HTTP_200_OK)
+
+	return Response(status_code = HTTP_400_BAD_REQUEST)
+
+
+async def get_asset(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	asset_id = request.path_params.get('asset_id')
+	action = request.query_params.get('action')
+
+	if session_id and asset_id:
+		asset = asset_store.get_asset(session_id, asset_id)
+
+		if asset:
+			if action == 'download':
+				return FileResponse(asset.get('path'), filename = asset.get('name'))
+
+			return JSONResponse(translate_asset(asset), status_code = HTTP_200_OK)
+
+	return Response(status_code = HTTP_404_NOT_FOUND)
+
+
+async def delete_asset(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	asset_id = request.path_params.get('asset_id')
+
+	if session_id and asset_id:
+		asset_set = asset_store.get_assets(session_id)
+
+		if asset_set and asset_id in asset_set:
+			remove_file(asset_set.get(asset_id).get('path'))
+			asset_store.delete_assets(session_id, [ asset_id ])
+			return Response(status_code = HTTP_200_OK)
+
+	return Response(status_code = HTTP_404_NOT_FOUND)
+
+
+async def delete_assets(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	body = await request.json()
+	asset_ids = body.get('asset_ids')
+
+	if session_id and asset_ids:
+		asset_set = asset_store.get_assets(session_id)
+
+		if asset_set:
+			for asset_id in asset_ids:
+				if asset_id in asset_set:
+					remove_file(asset_set.get(asset_id).get('path'))
+			asset_store.delete_assets(session_id, asset_ids)
+			return Response(status_code = HTTP_200_OK)
+
+	return Response(status_code = HTTP_404_NOT_FOUND)

facefusion/apis/endpoints/ping.py

@@ -0,0 +1,16 @@
+from starlette.websockets import WebSocket
+
+from facefusion.apis.api_helper import get_sec_websocket_protocol
+
+
+async def websocket_ping(websocket : WebSocket) -> None:
+	subprotocol = get_sec_websocket_protocol(websocket.scope)
+
+	await websocket.accept(subprotocol = subprotocol)
+
+	try:
+		while True:
+			await websocket.receive()
+
+	except Exception:
+		pass

facefusion/apis/endpoints/session.py

@@ -0,0 +1,148 @@
+import os
+import secrets
+from typing import Optional
+
+from starlette.datastructures import Headers
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_401_UNAUTHORIZED, HTTP_426_UPGRADE_REQUIRED
+from starlette.types import ASGIApp, Receive, Scope, Send
+
+from facefusion import session_context, session_manager, translator
+from facefusion.apis.api_helper import get_sec_websocket_protocol
+from facefusion.types import Token
+
+
+async def create_session(request : Request) -> JSONResponse:
+	body = await request.json()
+
+	if not body.get('api_key') or body.get('api_key') == os.getenv('FACEFUSION_API_KEY'):
+		session_id = secrets.token_urlsafe(16)
+		session = session_manager.create_session()
+		session_context.set_session_id(session_id)
+		session_manager.set_session(session_id, session)
+
+		return JSONResponse(
+		{
+			'access_token': session.get('access_token'),
+			'refresh_token': session.get('refresh_token')
+		}, status_code = HTTP_201_CREATED)
+
+	return JSONResponse(
+	{
+		'message': translator.get('something_went_wrong', 'facefusion.apis')
+	}, status_code = HTTP_401_UNAUTHORIZED)
+
+
+async def get_session(request : Request) -> JSONResponse:
+	access_token = extract_access_token(request.scope)
+
+	if access_token:
+		session_id = session_manager.find_session_id(access_token)
+
+		if session_id:
+			session = session_manager.get_session(session_id)
+
+			return JSONResponse(
+			{
+				'access_token': session.get('access_token'),
+				'refresh_token': session.get('refresh_token'),
+				'created_at': session.get('created_at').isoformat(),
+				'expires_at': session.get('expires_at').isoformat()
+			}, status_code = HTTP_200_OK)
+
+	return JSONResponse(
+	{
+		'message': translator.get('something_went_wrong', 'facefusion.apis')
+	}, status_code = HTTP_401_UNAUTHORIZED)
+
+
+async def refresh_session(request : Request) -> JSONResponse:
+	body = await request.json()
+
+	for session_id, session in session_manager.SESSIONS.items():
+		if session.get('refresh_token') == body.get('refresh_token'):
+			__session__ = session_manager.create_session()
+			session_manager.set_session(session_id, __session__)
+
+			return JSONResponse(
+			{
+				'access_token': __session__.get('access_token'),
+				'refresh_token': __session__.get('refresh_token')
+			}, status_code = HTTP_200_OK)
+
+	return JSONResponse(
+	{
+		'message': translator.get('something_went_wrong', 'facefusion.apis')
+	}, status_code = HTTP_401_UNAUTHORIZED)
+
+
+async def destroy_session(request : Request) -> JSONResponse:
+	access_token = extract_access_token(request.scope)
+
+	if access_token:
+		session_id = session_manager.find_session_id(access_token)
+
+		if session_id:
+			session_manager.clear_session(session_id)
+
+			return JSONResponse(
+			{
+				'message': translator.get('ok', 'facefusion.apis')
+			}, status_code = HTTP_200_OK)
+
+	return JSONResponse(
+	{
+		'message': translator.get('something_went_wrong', 'facefusion.apis')
+	}, status_code = HTTP_401_UNAUTHORIZED)
+
+
+def create_session_guard(app : ASGIApp) -> ASGIApp:
+	async def middleware(scope : Scope, receive : Receive, send : Send) -> None:
+		access_token = extract_access_token(scope)
+
+		if access_token:
+			session_id = session_manager.find_session_id(access_token)
+
+			if session_id:
+				if session_manager.validate_session(session_id):
+					session_context.set_session_id(session_id)
+					return await app(scope, receive, send)
+
+				response = JSONResponse(
+				{
+					'message': translator.get('invalid_access_token', 'facefusion.apis')
+				}, status_code = HTTP_426_UPGRADE_REQUIRED)
+
+				return await response(scope, receive, send)
+
+		response = JSONResponse(
+		{
+			'message': translator.get('invalid_access_token', 'facefusion.apis')
+		}, status_code = HTTP_401_UNAUTHORIZED)
+
+		return await response(scope, receive, send)
+
+	return middleware
+
+
+def extract_access_token(scope : Scope) -> Optional[Token]:
+	if scope.get('type') == 'http':
+		auth_header = Headers(scope = scope).get('Authorization')
+
+		if auth_header:
+			auth_prefix, _, access_token = auth_header.partition(' ')
+
+			if auth_prefix.lower() == 'bearer' and access_token:
+				return access_token
+
+	if scope.get('type') == 'websocket':
+		subprotocol = get_sec_websocket_protocol(scope)
+
+		if subprotocol:
+			protocol_prefix, _, access_token = subprotocol.partition('.')
+
+			if protocol_prefix == 'access_token' and access_token:
+				return access_token
+
+	return None

facefusion/apis/endpoints/state.py

@@ -0,0 +1,80 @@
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND
+
+from facefusion import args_store, session_manager, state_manager, translator
+from facefusion.apis import asset_store
+from facefusion.apis.endpoints.session import extract_access_token
+
+
+async def get_state(request : Request) -> JSONResponse:
+	api_args = args_store.filter_api_args(state_manager.get_state())
+	return JSONResponse(state_manager.collect_state(api_args), status_code = HTTP_200_OK)
+
+
+async def set_state(request : Request) -> JSONResponse:
+	action = request.query_params.get('action')
+	asset_type = request.query_params.get('asset_type')
+
+	if action == 'select' and asset_type == 'source':
+		return await select_source(request)
+
+	if action == 'select' and asset_type == 'target':
+		return await select_target(request)
+
+	body = await request.json()
+	api_args = args_store.get_api_args()
+
+	for key, value in body.items():
+		if key in api_args:
+			state_manager.set_item(key, value)
+
+	__api_args__ = args_store.filter_api_args(state_manager.get_state())
+	return JSONResponse(state_manager.collect_state(__api_args__), status_code = HTTP_200_OK)
+
+
+async def select_source(request : Request) -> JSONResponse:
+	body = await request.json()
+	asset_ids = body.get('asset_ids')
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+
+	if isinstance(asset_ids, list) and session_id:
+		source_paths = []
+
+		for asset_id in asset_ids:
+			asset = asset_store.get_asset(session_id, asset_id)
+
+			if asset:
+				source_paths.append(asset.get('path'))
+
+		state_manager.set_item('source_paths', source_paths)
+
+		__api_args__ = args_store.filter_api_args(state_manager.get_state())
+		return JSONResponse(state_manager.collect_state(__api_args__), status_code = HTTP_200_OK)
+
+	return JSONResponse(
+	{
+		'message': translator.get('source_asset_not_found', 'facefusion.apis')
+	}, status_code = HTTP_404_NOT_FOUND)
+
+
+async def select_target(request : Request) -> JSONResponse:
+	body = await request.json()
+	asset_id = body.get('asset_id')
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+
+	if isinstance(asset_id, str) and session_id:
+		asset = asset_store.get_asset(session_id, asset_id)
+
+		if asset:
+			state_manager.set_item('target_path', asset.get('path'))
+
+			__api_args__ = args_store.filter_api_args(state_manager.get_state())
+			return JSONResponse(state_manager.collect_state(__api_args__), status_code = HTTP_200_OK)
+
+	return JSONResponse(
+	{
+		'message': translator.get('target_asset_not_found', 'facefusion.apis')
+	}, status_code = HTTP_404_NOT_FOUND)

facefusion/apis/endpoints/stream.py

@@ -0,0 +1,52 @@
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_404_NOT_FOUND
+from starlette.websockets import WebSocket, WebSocketState
+
+from facefusion import session_context, session_manager
+from facefusion.apis.api_helper import get_sec_websocket_protocol
+from facefusion.apis.endpoints.session import extract_access_token
+from facefusion.apis.stream_manager import destroy_stream, process_image, process_video
+
+
+async def websocket_stream(websocket : WebSocket) -> None:
+	subprotocol = get_sec_websocket_protocol(websocket.scope)
+	access_token = extract_access_token(websocket.scope)
+	session_id = session_manager.find_session_id(access_token)
+	session_context.set_session_id(session_id)
+
+	await websocket.accept(subprotocol = subprotocol)
+	await process_image(websocket)
+
+	if websocket.client_state == WebSocketState.CONNECTED:
+		await websocket.close()
+
+
+async def post_stream(request : Request) -> Response:
+	headers =\
+	{
+		'Location': request.url_for('delete_stream').path
+	}
+	content_type = request.headers.get('content-type')
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+	session_context.set_session_id(session_id)
+
+	if session_id and content_type == 'application/sdp':
+		sdp_offer = await request.body()
+		sdp_answer = process_video(session_id, sdp_offer.decode())
+
+		if sdp_answer:
+			return Response(sdp_answer, status_code = HTTP_201_CREATED, media_type = 'application/sdp', headers = headers)
+
+	return Response(status_code = HTTP_404_NOT_FOUND)
+
+
+async def delete_stream(request : Request) -> Response:
+	access_token = extract_access_token(request.scope)
+	session_id = session_manager.find_session_id(access_token)
+
+	if session_id and destroy_stream(session_id):
+		return Response(status_code = HTTP_200_OK)
+
+	return Response(status_code = HTTP_404_NOT_FOUND)

facefusion/apis/locales.py

@@ -0,0 +1,14 @@
+from facefusion.types import Locales
+
+LOCALES : Locales =\
+{
+	'en':
+	{
+		'ok': 'ok',
+		'something_went_wrong': 'something went wrong',
+		'invalid_access_token': 'invalid access token',
+		'invalid_refresh_token': 'invalid refresh token',
+		'source_asset_not_found': 'source asset not found',
+		'target_asset_not_found': 'target asset not found'
+	}
+}

facefusion/apis/locals.py

@@ -0,0 +1,12 @@
+from facefusion.types import Locals
+
+LOCALS : Locals =\
+{
+	'en':
+	{
+		'ok': 'ok',
+		'something_went_wrong': 'something went wrong',
+		'invalid_access_token': 'invalid access token',
+		'invalid_refresh_token': 'invalid refresh token'
+	}
+}

facefusion/apis/metrics.py

@@ -0,0 +1,76 @@
+import asyncio
+from functools import lru_cache
+from typing import Any, Dict, Optional, cast
+
+from starlette.datastructures import Headers
+from starlette.websockets import WebSocket, WebSocketDisconnect
+
+from facefusion import state_manager
+from facefusion.execution import detect_execution_devices
+from facefusion.system import get_cpu_info, get_disk_info, get_load_average, get_network_info
+from facefusion.system import get_operating_system_info, get_python_info, get_ram_info, get_temperature_info
+from facefusion.types import SystemInfo
+
+
+@lru_cache(maxsize = 1)
+def get_cached_static_system_info() -> Dict[str, Any]:
+	return\
+	{
+		'operating_system': get_operating_system_info(),
+		'python': get_python_info()
+	}
+
+
+@lru_cache(maxsize = 1)
+def get_cached_semi_static_system_info(temp_path : Optional[str]) -> Dict[str, Any]:
+	return\
+	{
+		'disk': get_disk_info(temp_path),
+		'network': get_network_info()
+	}
+
+
+def get_optimized_system_info(temp_path : Optional[str] = None) -> SystemInfo:
+	static_data = get_cached_static_system_info()
+	semi_static_data = get_cached_semi_static_system_info(temp_path)
+	dynamic_data : Dict[str, Any] =\
+	{
+		'cpu': get_cpu_info(),
+		'ram': get_ram_info(),
+		'temperatures': get_temperature_info(),
+		'load_average': get_load_average()
+	}
+
+	return cast(SystemInfo, {**static_data, **semi_static_data, **dynamic_data})
+
+
+async def websocket_metrics(websocket : WebSocket) -> None:
+	subprotocol = get_requested_subprotocol(websocket)
+	await websocket.accept(subprotocol = subprotocol)
+
+	try:
+		while True:
+			temp_path = state_manager.get_temp_path()
+			execution_devices = detect_execution_devices()
+			system_info = get_optimized_system_info(temp_path)
+			metrics =\
+			{
+				'devices': execution_devices,
+				'system': system_info
+			}
+			await websocket.send_json(metrics)
+			await asyncio.sleep(2)
+
+	except (WebSocketDisconnect, Exception):
+		pass
+
+
+def get_requested_subprotocol(websocket : WebSocket) -> Optional[str]:
+	headers = Headers(scope = websocket.scope)
+	protocol_header = headers.get('Sec-WebSocket-Protocol')
+
+	if protocol_header:
+		protocol, _, _ = protocol_header.partition(',')
+		return protocol.strip()
+
+	return None

facefusion/apis/remote.py

@@ -0,0 +1,384 @@
+import os
+import tempfile
+from typing import Any, Dict, List
+
+import httpx
+import yt_dlp  # type: ignore
+from gallery_dl import config as gallery_config, extractor as gallery_extractor, job as gallery_job
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_400_BAD_REQUEST, HTTP_500_INTERNAL_SERVER_ERROR
+
+from facefusion import logger
+from facefusion.apis import asset_store
+from facefusion.choices import audio_formats
+from facefusion.session_context import get_session_id
+
+
+def resolve_image_urls(url : str) -> List[str]:
+	gallery_config.load()
+	image_urls : List[str] = []
+
+	try:
+		for extractor_instance in gallery_extractor.extractors():
+			if extractor_instance.pattern and extractor_instance.pattern.match(url):
+				logger.info(f'Detected gallery URL using extractor: {extractor_instance.__name__}', __name__)
+				extractor_obj = extractor_instance.from_url(url)
+
+				if extractor_obj:
+					for msg in extractor_obj:
+						if isinstance(msg, tuple) and len(msg) >= 2:
+							msg_type = msg[0]
+							if msg_type == 5:
+								image_data = msg[1]
+								image_url = image_data.get('url')
+								if image_url:
+									image_urls.append(image_url)
+				break
+
+		if not image_urls:
+			logger.info('Not a gallery URL, treating as direct image URL', __name__)
+			image_urls = [url]
+
+	except Exception as e:
+		logger.error(f'Failed to extract image URLs: {e}', __name__)
+		logger.info('Falling back to treating as direct image URL', __name__)
+		image_urls = [url]
+
+	return image_urls
+
+
+def download_images_from_url(url : str, asset_type : str) -> List[str]:
+	gallery_config.load()
+	temp_dir = tempfile.gettempdir()
+	asset_ids : List[str] = []
+
+	is_gallery = False
+	for extractor_instance in gallery_extractor.extractors():
+		if extractor_instance.pattern and extractor_instance.pattern.match(url):
+			logger.info(f'Detected gallery URL using extractor: {extractor_instance.__name__}', __name__)
+			is_gallery = True
+
+			output_dir = os.path.join(temp_dir, f'facefusion_gallery_{os.urandom(8).hex()}')
+			os.makedirs(output_dir, exist_ok = True)
+
+			gallery_config.set((), 'base-directory', output_dir)
+			gallery_config.set((), 'skip', False)
+
+			gdl_job = gallery_job.DownloadJob(url)
+			gdl_job.run()
+
+			session_id = get_session_id()
+			for root, dirs, files in os.walk(output_dir):
+				for filename in files:
+					file_path = os.path.join(root, filename)
+					asset = asset_store.create_asset(session_id, asset_type, file_path)
+					if asset:
+						asset_ids.append(asset.get('id'))
+						logger.info(f'Registered image as asset {asset.get("id")}', __name__)
+
+			break
+
+	if not is_gallery:
+		logger.info('Not a gallery URL, treating as direct image URL', __name__)
+		with httpx.stream('GET', url, timeout = 30, follow_redirects = True) as response:
+			response.raise_for_status()
+
+			content_type = response.headers.get('content-type', '')
+			if not content_type.startswith('image/'):
+				raise ValueError(f'URL does not point to an image. Content-Type: {content_type}')
+
+			file_extension = None
+			if 'image/jpeg' in content_type or 'image/jpg' in content_type:
+				file_extension = '.jpg'
+			if 'image/png' in content_type:
+				file_extension = '.png'
+			if 'image/gif' in content_type:
+				file_extension = '.gif'
+			if 'image/webp' in content_type:
+				file_extension = '.webp'
+
+			if not file_extension:
+				url_path = url.split('?')[0]
+				if '.' in url_path:
+					file_extension = '.' + url_path.split('.')[-1].lower()
+				else:
+					file_extension = '.jpg'
+
+			filename = f'facefusion_image_{os.urandom(8).hex()}{file_extension}'
+			file_path = os.path.join(temp_dir, filename)
+
+			with open(file_path, 'wb') as f:
+				for chunk in response.iter_bytes(chunk_size = 8192):
+					f.write(chunk)
+
+		session_id = get_session_id()
+		asset = asset_store.create_asset(session_id, asset_type, file_path)
+		if asset:
+			asset_ids.append(asset.get('id'))
+			logger.info(f'Downloaded and registered image as asset {asset.get("id")}', __name__)
+
+	return asset_ids
+
+
+def download_audio_from_url(url : str, asset_type : str) -> List[str]:
+	temp_dir = tempfile.gettempdir()
+	asset_ids : List[str] = []
+
+	# Extract file extension from URL
+	url_path = url.split('?')[0]
+	url_extension = os.path.splitext(url_path)[1].lstrip('.')
+
+	# Validate extension against supported audio formats
+	if url_extension not in audio_formats:
+		raise ValueError(f'Unsupported audio format: {url_extension}. Supported formats: {", ".join(audio_formats)}')
+
+	logger.info(f'Downloading audio from URL with extension: {url_extension}', __name__)
+	with httpx.stream('GET', url, timeout = 30, follow_redirects = True) as response:
+		response.raise_for_status()
+
+		filename = f'facefusion_audio_{os.urandom(8).hex()}.{url_extension}'
+		file_path = os.path.join(temp_dir, filename)
+
+		with open(file_path, 'wb') as f:
+			for chunk in response.iter_bytes(chunk_size = 8192):
+				f.write(chunk)
+
+	session_id = get_session_id()
+	asset = asset_store.create_asset(session_id, asset_type, file_path)
+	if asset:
+		asset_ids.append(asset.get('id'))
+		logger.info(f'Downloaded and registered audio as asset {asset.get("id")}', __name__)
+
+	return asset_ids
+
+
+async def remote(request : Request) -> JSONResponse:
+	body = await request.json()
+	url = body.get('url')
+	action = request.query_params.get('action')
+	media_type = request.query_params.get('media_type', 'video')
+	asset_type = request.query_params.get('asset_type', 'target')
+
+	if not action:
+		return JSONResponse({'message': 'No action provided. Must be "resolve" or "download"'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if action not in ['resolve', 'download']:
+		return JSONResponse({'message': 'Invalid action. Must be "resolve" or "download"'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if media_type not in ['image', 'video', 'audio']:
+		return JSONResponse({'message': 'Invalid media_type. Must be "image", "video", or "audio"'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if asset_type not in ['source', 'target']:
+		return JSONResponse({'message': 'Invalid asset_type. Must be "source" or "target"'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if not url:
+		return JSONResponse({'message': 'No URL provided'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if not isinstance(url, str):
+		return JSONResponse({'message': 'URL must be a string'}, status_code = HTTP_400_BAD_REQUEST)
+
+	url = url.strip()
+
+	if not url.startswith('http://') and not url.startswith('https://'):
+		return JSONResponse({'message': 'URL must start with http:// or https://'}, status_code = HTTP_400_BAD_REQUEST)
+
+	quality = body.get('quality', '720p')
+
+	if quality not in ['360p', '480p', '720p', '1080p']:
+		return JSONResponse({'message': 'Quality must be 360p, 480p, 720p, or 1080p'}, status_code = HTTP_400_BAD_REQUEST)
+
+	if action == 'resolve':
+		if media_type == 'image':
+			image_urls = resolve_image_urls(url)
+			logger.info(f'Resolved {len(image_urls)} image URL(s)', __name__)
+
+			response_data =\
+			{
+				'message': 'Image URL(s) resolved successfully',
+				'image_urls': image_urls,
+				'count': len(image_urls)
+			}
+
+			return JSONResponse(response_data, status_code = HTTP_200_OK)
+
+		quality_map =\
+		{
+			'360p': 'bestvideo[height<=360][ext=mp4]+bestaudio[ext=m4a]/best[height<=360]',
+			'480p': 'bestvideo[height<=480][ext=mp4]+bestaudio[ext=m4a]/best[height<=480]',
+			'720p': 'bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720]',
+			'1080p': 'bestvideo[height<=1080][ext=mp4]+bestaudio[ext=m4a]/best[height<=1080]'
+		}
+
+		ydl_opts : Dict[str, Any] =\
+		{
+			'format': quality_map[quality],
+			'quiet': True,
+			'no_warnings': True
+		}
+
+		logger.info(f'Extracting stream URL from {url} at {quality}', __name__)
+
+		try:
+			ydl = yt_dlp.YoutubeDL(ydl_opts)
+			info = ydl.extract_info(url, download = False)
+		except Exception as e:
+			logger.error(f'Failed to extract video information: {e}', __name__)
+			return JSONResponse({'message': f'Failed to extract video information: {str(e)}'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+		if not info:
+			logger.error('Failed to extract video information', __name__)
+			return JSONResponse({'message': 'Failed to extract video information'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+		stream_url = info.get('url')
+
+		if not stream_url:
+			if 'requested_formats' in info and len(info['requested_formats']) > 0:
+				stream_url = info['requested_formats'][0].get('url')
+				logger.info('Using URL from requested_formats (video track)', __name__)
+			elif 'formats' in info and len(info['formats']) > 0:
+				for fmt in reversed(info['formats']):
+					if fmt.get('url') and fmt.get('vcodec') != 'none':
+						stream_url = fmt['url']
+						logger.info(f'Using URL from format: {fmt.get("format_id")}', __name__)
+						break
+
+		if not stream_url:
+			logger.error('No stream URL found in any format', __name__)
+			logger.debug(f'Available keys in info: {list(info.keys())}', __name__)
+			return JSONResponse({'message': 'No stream URL found'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+		audio_url = None
+		if 'requested_formats' in info and len(info['requested_formats']) > 1:
+			audio_url = info['requested_formats'][1].get('url')
+			if audio_url:
+				logger.info('Found separate audio track URL', __name__)
+
+		duration = info.get('duration')
+		fps = info.get('fps')
+		width = info.get('width')
+		height = info.get('height')
+
+		total_frames = None
+		if duration and fps:
+			total_frames = int(duration * fps)
+			logger.info(f'Calculated total frames: {total_frames} ({duration}s * {fps} fps)', __name__)
+
+		logger.info('Stream URL extracted successfully', __name__)
+
+		response_data =\
+		{
+			'message': 'Stream URL resolved successfully',
+			'stream_url': stream_url,
+			'audio_url': audio_url,
+			'duration': duration,
+			'fps': fps,
+			'total_frames': total_frames,
+			'width': width,
+			'height': height
+		}
+
+		return JSONResponse(response_data, status_code = HTTP_200_OK)
+
+	if action == 'download':
+		if media_type == 'image':
+			try:
+				asset_ids = download_images_from_url(url, asset_type)
+			except ValueError as e:
+				return JSONResponse({'message': str(e)}, status_code = HTTP_400_BAD_REQUEST)
+			except Exception as e:
+				logger.error(f'Failed to download image(s): {e}', __name__)
+				return JSONResponse({'message': f'Failed to download image(s): {str(e)}'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+			response_data =\
+			{
+				'message': f'Downloaded and registered {len(asset_ids)} image(s)',
+				'asset_ids': asset_ids,
+				'count': len(asset_ids)
+			}
+
+			return JSONResponse(response_data, status_code = HTTP_201_CREATED)
+
+		if media_type == 'audio':
+			try:
+				asset_ids = download_audio_from_url(url, asset_type)
+			except ValueError as e:
+				return JSONResponse({'message': str(e)}, status_code = HTTP_400_BAD_REQUEST)
+			except Exception as e:
+				logger.error(f'Failed to download audio: {e}', __name__)
+				return JSONResponse({'message': f'Failed to download audio: {str(e)}'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+			response_data =\
+			{
+				'message': f'Downloaded and registered {len(asset_ids)} audio file(s)',
+				'asset_ids': asset_ids,
+				'count': len(asset_ids)
+			}
+
+			return JSONResponse(response_data, status_code = HTTP_201_CREATED)
+
+		quality_map =\
+		{
+			'360p': 'bestvideo[height<=360][ext=mp4]+bestaudio[ext=m4a]/best[height<=360][ext=mp4]/best[height<=360]',
+			'480p': 'bestvideo[height<=480][ext=mp4]+bestaudio[ext=m4a]/best[height<=480][ext=mp4]/best[height<=480]',
+			'720p': 'bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720][ext=mp4]/best[height<=720]',
+			'1080p': 'bestvideo[height<=1080][ext=mp4]+bestaudio[ext=m4a]/best[height<=1080][ext=mp4]/best[height<=1080]'
+		}
+
+		temp_dir = tempfile.gettempdir()
+		output_path = os.path.join(temp_dir, 'facefusion_remote_%(id)s.%(ext)s')
+
+		download_opts : Dict[str, Any] =\
+		{
+			'format': quality_map[quality],
+			'outtmpl': output_path,
+			'quiet': False,
+			'no_warnings': False
+		}
+
+		logger.info(f'Downloading video from {url} at {quality}', __name__)
+
+		ydl = yt_dlp.YoutubeDL(download_opts)
+		info = ydl.extract_info(url, download = True)
+
+		if not info:
+			logger.error('Failed to download video', __name__)
+			return JSONResponse({'message': 'Failed to download video'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+		downloaded_file = ydl.prepare_filename(info)
+		if not os.path.exists(downloaded_file):
+			logger.error(f'Downloaded file not found: {downloaded_file}', __name__)
+			return JSONResponse({'message': 'Downloaded file not found'}, status_code = HTTP_500_INTERNAL_SERVER_ERROR)
+
+		duration = info.get('duration')
+		fps = info.get('fps')
+		width = info.get('width')
+		height = info.get('height')
+
+		total_frames = None
+		if duration and fps:
+			total_frames = int(duration * fps)
+			logger.info(f'Calculated total frames: {total_frames} ({duration}s * {fps} fps)', __name__)
+
+		session_id = get_session_id()
+		asset = asset_store.create_asset(session_id, asset_type, downloaded_file)
+		asset_id = asset.get('id') if asset else None
+		logger.info(f'Video downloaded and registered as asset {asset_id}', __name__)
+
+		response_data =\
+		{
+			'message': 'Video downloaded and registered as asset',
+			'asset_id': asset_id,
+			'metadata':
+			{
+				'duration': duration,
+				'fps': fps,
+				'total_frames': total_frames,
+				'width': width,
+				'height': height
+			}
+		}
+
+		return JSONResponse(response_data, status_code = HTTP_201_CREATED)
+
+	return JSONResponse({'message': 'Invalid request'}, status_code = HTTP_400_BAD_REQUEST)

facefusion/apis/stream_audio.py

@@ -0,0 +1,71 @@
+import ctypes
+import time
+from functools import partial
+from queue import Queue
+from typing import Optional, Tuple
+
+import numpy
+
+from facefusion import rtc
+from facefusion.apis.stream_event import create_receive_event
+from facefusion.codecs import opus_decoder, opus_encoder
+from facefusion.types import AudioCodec, AudioFrame, OpusDecoder, RtcPeer, RtcPeerAudio
+
+
+def run_audio_encode_loop(rtc_peer : RtcPeer, audio_queue : Queue[Tuple[float, AudioFrame]]) -> None:
+	temp_audio_time, temp_audio_frame = audio_queue.get()
+	audio_encoder = opus_encoder.create(48000, 2)
+	audio_timestamp = 0
+
+	while numpy.any(temp_audio_frame):
+		audio_frame_size = len(temp_audio_frame) // 2
+		audio_buffer = opus_encoder.encode(audio_encoder, temp_audio_frame.tobytes(), audio_frame_size)
+
+		if audio_buffer:
+			rtc.send_audio(rtc_peer, audio_buffer, audio_timestamp)
+			audio_timestamp += audio_frame_size
+
+		temp_audio_time, temp_audio_frame = audio_queue.get()
+
+	opus_encoder.destroy(audio_encoder)
+
+
+def receive_audio_frames(rtc_peer_audio : RtcPeerAudio, audio_queue : Queue[Tuple[float, AudioFrame]]) -> None:
+	audio_track = rtc_peer_audio.get('receiver_track')
+	audio_codec = rtc_peer_audio.get('codec')
+	audio_decoder = create_audio_decoder(audio_codec)
+
+	audio_frame_handler = partial(handle_audio_frame, audio_codec, audio_decoder, audio_queue)
+	receive_event = create_receive_event(audio_track, audio_frame_handler)
+	receive_event.wait()
+
+	empty_audio_frame = numpy.empty(0)
+	audio_queue.put((0.0, empty_audio_frame))
+	destroy_audio_decoder(audio_codec, audio_decoder)
+
+
+def decode_audio_frame(audio_codec : AudioCodec, audio_decoder : OpusDecoder, input_buffer : bytes) -> Optional[bytes]:
+	if audio_codec == 'opus':
+		return opus_decoder.decode(audio_decoder, input_buffer, 960, 2)
+	return None
+
+
+def create_audio_decoder(audio_codec : AudioCodec) -> Optional[OpusDecoder]:
+	if audio_codec == 'opus':
+		return opus_decoder.create(48000, 2)
+	return None
+
+
+def destroy_audio_decoder(audio_codec : AudioCodec, audio_decoder : OpusDecoder) -> None:
+	if audio_codec == 'opus':
+		opus_decoder.destroy(audio_decoder)
+
+
+#todo: Alias Time for float
+def handle_audio_frame(audio_codec : AudioCodec, audio_decoder : OpusDecoder, audio_queue : Queue[Tuple[float, AudioFrame]], track : int, data : ctypes.c_void_p, size : int, info : ctypes.c_void_p, pointer : ctypes.c_void_p) -> None:
+	audio_buffer = ctypes.string_at(data, size)
+	audio_frame = decode_audio_frame(audio_codec, audio_decoder, audio_buffer)
+
+	if audio_frame:
+		temp_audio_frame = numpy.frombuffer(audio_frame, dtype = numpy.float32)
+		audio_queue.put((time.monotonic(), temp_audio_frame))

facefusion/apis/stream_event.py

@@ -0,0 +1,24 @@
+import ctypes
+import threading
+from functools import partial
+
+from facefusion.libraries import datachannel as datachannel_module
+from facefusion.types import FrameHandler
+
+
+def create_receive_event(track : int, frame_handler : FrameHandler) -> threading.Event:
+	datachannel_library = datachannel_module.create_static_library()
+	receive_event = threading.Event()
+
+	frame_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(frame_handler)
+	close_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_void_p)(partial(dispatch_event, receive_event))
+	datachannel_library.rtcSetFrameCallback(track, frame_callback)
+	datachannel_library.rtcSetClosedCallback(track, close_callback)
+	receive_event.frame_callback = frame_callback  # type: ignore[attr-defined]
+	receive_event.close_callback = close_callback  # type: ignore[attr-defined]
+
+	return receive_event
+
+
+def dispatch_event(event : threading.Event, track : int, pointer : ctypes.c_void_p) -> None:
+	event.set()

facefusion/apis/stream_manager.py

@@ -0,0 +1,137 @@
+import ctypes
+import threading
+from collections.abc import AsyncIterator
+from concurrent.futures import Future, ThreadPoolExecutor
+from queue import Queue
+from typing import Optional, Tuple
+
+import cv2
+import numpy
+from starlette.websockets import WebSocket
+
+from facefusion import rtc, rtc_store, state_manager, streamer
+from facefusion.apis.stream_audio import receive_audio_frames, run_audio_encode_loop
+from facefusion.apis.stream_video import receive_video_frames, run_video_encode_loop
+from facefusion.libraries import datachannel as datachannel_module
+from facefusion.types import AudioCodec, AudioFrame, PeerConnection, Resolution, RtcPeer, RtcPeerAudio, SdpAnswer, SdpOffer, SessionId, VideoCodec, VisionFrame
+
+
+async def process_image(websocket : WebSocket) -> None:
+	capture_vision_frame = await anext(receive_vision_frames(websocket), None)
+
+	if numpy.any(capture_vision_frame):
+		output_vision_frame = streamer.process_stream_frame(capture_vision_frame)
+		is_success, output_frame_buffer = cv2.imencode('.jpg', output_vision_frame)
+
+		if is_success:
+			await websocket.send_bytes(output_frame_buffer.tobytes())
+
+
+async def receive_vision_frames(websocket : WebSocket) -> AsyncIterator[VisionFrame]:
+	websocket_event = await websocket.receive()
+
+	while websocket_event.get('type') == 'websocket.receive':
+		frame_buffer = websocket_event.get('bytes') or bytes()
+		vision_frame = cv2.imdecode(numpy.frombuffer(frame_buffer, numpy.uint8), cv2.IMREAD_COLOR)
+
+		if numpy.any(vision_frame):
+			yield vision_frame
+
+		websocket_event = await websocket.receive()
+
+
+def process_video(session_id : SessionId, sdp_offer : SdpOffer) -> Optional[SdpAnswer]:
+	video_codec : VideoCodec = 'vp8'
+
+	if rtc.get_payload_type(sdp_offer, 'vp9'):
+		video_codec = 'vp9'
+
+	if rtc.get_payload_type(sdp_offer, 'av1'):
+		video_codec = 'av1'
+
+	video_payload_type = rtc.get_payload_type(sdp_offer, video_codec)
+
+	if video_payload_type:
+		peer_connection : PeerConnection = rtc.create_peer_connection()
+		video_receiver_track = rtc.add_video_track(peer_connection, 'recvonly', video_codec, video_payload_type)
+		video_sender_track = rtc.add_video_track(peer_connection, 'sendonly', video_codec, video_payload_type)
+
+		sender_bitrate = ctypes.c_uint(0)
+		receiver_bitrate = ctypes.c_uint(8000)
+		rtc.wire_sender_bitrate(video_sender_track, sender_bitrate)
+
+		audio_codec : AudioCodec = 'opus'
+		audio_payload_type = rtc.get_payload_type(sdp_offer, audio_codec)
+
+		if audio_payload_type:
+			audio_receiver_track = rtc.add_audio_track(peer_connection, 'recvonly', audio_codec, audio_payload_type)
+			audio_sender_track = rtc.add_audio_track(peer_connection, 'sendonly', audio_codec, audio_payload_type)
+
+		rtc.set_remote_description(peer_connection, sdp_offer)
+		sdp_answer = rtc.create_sdp_answer(peer_connection)
+
+		if sdp_answer:
+			rtc_peer : RtcPeer =\
+			{
+				'peer_connection': peer_connection,
+				'video':
+				{
+					'sender_track': video_sender_track,
+					'receiver_track': video_receiver_track,
+					'codec': video_codec
+				},
+				'sender_bitrate': sender_bitrate,
+				'receiver_bitrate': receiver_bitrate
+			}
+
+			if audio_payload_type:
+				rtc_peer['audio'] = RtcPeerAudio(
+					sender_track = audio_sender_track,
+					receiver_track = audio_receiver_track,
+					codec = audio_codec
+				)
+
+			rtc_store.init_peers(session_id)
+			rtc_store.get_peers(session_id).append(rtc_peer)
+
+			threading.Thread(target = run_peer_loop, args = (session_id, rtc_peer), daemon = True).start()
+
+			return sdp_answer
+
+		datachannel_module.create_static_library().rtcDeletePeerConnection(peer_connection)
+
+	return None
+
+
+def run_peer_loop(session_id : SessionId, rtc_peer : RtcPeer) -> None:
+	execution_thread_count = state_manager.get_item('execution_thread_count')
+	#todo: is bytes, Resolution not a XXXPointer type
+	video_queue : Queue[Tuple[float, Future[Tuple[bytes, Resolution]]]] = Queue(maxsize = execution_thread_count)
+	audio_queue : Queue[Tuple[float, AudioFrame]] = Queue(maxsize = execution_thread_count * 10)
+	video_executor = ThreadPoolExecutor(max_workers = execution_thread_count)
+
+	video_receiver_thread = threading.Thread(target = receive_video_frames, args = (rtc_peer.get('video'), video_queue, video_executor), daemon = True)
+	video_encoder_thread = threading.Thread(target = run_video_encode_loop, args = (rtc_peer, video_queue), daemon = True)
+	video_receiver_thread.start()
+	video_encoder_thread.start()
+
+	if rtc_peer.get('audio'):
+		audio_receiver_thread = threading.Thread(target = receive_audio_frames, args = (rtc_peer.get('audio'), audio_queue), daemon = True)
+		audio_encoder_thread = threading.Thread(target = run_audio_encode_loop, args = (rtc_peer, audio_queue), daemon = True)
+		audio_receiver_thread.start()
+		audio_encoder_thread.start()
+		audio_receiver_thread.join()
+		audio_encoder_thread.join()
+
+	video_receiver_thread.join()
+	video_encoder_thread.join()
+	video_executor.shutdown(wait = True)
+	rtc_store.delete_peers(session_id)
+
+
+def destroy_stream(session_id : SessionId) -> bool:
+	if rtc_store.has_peers(session_id):
+		rtc_store.delete_peers(session_id)
+		return not rtc_store.has_peers(session_id)
+
+	return False

facefusion/apis/stream_video.py

@@ -0,0 +1,204 @@
+import ctypes
+import time
+from concurrent.futures import Future, ThreadPoolExecutor
+from functools import partial
+from queue import Queue
+from typing import Optional, Tuple
+
+import cv2
+import numpy
+
+from facefusion import rtc, streamer
+from facefusion.apis.stream_event import create_receive_event
+from facefusion.codecs import aom_decoder, aom_encoder, vpx_decoder, vpx_encoder
+from facefusion.types import AomDecoder, AomEncoder, AomPointer, BitRate, Resolution, RtcPeer, RtcPeerVideo, VideoCodec, VisionFrame, VpxDecoder, VpxEncoder, VpxPointer
+
+
+def run_video_encode_loop(rtc_peer : RtcPeer, video_queue : Queue[Tuple[float, Future[Tuple[bytes, Resolution]]]]) -> None:
+	video_codec = rtc_peer.get('video').get('codec')
+	video_time, video_future = video_queue.get()
+	video_buffer, video_resolution = video_future.result()
+
+	if video_buffer:
+		temp_resolution : Resolution = video_resolution
+		temp_bitrate : BitRate = 8000
+		video_encoder = create_video_encoder(video_codec, temp_resolution, temp_bitrate)
+		temp_video_time = video_time
+		frame_index = 0
+
+		while video_buffer:
+			encode_start = time.monotonic()
+			sender_bitrate = calculate_sender_bitrate(rtc_peer, temp_bitrate)
+
+			if video_resolution[0] - temp_resolution[0] or video_resolution[1] - temp_resolution[1]:
+				temp_resolution = video_resolution
+				update_video_encoder_resolution(video_codec, video_encoder, temp_resolution)
+
+			if sender_bitrate - temp_bitrate:
+				temp_bitrate = sender_bitrate
+				update_video_encoder_bitrate(video_codec, video_encoder, temp_bitrate)
+
+			__video_buffer__ = encode_video_frame(video_codec, video_encoder, video_buffer, temp_resolution, frame_index)
+
+			if __video_buffer__:
+				video_timestamp = int(video_time * 90000)
+				rtc.send_video(rtc_peer, __video_buffer__, video_timestamp)
+
+			encode_time = time.monotonic() - encode_start
+			frame_interval = video_time - temp_video_time
+			temp_video_time = video_time
+
+			receiver_bitrate = calculate_receiver_bitrate(rtc_peer, encode_time, frame_interval)
+			rtc.adapt_receiver_bitrate(rtc_peer, receiver_bitrate)
+			frame_index += 1
+
+			video_time, video_future = video_queue.get()
+			video_buffer, video_resolution = video_future.result()
+
+		destroy_video_encoder(video_codec, video_encoder)
+		rtc.clear_bitrate(rtc_peer)
+
+
+def receive_video_frames(rtc_peer_video : RtcPeerVideo, video_queue : Queue[Tuple[float, Future[Tuple[bytes, Resolution]]]], video_executor : ThreadPoolExecutor) -> None:
+	video_track = rtc_peer_video.get('receiver_track')
+	video_codec = rtc_peer_video.get('codec')
+	video_decoder = create_video_decoder(video_codec)
+
+	video_frame_handler = partial(handle_video_frame, video_codec, video_decoder, video_queue, video_executor)
+	receive_event = create_receive_event(video_track, video_frame_handler)
+	receive_event.wait()
+
+	empty_future : Future[Tuple[bytes, Resolution]] = Future()
+	empty_future.set_result((bytes(), (0, 0)))
+	video_queue.put((0.0, empty_future))
+	destroy_video_decoder(video_codec, video_decoder)
+
+
+def process_video_frame(input_vision_frame : VisionFrame) -> Tuple[bytes, Resolution]:
+	output_vision_frame = streamer.process_stream_frame(input_vision_frame)
+	output_resolution : Resolution = (output_vision_frame.shape[1], output_vision_frame.shape[0])
+	output_buffer = cv2.cvtColor(output_vision_frame, cv2.COLOR_BGR2YUV_I420).tobytes()
+	return output_buffer, output_resolution
+
+
+def calculate_receiver_bitrate(rtc_peer : RtcPeer, encode_time : float, frame_interval : float) -> BitRate:
+	min_bitrate : BitRate = 500
+	max_bitrate : BitRate = 8000
+	bitrate : BitRate = rtc_peer.get('receiver_bitrate').value
+
+	if frame_interval > 0:
+		scale = frame_interval / encode_time
+		bitrate = int(bitrate * scale)
+		bitrate = max(min_bitrate, min(max_bitrate, bitrate))
+
+	return bitrate
+
+
+#todo: does not feel final as this is an clamp and not calculate
+def calculate_sender_bitrate(rtc_peer : RtcPeer, bitrate : BitRate) -> BitRate:
+	min_bitrate : BitRate = 500
+	max_bitrate : BitRate = 8000
+	peer_bitrate : BitRate = rtc_peer.get('sender_bitrate').value
+
+	if peer_bitrate > 0:
+		bitrate = max(min_bitrate, min(max_bitrate, peer_bitrate))
+
+	return bitrate
+
+
+def decode_video_frame(video_codec : VideoCodec, video_decoder : VpxDecoder | AomDecoder, input_buffer : bytes) -> Optional[VisionFrame]:
+	if video_codec == 'av1':
+		aom_pointer = aom_decoder.decode(video_decoder, input_buffer)
+
+		if aom_pointer:
+			return normalize_vision_frame(aom_pointer)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		vpx_pointer = vpx_decoder.decode(video_decoder, input_buffer)
+
+		if vpx_pointer:
+			return normalize_vision_frame(vpx_pointer)
+
+	return None
+
+
+def encode_video_frame(video_codec : VideoCodec, video_encoder : VpxEncoder | AomEncoder, input_buffer : bytes, frame_resolution : Resolution, frame_index : int) -> bytes:
+	if video_codec == 'av1':
+		return aom_encoder.encode(video_encoder, input_buffer, frame_resolution, frame_index)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		return vpx_encoder.encode(video_encoder, input_buffer, frame_resolution, frame_index)
+
+	return bytes()
+
+
+def normalize_vision_frame(frame_pointer : AomPointer | VpxPointer) -> VisionFrame:
+	frame_width, frame_height = frame_pointer.get('resolution')
+	vision_frame = numpy.frombuffer(frame_pointer.get('buffer'), dtype = numpy.uint8).reshape((frame_height * 3 // 2, frame_width))
+	return cv2.cvtColor(vision_frame, cv2.COLOR_YUV2BGR_I420)
+
+
+def create_video_decoder(video_codec : VideoCodec) -> Optional[VpxDecoder | AomDecoder]:
+	if video_codec == 'av1':
+		return aom_decoder.create(8)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		return vpx_decoder.create(video_codec, 8)
+
+	return None
+
+
+def create_video_encoder(video_codec : VideoCodec, frame_resolution : Resolution, bitrate : BitRate) -> Optional[VpxEncoder | AomEncoder]:
+	if video_codec == 'av1':
+		return aom_encoder.create(frame_resolution, bitrate, 8, 10)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		return vpx_encoder.create(video_codec, frame_resolution, bitrate, 8, 10)
+
+	return None
+
+
+def destroy_video_decoder(video_codec : VideoCodec, video_decoder : VpxDecoder | AomDecoder) -> None:
+	if video_codec == 'av1':
+		aom_decoder.destroy(video_decoder)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		vpx_decoder.destroy(video_decoder)
+
+
+def destroy_video_encoder(video_codec : VideoCodec, video_encoder : VpxEncoder | AomEncoder) -> None:
+	if video_codec == 'av1':
+		aom_encoder.destroy(video_encoder)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		vpx_encoder.destroy(video_encoder)
+
+
+def update_video_encoder_resolution(video_codec : VideoCodec, video_encoder : VpxEncoder | AomEncoder, frame_resolution : Resolution) -> bool:
+	if video_codec == 'av1':
+		return aom_encoder.update_resolution(video_encoder, frame_resolution)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		return vpx_encoder.update_resolution(video_encoder, frame_resolution)
+
+	return False
+
+
+def update_video_encoder_bitrate(video_codec : VideoCodec, video_encoder : VpxEncoder | AomEncoder, bitrate : BitRate) -> bool:
+	if video_codec == 'av1':
+		return aom_encoder.update_bitrate(video_encoder, bitrate)
+
+	if video_codec in [ 'vp8', 'vp9' ]:
+		return vpx_encoder.update_bitrate(video_encoder, bitrate)
+
+	return False
+
+
+#todo: we can remove the dead args or pass audio buffer
+def handle_video_frame(video_codec : VideoCodec, video_decoder : VpxDecoder | AomDecoder, video_queue : Queue[Tuple[float, Future[Tuple[bytes, Resolution]]]], video_executor : ThreadPoolExecutor, track : int, data : ctypes.c_void_p, size : int, info : ctypes.c_void_p, pointer : ctypes.c_void_p) -> None:
+	video_buffer = ctypes.string_at(data, size)
+	vision_frame = decode_video_frame(video_codec, video_decoder, video_buffer)
+
+	if numpy.any(vision_frame) and video_queue.qsize() < video_queue.maxsize:
+		video_future = video_executor.submit(process_video_frame, vision_frame)
+		video_queue.put((time.monotonic(), video_future))

facefusion/apis/timeline.py

@@ -0,0 +1,207 @@
+import base64
+import subprocess
+from typing import List, Optional
+
+import cv2
+import numpy
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST
+
+from facefusion import logger
+from facefusion.apis import asset_store
+from facefusion.filesystem import is_video
+from facefusion.video_manager import get_video_capture
+from facefusion.vision import fit_contain_frame
+
+
+def extract_frame_at_timestamp(stream_url : str, timestamp : float, width : int, height : int) -> Optional[numpy.ndarray]:
+	ffmpeg_command =\
+	[
+		'ffmpeg',
+		'-user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+		'-ss', str(timestamp),
+		'-i', stream_url,
+		'-vf', f'scale={width}:{height}',
+		'-frames:v', '1',
+		'-f', 'rawvideo',
+		'-pix_fmt', 'bgr24',
+		'-'
+	]
+
+	try:
+		result = subprocess.run(ffmpeg_command, capture_output = True, timeout = 10)
+		if result.returncode == 0 and result.stdout:
+			frame_size = width * height * 3
+			if len(result.stdout) >= frame_size:
+				frame = numpy.frombuffer(result.stdout[:frame_size], dtype = numpy.uint8).reshape((height, width, 3))
+				return frame
+	except Exception as e:
+		logger.debug(f'Failed to extract frame at {timestamp}s: {e}', __name__)
+
+	return None
+
+
+async def get_timeline(request: Request) -> JSONResponse:
+    """
+    Return N preview frames (as base64 JPEGs) from the target video,
+    resized to specified resolution for timeline preview.
+
+    Route: /timeline/{count:int}?target_path=...&is_remote_stream=true&duration=120&fps=30&target_width=1920&target_height=1080&width=160&height=120
+    """
+
+    # Extract and validate requested count
+    try:
+        count = int(request.path_params.get('count', 0))
+    except (TypeError, ValueError):
+        return JSONResponse({'message': 'Invalid count parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    if count <= 0:
+        return JSONResponse({'message': 'Count must be a positive integer'}, status_code=HTTP_400_BAD_REQUEST)
+
+    # Extract and validate preview resolution parameters
+    try:
+        preview_width = int(request.query_params.get('width', 160))
+        preview_height = int(request.query_params.get('height', 120))
+    except (TypeError, ValueError):
+        return JSONResponse({'message': 'Invalid width or height parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    if preview_width <= 0 or preview_height <= 0 or preview_width > 1920 or preview_height > 1080:
+        return JSONResponse({'message': 'Width and height must be between 1 and 1920x1080'}, status_code=HTTP_400_BAD_REQUEST)
+
+    # Extract target_path or asset_id (one is required)
+    target_path = request.query_params.get('target_path')
+    asset_id = request.query_params.get('asset_id')
+
+    # Extract is_remote_stream flag
+    is_remote_stream_param = request.query_params.get('is_remote_stream', 'false').lower()
+    is_remote_stream = is_remote_stream_param in ['true', '1', 'yes']
+
+    # Resolve asset_id to path if provided (for local files)
+    if asset_id and not target_path:
+        from facefusion.session_context import get_session_id
+
+        session_id = get_session_id()
+        asset = asset_store.get_asset(session_id, asset_id)
+        if not asset:
+            return JSONResponse({'message': f'Asset not found: {asset_id}'}, status_code=HTTP_400_BAD_REQUEST)
+
+        target_path = asset.get('path')
+        if not target_path:
+            return JSONResponse({'message': 'Asset has no path'}, status_code=HTTP_400_BAD_REQUEST)
+
+        is_remote_stream = False  # Assets are always local files
+        logger.debug(f'Resolved asset_id {asset_id} to path for timeline preview', __name__)
+
+    # Now check if we have a target_path
+    if not target_path:
+        return JSONResponse({'message': 'Missing required parameter: either target_path or asset_id'}, status_code=HTTP_400_BAD_REQUEST)
+
+    # Extract video metadata (optional for local files, required for remote streams)
+    duration = None
+    fps = None
+    width = 1280
+    height = 720
+
+    if request.query_params.get('duration'):
+        try:
+            duration = float(request.query_params.get('duration'))
+        except (TypeError, ValueError):
+            return JSONResponse({'message': 'Invalid duration parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    if request.query_params.get('fps'):
+        try:
+            fps = float(request.query_params.get('fps'))
+        except (TypeError, ValueError):
+            return JSONResponse({'message': 'Invalid fps parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    if request.query_params.get('target_width'):
+        try:
+            width = int(request.query_params.get('target_width'))
+        except (TypeError, ValueError):
+            return JSONResponse({'message': 'Invalid target_width parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    if request.query_params.get('target_height'):
+        try:
+            height = int(request.query_params.get('target_height'))
+        except (TypeError, ValueError):
+            return JSONResponse({'message': 'Invalid target_height parameter'}, status_code=HTTP_400_BAD_REQUEST)
+
+    previews: List[str] = []
+
+    if is_remote_stream:
+        if not duration or duration <= 0:
+            return JSONResponse({'message': 'Duration not available for remote stream'}, status_code=HTTP_400_BAD_REQUEST)
+
+        frame_total = 0
+        if duration and fps:
+            try:
+                frame_total = int(float(duration) * float(fps))
+            except Exception:
+                frame_total = 0
+
+        sample_count = min(count, frame_total) if frame_total > 0 else count
+        timestamps = list(numpy.linspace(0, float(duration), num=sample_count, endpoint=False))
+
+        logger.info(f'Extracting {sample_count} frames from remote stream using ffmpeg', __name__)
+
+        for timestamp in timestamps:
+            frame = extract_frame_at_timestamp(target_path, timestamp, width, height)
+            if frame is None:
+                logger.warn(f'Failed to extract frame at {timestamp}s', __name__)
+                continue
+
+            thumb_bgr = fit_contain_frame(frame, (preview_width, preview_height))
+            if thumb_bgr.shape[1] != preview_width or thumb_bgr.shape[0] != preview_height:
+                thumb_bgr = cv2.resize(thumb_bgr, (preview_width, preview_height))
+
+            ok_enc, buf = cv2.imencode('.jpg', thumb_bgr, [cv2.IMWRITE_JPEG_QUALITY, 50])
+            if not ok_enc:
+                logger.warn(f'JPEG encode failed for timestamp {timestamp}s', __name__)
+                continue
+
+            b64 = base64.b64encode(buf.tobytes()).decode('ascii')
+            previews.append(b64)
+    else:
+        video_capture = get_video_capture(target_path)
+        if not video_capture or not video_capture.isOpened():
+            logger.error(f'Unable to open video capture for target: {target_path}', __name__)
+            return JSONResponse({'message': 'Unable to open target video'}, status_code=HTTP_400_BAD_REQUEST)
+
+        frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
+
+        if frame_total <= 0 and is_video(target_path):
+            return JSONResponse({'message': 'Could not determine frame count for target video'}, status_code=HTTP_400_BAD_REQUEST)
+
+        sample_count = min(count, frame_total)
+        indices: List[int] = list(numpy.linspace(1, frame_total, num=sample_count, endpoint=True, dtype=int))
+
+        for frame_number in indices:
+            video_capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, frame_number - 1))
+            ok_read, frame = video_capture.read()
+            if not ok_read or frame is None:
+                logger.warn(f'Failed reading frame {frame_number}', __name__)
+                continue
+
+            thumb_bgr = fit_contain_frame(frame, (preview_width, preview_height))
+            if thumb_bgr.shape[1] != preview_width or thumb_bgr.shape[0] != preview_height:
+                thumb_bgr = cv2.resize(thumb_bgr, (preview_width, preview_height))
+
+            ok_enc, buf = cv2.imencode('.jpg', thumb_bgr, [cv2.IMWRITE_JPEG_QUALITY, 50])
+            if not ok_enc:
+                logger.warn(f'JPEG encode failed for frame {frame_number}', __name__)
+                continue
+            b64 = base64.b64encode(buf.tobytes()).decode('ascii')
+            previews.append(b64)
+
+    logger.info(f'Returned {len(previews)}/{sample_count} timeline frames at {preview_width}x{preview_height}', __name__)
+
+    return JSONResponse({
+        'message': 'ok',
+        'count': len(previews),
+        'requested': count,
+        'width': preview_width,
+        'height': preview_height,
+        'format': 'jpeg',
+        'frames': previews
+    }, status_code=HTTP_200_OK)

facefusion/apis/version.py

@@ -0,0 +1,98 @@
+import subprocess
+from functools import lru_cache
+from typing import Optional
+
+from starlette.datastructures import Headers
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.types import ASGIApp, Receive, Scope, Send
+from starlette.websockets import WebSocket
+
+
+@lru_cache(maxsize = 1)
+def get_api_version() -> str:
+	try:
+		result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output = True, text = True, check = True)
+		return result.stdout.strip()
+	except Exception:
+		return 'unknown'
+
+
+def check_version_match(request : Request) -> Optional[JSONResponse]:
+	client_version = request.headers.get('X-API-Version')
+	server_version = get_api_version()
+
+	if not client_version:
+		return JSONResponse({'error': 'Missing X-API-Version header', 'server_version': server_version}, status_code = 400)
+
+	if client_version != server_version:
+		return JSONResponse({'error': 'Version mismatch', 'client_version': client_version, 'server_version': server_version}, status_code = 409)
+
+	return None
+
+
+def check_version_match_websocket(websocket : WebSocket) -> Optional[str]:
+	client_version = websocket.headers.get('X-API-Version')
+	server_version = get_api_version()
+
+	if not client_version:
+		return f'Missing X-API-Version header, server version: {server_version}'
+
+	if client_version != server_version:
+		return f'Version mismatch: client={client_version}, server={server_version}'
+
+	return None
+
+
+async def version_guard_middleware(scope : Scope, receive : Receive, send : Send, app : ASGIApp) -> None:
+	if scope['type'] == 'http':
+		# Skip version check for OPTIONS requests (CORS preflight)
+		if scope.get('method') == 'OPTIONS':
+			await app(scope, receive, send)
+			return
+
+		headers = Headers(scope = scope)
+		client_version = headers.get('X-API-Version')
+		server_version = get_api_version()
+
+		if not client_version:
+			response = JSONResponse({'error': 'Missing X-API-Version header', 'server_version': server_version}, status_code = 400)
+			await response(scope, receive, send)
+			return
+
+		if client_version != server_version:
+			response = JSONResponse({'error': 'Version mismatch', 'client_version': client_version, 'server_version': server_version}, status_code = 409)
+			await response(scope, receive, send)
+			return
+
+	if scope['type'] == 'websocket':
+		headers = Headers(scope = scope)
+		client_version = headers.get('X-API-Version')
+
+		# For WebSocket connections, also check subprotocols since browsers can't set custom headers
+		if not client_version:
+			protocol_header = headers.get('Sec-WebSocket-Protocol')
+			if protocol_header:
+				# Parse subprotocols to find api_version
+				protocols = [p.strip() for p in protocol_header.split(',')]
+				for protocol in protocols:
+					if protocol.startswith('api_version.'):
+						client_version = protocol.split('.', 1)[1]
+						break
+
+		server_version = get_api_version()
+
+		if not client_version or client_version != server_version:
+			websocket = WebSocket(scope, receive = receive, send = send)
+			reason = f'Missing X-API-Version header, server version: {server_version}' if not client_version else f'Version mismatch: client={client_version}, server={server_version}'
+			await websocket.close(code = 1008, reason = reason)
+			return
+
+	await app(scope, receive, send)
+
+
+def create_version_guard(app : ASGIApp) -> ASGIApp:
+	async def version_guard_app(scope : Scope, receive : Receive, send : Send) -> None:
+		await version_guard_middleware(scope, receive, send, app)
+
+	return version_guard_app

facefusion/app_context.py

@@ -10,7 +10,7 @@ def detect_app_context() -> AppContext:
 	while frame:
 		if os.path.join('facefusion', 'jobs') in frame.f_code.co_filename:
 			return 'cli'
-		if os.path.join('facefusion', 'uis') in frame.f_code.co_filename:
-			return 'ui'
+		if os.path.join('facefusion', 'apis') in frame.f_code.co_filename:
+			return 'api'
 		frame = frame.f_back
 	return 'cli'

facefusion/args_helper.py

@@ -1,47 +1,15 @@
-from facefusion import state_manager
-from facefusion.filesystem import get_file_name, is_image, is_video, resolve_file_paths
-from facefusion.jobs import job_store
-from facefusion.normalizer import normalize_fps, normalize_padding
+from facefusion.filesystem import get_file_name, is_video, resolve_file_paths
+from facefusion.normalizer import normalize_fps, normalize_space
 from facefusion.processors.core import get_processors_modules
 from facefusion.types import ApplyStateItem, Args
-from facefusion.vision import create_image_resolutions, create_video_resolutions, detect_image_resolution, detect_video_fps, detect_video_resolution, pack_resolution
-
-
-def reduce_step_args(args : Args) -> Args:
-	step_args =\
-	{
-		key: args[key] for key in args if key in job_store.get_step_keys()
-	}
-	return step_args
-
-
-def reduce_job_args(args : Args) -> Args:
-	job_args =\
-	{
-		key: args[key] for key in args if key in job_store.get_job_keys()
-	}
-	return job_args
-
-
-def collect_step_args() -> Args:
-	step_args =\
-	{
-		key: state_manager.get_item(key) for key in job_store.get_step_keys() #type:ignore[arg-type]
-	}
-	return step_args
-
-
-def collect_job_args() -> Args:
-	job_args =\
-	{
-		key: state_manager.get_item(key) for key in job_store.get_job_keys() #type:ignore[arg-type]
-	}
-	return job_args
+from facefusion.vision import detect_video_fps
 
 
 def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
 	# general
 	apply_state_item('command', args.get('command'))
+	# workflow
+	apply_state_item('workflow', args.get('workflow'))
 	# paths
 	apply_state_item('temp_path', args.get('temp_path'))
 	apply_state_item('jobs_path', args.get('jobs_path'))
@@ -55,6 +23,7 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
 	# face detector
 	apply_state_item('face_detector_model', args.get('face_detector_model'))
 	apply_state_item('face_detector_size', args.get('face_detector_size'))
+	apply_state_item('face_detector_margin', normalize_space(args.get('face_detector_margin')))
 	apply_state_item('face_detector_angles', args.get('face_detector_angles'))
 	apply_state_item('face_detector_score', args.get('face_detector_score'))
 	# face landmarker
@@ -77,34 +46,23 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
 	apply_state_item('face_mask_areas', args.get('face_mask_areas'))
 	apply_state_item('face_mask_regions', args.get('face_mask_regions'))
 	apply_state_item('face_mask_blur', args.get('face_mask_blur'))
-	apply_state_item('face_mask_padding', normalize_padding(args.get('face_mask_padding')))
+	apply_state_item('face_mask_padding', normalize_space(args.get('face_mask_padding')))
+	# voice extractor
+	apply_state_item('voice_extractor_model', args.get('voice_extractor_model'))
 	# frame extraction
 	apply_state_item('trim_frame_start', args.get('trim_frame_start'))
 	apply_state_item('trim_frame_end', args.get('trim_frame_end'))
 	apply_state_item('temp_frame_format', args.get('temp_frame_format'))
-	apply_state_item('keep_temp', args.get('keep_temp'))
 	# output creation
 	apply_state_item('output_image_quality', args.get('output_image_quality'))
-	if is_image(args.get('target_path')):
-		output_image_resolution = detect_image_resolution(args.get('target_path'))
-		output_image_resolutions = create_image_resolutions(output_image_resolution)
-		if args.get('output_image_resolution') in output_image_resolutions:
-			apply_state_item('output_image_resolution', args.get('output_image_resolution'))
-		else:
-			apply_state_item('output_image_resolution', pack_resolution(output_image_resolution))
+	apply_state_item('output_image_scale', args.get('output_image_scale'))
 	apply_state_item('output_audio_encoder', args.get('output_audio_encoder'))
 	apply_state_item('output_audio_quality', args.get('output_audio_quality'))
 	apply_state_item('output_audio_volume', args.get('output_audio_volume'))
 	apply_state_item('output_video_encoder', args.get('output_video_encoder'))
 	apply_state_item('output_video_preset', args.get('output_video_preset'))
 	apply_state_item('output_video_quality', args.get('output_video_quality'))
-	if is_video(args.get('target_path')):
-		output_video_resolution = detect_video_resolution(args.get('target_path'))
-		output_video_resolutions = create_video_resolutions(output_video_resolution)
-		if args.get('output_video_resolution') in output_video_resolutions:
-			apply_state_item('output_video_resolution', args.get('output_video_resolution'))
-		else:
-			apply_state_item('output_video_resolution', pack_resolution(output_video_resolution))
+	apply_state_item('output_video_scale', args.get('output_video_scale'))
 	if args.get('output_video_fps') or is_video(args.get('target_path')):
 		output_video_fps = normalize_fps(args.get('output_video_fps')) or detect_video_fps(args.get('target_path'))
 		apply_state_item('output_video_fps', output_video_fps)
@@ -113,24 +71,22 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
 	apply_state_item('processors', args.get('processors'))
 	for processor_module in get_processors_modules(available_processors):
 		processor_module.apply_args(args, apply_state_item)
-	# uis
-	apply_state_item('open_browser', args.get('open_browser'))
-	apply_state_item('ui_layouts', args.get('ui_layouts'))
-	apply_state_item('ui_workflow', args.get('ui_workflow'))
 	# execution
-	apply_state_item('execution_device_id', args.get('execution_device_id'))
+	apply_state_item('execution_device_ids', args.get('execution_device_ids'))
 	apply_state_item('execution_providers', args.get('execution_providers'))
 	apply_state_item('execution_thread_count', args.get('execution_thread_count'))
-	apply_state_item('execution_queue_count', args.get('execution_queue_count'))
 	# download
 	apply_state_item('download_providers', args.get('download_providers'))
 	apply_state_item('download_scope', args.get('download_scope'))
 	# benchmark
+	apply_state_item('benchmark_mode', args.get('benchmark_mode'))
 	apply_state_item('benchmark_resolutions', args.get('benchmark_resolutions'))
 	apply_state_item('benchmark_cycle_count', args.get('benchmark_cycle_count'))
+	# api
+	apply_state_item('api_host', args.get('api_host'))
+	apply_state_item('api_port', args.get('api_port'))
 	# memory
 	apply_state_item('video_memory_strategy', args.get('video_memory_strategy'))
-	apply_state_item('system_memory_limit', args.get('system_memory_limit'))
 	# misc
 	apply_state_item('log_level', args.get('log_level'))
 	apply_state_item('halt_on_error', args.get('halt_on_error'))

facefusion/args_store.py

@@ -0,0 +1,66 @@
+from typing import List
+
+from facefusion.types import Args, ArgsStore, Scope
+
+
+ARGS_STORE : ArgsStore =\
+{
+	'api': [],
+	'cli': [],
+	'sys': []
+}
+
+
+def get_api_args() -> List[str]:
+	return ARGS_STORE.get('api')
+
+
+def get_sys_args() -> List[str]:
+	return ARGS_STORE.get('sys')
+
+
+def get_cli_args() -> List[str]:
+	return ARGS_STORE.get('cli')
+
+
+def register_args(keys : List[str], scopes : List[Scope]) -> None:
+	for key in keys:
+		for scope in scopes:
+			if scope == 'api':
+				ARGS_STORE['api'].append(key)
+			if scope == 'cli':
+				ARGS_STORE['cli'].append(key)
+			if scope == 'sys':
+				ARGS_STORE['sys'].append(key)
+
+
+def filter_api_args(args : Args) -> Args:
+	api_args =\
+	{
+		key: args.get(key) for key in args if key in get_api_args() #type:ignore[literal-required]
+	}
+	return api_args
+
+
+def filter_sys_args(args : Args) -> Args:
+	sys_args =\
+	{
+		key: args.get(key) for key in args if key in get_sys_args() #type:ignore[literal-required]
+	}
+	return sys_args
+
+
+def filter_cli_args(args : Args) -> Args:
+	cli_args =\
+	{
+		key: args.get(key) for key in args if key in get_cli_args() #type:ignore[literal-required]
+	}
+	return cli_args
+
+
+def filter_step_args(args : Args) -> Args:
+	step_args =\
+	{
+		key: args.get(key) for key in args if key in get_cli_args() and key not in get_sys_args() #type:ignore[literal-required]
+	}
+	return step_args

facefusion/asset_store.py

@@ -0,0 +1,111 @@
+import os
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, TypeAlias
+
+from facefusion import filesystem, state_manager
+from facefusion.session_context import get_session_id
+
+AssetRegistry : TypeAlias = Dict[str, Dict[str, Any]]
+
+
+def get_asset_registry() -> AssetRegistry:
+	registry = state_manager.get_item('asset_registry')
+	if not registry:
+		registry = {}
+		state_manager.set_item('asset_registry', registry)
+	return registry
+
+
+def register(asset_type : str, file_path : str, filename : str = None, metadata : Optional[Dict[str, Any]] = None) -> str:
+	if asset_type not in ['source', 'target', 'output']:
+		raise ValueError(f"Invalid asset_type: {asset_type}. Must be 'source', 'target', or 'output'")
+
+	asset_id = str(uuid.uuid4())
+	session_id = get_session_id()
+
+	if not session_id:
+		raise ValueError("No active session - cannot register asset without session_id")
+
+	if not filename:
+		filename = os.path.basename(file_path)
+
+	file_size = os.path.getsize(file_path)
+	file_format = filesystem.get_file_format(file_path)
+	media_type = None
+
+	if filesystem.is_image(file_path):
+		media_type = 'image'
+	if filesystem.is_video(file_path):
+		media_type = 'video'
+	if filesystem.is_audio(file_path):
+		media_type = 'audio'
+
+	asset_data =\
+	{
+		'id': asset_id,
+		'session_id': session_id,
+		'type': asset_type,
+		'media_type': media_type,
+		'format': file_format,
+		'path': file_path,
+		'filename': filename,
+		'size': file_size,
+		'created_at': datetime.now(timezone.utc).isoformat()
+	}
+
+	if metadata:
+		asset_data['metadata'] = metadata
+
+	registry = get_asset_registry()
+	registry[asset_id] = asset_data
+	state_manager.set_item('asset_registry', registry)
+
+	return asset_id
+
+
+def get_asset(asset_id : str) -> Optional[Dict[str, Any]]:
+	registry = get_asset_registry()
+	return registry.get(asset_id)
+
+
+def list_assets(asset_type : Optional[str] = None) -> List[Dict[str, Any]]:
+	registry = get_asset_registry()
+	session_id = get_session_id()
+
+	assets = list(registry.values())
+
+	if session_id:
+		assets = [a for a in assets if a.get('session_id') == session_id]
+
+	if asset_type:
+		if asset_type not in ['source', 'target', 'output']:
+			raise ValueError(f"Invalid asset_type: {asset_type}")
+		assets = [a for a in assets if a.get('type') == asset_type]
+
+	return assets
+
+
+def delete_asset(asset_id : str) -> bool:
+	registry = get_asset_registry()
+	asset = registry.get(asset_id)
+
+	if not asset:
+		return False
+
+	file_path = asset.get('path')
+	if file_path and os.path.exists(file_path):
+		os.remove(file_path)
+
+	del registry[asset_id]
+	state_manager.set_item('asset_registry', registry)
+
+	return True
+
+
+def cleanup_session_assets(session_id : str) -> None:
+	registry = get_asset_registry()
+	assets_to_delete = [aid for aid, asset in registry.items() if asset.get('session_id') == session_id]
+
+	for asset_id in assets_to_delete:
+		delete_asset(asset_id)

facefusion/audio.py

@@ -1,5 +1,6 @@
+import math
 from functools import lru_cache
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Tuple
 
 import numpy
 import scipy
@@ -7,11 +8,12 @@ from numpy.typing import NDArray
 
 from facefusion.ffmpeg import read_audio_buffer
 from facefusion.filesystem import is_audio
-from facefusion.types import Audio, AudioFrame, Fps, Mel, MelFilterBank, Spectrogram
+from facefusion.media_helper import restrict_trim_frame
+from facefusion.types import Audio, AudioFrame, Duration, Fps, Mel, MelFilterBank, Spectrogram
 from facefusion.voice_extractor import batch_extract_voice
 
 
-@lru_cache()
+@lru_cache(maxsize = 64)
 def read_static_audio(audio_path : str, fps : Fps) -> Optional[List[AudioFrame]]:
 	return read_audio(audio_path, fps)
 
@@ -31,7 +33,7 @@ def read_audio(audio_path : str, fps : Fps) -> Optional[List[AudioFrame]]:
 	return None
 
 
-@lru_cache()
+@lru_cache(maxsize = 64)
 def read_static_voice(audio_path : str, fps : Fps) -> Optional[List[AudioFrame]]:
 	return read_voice(audio_path, fps)
 
@@ -118,12 +120,12 @@ def convert_mel_to_hertz(mel : Mel) -> NDArray[Any]:
 
 def create_mel_filter_bank() -> MelFilterBank:
 	audio_sample_rate = 16000
-	audio_min_frequency = 55.0
-	audio_max_frequency = 7600.0
+	audio_frequency_min = 55.0
+	audio_frequency_max = 7600.0
 	mel_filter_total = 80
 	mel_bin_total = 800
 	mel_filter_bank = numpy.zeros((mel_filter_total, mel_bin_total // 2 + 1))
-	mel_frequency_range = numpy.linspace(convert_hertz_to_mel(audio_min_frequency), convert_hertz_to_mel(audio_max_frequency), mel_filter_total + 2)
+	mel_frequency_range = numpy.linspace(convert_hertz_to_mel(audio_frequency_min), convert_hertz_to_mel(audio_frequency_max), mel_filter_total + 2)
 	indices = numpy.floor((mel_bin_total + 1) * convert_mel_to_hertz(mel_frequency_range) / audio_sample_rate).astype(numpy.int16)
 
 	for index in range(mel_filter_total):
@@ -141,3 +143,29 @@ def create_spectrogram(audio : Audio) -> Spectrogram:
 	spectrogram = scipy.signal.stft(audio, nperseg = mel_bin_total, nfft = mel_bin_total, noverlap = mel_bin_overlap)[2]
 	spectrogram = numpy.dot(mel_filter_bank, numpy.abs(spectrogram))
 	return spectrogram
+
+
+def count_audio_frame_total(audio_path : str, fps : Fps) -> int:
+	audio_duration = detect_audio_duration(audio_path)
+	if audio_duration > 0:
+		return math.ceil(audio_duration * fps)
+	return 0
+
+
+def detect_audio_duration(audio_path : str) -> Duration:
+	audio_sample_rate = 48000
+	audio_sample_size = 16
+	audio_channel_total = 2
+
+	if is_audio(audio_path):
+		audio_buffer = read_audio_buffer(audio_path, audio_sample_rate, audio_sample_size, audio_channel_total)
+		if audio_buffer:
+			audio = numpy.frombuffer(audio_buffer, dtype = numpy.int16).reshape(-1, audio_channel_total)
+			audio_duration = len(audio) / audio_sample_rate
+			return audio_duration
+	return 0
+
+
+def restrict_trim_audio_frame(audio_path : str, fps : Fps, trim_frame_start : Optional[int], trim_frame_end : Optional[int]) -> Tuple[int, int]:
+	audio_frame_total = count_audio_frame_total(audio_path, fps)
+	return restrict_trim_frame(audio_frame_total, trim_frame_start, trim_frame_end)

facefusion/benchmarker.py

@@ -3,15 +3,16 @@ import os
 import statistics
 import tempfile
 from time import perf_counter
-from typing import Generator, List
+from typing import Iterator, List
 
 import facefusion.choices
-from facefusion import core, state_manager
+from facefusion import content_analyser, core, state_manager
 from facefusion.cli_helper import render_table
 from facefusion.download import conditional_download, resolve_download_url
+from facefusion.face_store import clear_static_faces
 from facefusion.filesystem import get_file_extension
 from facefusion.types import BenchmarkCycleSet
-from facefusion.vision import count_video_frame_total, detect_video_fps, detect_video_resolution, pack_resolution
+from facefusion.vision import count_video_frame_total, detect_video_fps
 
 
 def pre_check() -> bool:
@@ -30,7 +31,7 @@ def pre_check() -> bool:
 	return True
 
 
-def run() -> Generator[List[BenchmarkCycleSet], None, None]:
+def run() -> Iterator[List[BenchmarkCycleSet]]:
 	benchmark_resolutions = state_manager.get_item('benchmark_resolutions')
 	benchmark_cycle_count = state_manager.get_item('benchmark_cycle_count')
 
@@ -42,11 +43,11 @@ def run() -> Generator[List[BenchmarkCycleSet], None, None]:
 	state_manager.init_item('video_memory_strategy', 'tolerant')
 
 	benchmarks = []
-	target_paths = [facefusion.choices.benchmark_set.get(benchmark_resolution) for benchmark_resolution in benchmark_resolutions if benchmark_resolution in facefusion.choices.benchmark_set]
+	target_paths = [ facefusion.choices.benchmark_set.get(benchmark_resolution) for benchmark_resolution in benchmark_resolutions if benchmark_resolution in facefusion.choices.benchmark_set ]
 
 	for target_path in target_paths:
-		state_manager.set_item('target_path', target_path)
-		state_manager.set_item('output_path', suggest_output_path(state_manager.get_item('target_path')))
+		state_manager.init_item('target_path', target_path)
+		state_manager.init_item('output_path', suggest_output_path(state_manager.get_item('target_path')))
 		benchmarks.append(cycle(benchmark_cycle_count))
 		yield benchmarks
 
@@ -54,13 +55,17 @@ def run() -> Generator[List[BenchmarkCycleSet], None, None]:
 def cycle(cycle_count : int) -> BenchmarkCycleSet:
 	process_times = []
 	video_frame_total = count_video_frame_total(state_manager.get_item('target_path'))
-	output_video_resolution = detect_video_resolution(state_manager.get_item('target_path'))
-	state_manager.set_item('output_video_resolution', pack_resolution(output_video_resolution))
-	state_manager.set_item('output_video_fps', detect_video_fps(state_manager.get_item('target_path')))
+	state_manager.init_item('output_video_fps', detect_video_fps(state_manager.get_item('target_path')))
 
-	core.conditional_process()
+	if state_manager.get_item('benchmark_mode') == 'warm':
+		core.conditional_process()
 
 	for index in range(cycle_count):
+		if state_manager.get_item('benchmark_mode') == 'cold':
+			content_analyser.analyse_image.cache_clear()
+			content_analyser.analyse_video.cache_clear()
+			clear_static_faces()
+
 		start_time = perf_counter()
 		core.conditional_process()
 		end_time = perf_counter()

facefusion/camera_manager.py

@@ -0,0 +1,53 @@
+from typing import List
+
+import cv2
+
+from facefusion.types import CameraPoolSet
+
+CAMERA_POOL_SET : CameraPoolSet =\
+{
+	'capture': {}
+}
+
+
+def get_local_camera_capture(camera_id : int) -> cv2.VideoCapture:
+	camera_key = str(camera_id)
+
+	if camera_key not in CAMERA_POOL_SET.get('capture'):
+		camera_capture = cv2.VideoCapture(camera_id)
+
+		if camera_capture.isOpened():
+			CAMERA_POOL_SET['capture'][camera_key] = camera_capture
+
+	return CAMERA_POOL_SET.get('capture').get(camera_key)
+
+
+def get_remote_camera_capture(camera_url : str) -> cv2.VideoCapture:
+	if camera_url not in CAMERA_POOL_SET.get('capture'):
+		camera_capture = cv2.VideoCapture(camera_url)
+
+		if camera_capture.isOpened():
+			CAMERA_POOL_SET['capture'][camera_url] = camera_capture
+
+	return CAMERA_POOL_SET.get('capture').get(camera_url)
+
+
+def clear_camera_pool() -> None:
+	for camera_capture in CAMERA_POOL_SET.get('capture').values():
+		camera_capture.release()
+
+	CAMERA_POOL_SET['capture'].clear()
+
+
+def detect_local_camera_ids(id_start : int, id_end : int) -> List[int]:
+	local_camera_ids = []
+
+	for camera_id in range(id_start, id_end):
+		cv2.setLogLevel(0)
+		camera_capture = get_local_camera_capture(camera_id)
+		cv2.setLogLevel(3)
+
+		if camera_capture and camera_capture.isOpened():
+			local_camera_ids.append(camera_id)
+
+	return local_camera_ids

facefusion/choices.py

@@ -2,14 +2,15 @@ import logging
 from typing import List, Sequence
 
 from facefusion.common_helper import create_float_range, create_int_range
-from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, BenchmarkResolution, BenchmarkSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorMode, FaceSelectorOrder, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, UiWorkflow, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, WebcamMode
+from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, BenchmarkMode, BenchmarkResolution, BenchmarkSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorMode, FaceSelectorOrder, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, VoiceExtractorModel, WorkFlow
 
 face_detector_set : FaceDetectorSet =\
 {
 	'many': [ '640x640' ],
 	'retinaface': [ '160x160', '320x320', '480x480', '512x512', '640x640' ],
 	'scrfd': [ '160x160', '320x320', '480x480', '512x512', '640x640' ],
-	'yolo_face': [ '640x640' ]
+	'yolo_face': [ '640x640' ],
+	'yunet': [ '640x640' ]
 }
 face_detector_models : List[FaceDetectorModel] = list(face_detector_set.keys())
 face_landmarker_models : List[FaceLandmarkerModel] = [ 'many', '2dfan4', 'peppa_wutz' ]
@@ -17,7 +18,7 @@ face_selector_modes : List[FaceSelectorMode] = [ 'many', 'one', 'reference' ]
 face_selector_orders : List[FaceSelectorOrder] = [ 'left-right', 'right-left', 'top-bottom', 'bottom-top', 'small-large', 'large-small', 'best-worst', 'worst-best' ]
 face_selector_genders : List[Gender] = [ 'female', 'male' ]
 face_selector_races : List[Race] = [ 'white', 'black', 'latino', 'asian', 'indian', 'arabic' ]
-face_occluder_models : List[FaceOccluderModel] = [ 'xseg_1', 'xseg_2', 'xseg_3' ]
+face_occluder_models : List[FaceOccluderModel] = [ 'many', 'xseg_1', 'xseg_2', 'xseg_3' ]
 face_parser_models : List[FaceParserModel] = [ 'bisenet_resnet_18', 'bisenet_resnet_34' ]
 face_mask_types : List[FaceMaskType] = [ 'box', 'occlusion', 'area', 'region' ]
 face_mask_area_set : FaceMaskAreaSet =\
@@ -42,6 +43,10 @@ face_mask_region_set : FaceMaskRegionSet =\
 face_mask_areas : List[FaceMaskArea] = list(face_mask_area_set.keys())
 face_mask_regions : List[FaceMaskRegion] = list(face_mask_region_set.keys())
 
+voice_extractor_models : List[VoiceExtractorModel] = [ 'kim_vocal_1', 'kim_vocal_2', 'uvr_mdxnet' ]
+
+workflows : List[WorkFlow] = [ 'auto', 'audio-to-image', 'image-to-image', 'image-to-video' ]
+
 audio_type_set : AudioTypeSet =\
 {
 	'flac': 'audio/flac',
@@ -65,8 +70,11 @@ video_type_set : VideoTypeSet =\
 	'm4v': 'video/mp4',
 	'mkv': 'video/x-matroska',
 	'mp4': 'video/mp4',
+	'mpeg': 'video/mpeg',
 	'mov': 'video/quicktime',
-	'webm': 'video/webm'
+	'mxf': 'application/mxf',
+	'webm': 'video/webm',
+	'wmv': 'video/x-ms-wmv'
 }
 audio_formats : List[AudioFormat] = list(audio_type_set.keys())
 image_formats : List[ImageFormat] = list(image_type_set.keys())
@@ -76,15 +84,13 @@ temp_frame_formats : List[TempFrameFormat] = [ 'bmp', 'jpeg', 'png', 'tiff' ]
 output_encoder_set : EncoderSet =\
 {
 	'audio': [ 'flac', 'aac', 'libmp3lame', 'libopus', 'libvorbis', 'pcm_s16le', 'pcm_s32le' ],
-	'video': [ 'libx264', 'libx265', 'libvpx-vp9', 'h264_nvenc', 'hevc_nvenc', 'h264_amf', 'hevc_amf', 'h264_qsv', 'hevc_qsv', 'h264_videotoolbox', 'hevc_videotoolbox', 'rawvideo' ]
+	'video': [ 'libx264', 'libx264rgb', 'libx265', 'libvpx-vp9', 'h264_nvenc', 'hevc_nvenc', 'h264_amf', 'hevc_amf', 'h264_qsv', 'hevc_qsv', 'h264_videotoolbox', 'hevc_videotoolbox', 'rawvideo' ]
 }
 output_audio_encoders : List[AudioEncoder] = output_encoder_set.get('audio')
 output_video_encoders : List[VideoEncoder] = output_encoder_set.get('video')
 output_video_presets : List[VideoPreset] = [ 'ultrafast', 'superfast', 'veryfast', 'faster', 'fast', 'medium', 'slow', 'slower', 'veryslow' ]
 
-image_template_sizes : List[float] = [ 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4 ]
-video_template_sizes : List[int] = [ 240, 360, 480, 540, 720, 1080, 1440, 2160, 4320 ]
-
+benchmark_modes : List[BenchmarkMode] = [ 'warm', 'cold' ]
 benchmark_set : BenchmarkSet =\
 {
 	'240p': '.assets/examples/target-240p.mp4',
@@ -97,15 +103,13 @@ benchmark_set : BenchmarkSet =\
 }
 benchmark_resolutions : List[BenchmarkResolution] = list(benchmark_set.keys())
 
-webcam_modes : List[WebcamMode] = [ 'inline', 'udp', 'v4l2' ]
-webcam_resolutions : List[str] = [ '320x240', '640x480', '800x600', '1024x768', '1280x720', '1280x960', '1920x1080', '2560x1440', '3840x2160' ]
-
 execution_provider_set : ExecutionProviderSet =\
 {
 	'cuda': 'CUDAExecutionProvider',
 	'tensorrt': 'TensorrtExecutionProvider',
 	'directml': 'DmlExecutionProvider',
 	'rocm': 'ROCMExecutionProvider',
+	'migraphx': 'MIGraphXExecutionProvider',
 	'openvino': 'OpenVINOExecutionProvider',
 	'coreml': 'CoreMLExecutionProvider',
 	'cpu': 'CPUExecutionProvider'
@@ -145,13 +149,11 @@ log_level_set : LogLevelSet =\
 }
 log_levels : List[LogLevel] = list(log_level_set.keys())
 
-ui_workflows : List[UiWorkflow] = [ 'instant_runner', 'job_runner', 'job_manager' ]
 job_statuses : List[JobStatus] = [ 'drafted', 'queued', 'completed', 'failed' ]
 
 benchmark_cycle_count_range : Sequence[int] = create_int_range(1, 10, 1)
 execution_thread_count_range : Sequence[int] = create_int_range(1, 32, 1)
-execution_queue_count_range : Sequence[int] = create_int_range(1, 4, 1)
-system_memory_limit_range : Sequence[int] = create_int_range(0, 128, 4)
+face_detector_margin_range : Sequence[int] = create_int_range(0, 100, 1)
 face_detector_angles : Sequence[Angle] = create_int_range(0, 270, 90)
 face_detector_score_range : Sequence[Score] = create_float_range(0.0, 1.0, 0.05)
 face_landmarker_score_range : Sequence[Score] = create_float_range(0.0, 1.0, 0.05)
@@ -160,6 +162,8 @@ face_mask_padding_range : Sequence[int] = create_int_range(0, 100, 1)
 face_selector_age_range : Sequence[int] = create_int_range(0, 100, 1)
 reference_face_distance_range : Sequence[float] = create_float_range(0.0, 1.0, 0.05)
 output_image_quality_range : Sequence[int] = create_int_range(0, 100, 1)
+output_image_scale_range : Sequence[float] = create_float_range(0.25, 8.0, 0.25)
 output_audio_quality_range : Sequence[int] = create_int_range(0, 100, 1)
 output_audio_volume_range : Sequence[int] = create_int_range(0, 100, 1)
 output_video_quality_range : Sequence[int] = create_int_range(0, 100, 1)
+output_video_scale_range : Sequence[float] = create_float_range(0.25, 8.0, 0.25)

facefusion/cli_helper.py

@@ -1,10 +1,10 @@
-from typing import Tuple
+from typing import List, Tuple
 
 from facefusion.logger import get_package_logger
-from facefusion.types import TableContents, TableHeaders
+from facefusion.types import TableContent, TableHeader
 
 
-def render_table(headers : TableHeaders, contents : TableContents) -> None:
+def render_table(headers : List[TableHeader], contents : List[List[TableContent]]) -> None:
 	package_logger = get_package_logger()
 	table_column, table_separator = create_table_parts(headers, contents)
 
@@ -19,7 +19,7 @@ def render_table(headers : TableHeaders, contents : TableContents) -> None:
 	package_logger.critical(table_separator)
 
 
-def create_table_parts(headers : TableHeaders, contents : TableContents) -> Tuple[str, str]:
+def create_table_parts(headers : List[TableHeader], contents : List[List[TableContent]]) -> Tuple[str, str]:
 	column_parts = []
 	separator_parts = []
 	widths = [ len(header) for header in headers ]

facefusion/codecs/aom_decoder.py

@@ -0,0 +1,70 @@
+import ctypes
+import struct
+from typing import Optional
+
+from facefusion.libraries import aom as aom_module
+from facefusion.types import AomDecoder, AomPointer
+
+
+def create(thread_count : int) -> Optional[AomDecoder]:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		aom_decoder = ctypes.create_string_buffer(128)
+		aom_codec = ctypes.c_void_p.in_dll(aom_library, 'aom_codec_av1_dx_algo')
+		config_buffer = ctypes.create_string_buffer(128)
+
+		struct.pack_into('I', config_buffer, 0, thread_count)
+		struct.pack_into('I', config_buffer, 12, 1)
+
+		if aom_library.aom_codec_dec_init_ver(aom_decoder, ctypes.byref(aom_codec), config_buffer, 0, 22) == 0:
+			return aom_decoder
+
+	return None
+
+
+def decode(aom_decoder : AomDecoder, input_buffer : bytes) -> Optional[AomPointer]:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library and input_buffer:
+		input_total = len(input_buffer)
+		temp_buffer = ctypes.create_string_buffer(input_buffer)
+
+		if aom_library.aom_codec_decode(aom_decoder, temp_buffer, input_total, None) == 0:
+			address = aom_library.aom_codec_get_frame(aom_decoder, ctypes.byref(ctypes.c_void_p(0)))
+
+			if address:
+				frame_width = ctypes.c_uint.from_address(address + 28).value & ~1
+				frame_height = ctypes.c_uint.from_address(address + 32).value & ~1
+
+				return AomPointer(
+					buffer = collect(address, frame_width, frame_height),
+					resolution = (frame_width, frame_height)
+				)
+
+	return None
+
+
+def collect(address : int, frame_width : int, frame_height : int) -> bytes:
+	output_parts = []
+
+	for index in range(3):
+		plane_pointer = ctypes.c_void_p.from_address(address + 64 + index * 8).value
+		stride = ctypes.c_int.from_address(address + 88 + index * 4).value
+		plane_width = frame_width >> (index > 0)
+		plane_height = frame_height >> (index > 0)
+
+		if stride == plane_width:
+			output_parts.append(ctypes.string_at(plane_pointer, plane_width * plane_height))
+		else:
+			for row in range(plane_height):
+				output_parts.append(ctypes.string_at(plane_pointer + row * stride, plane_width))
+
+	return bytes().join(output_parts)
+
+
+def destroy(aom_decoder : AomDecoder) -> None:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		aom_library.aom_codec_destroy(aom_decoder)

facefusion/codecs/aom_encoder.py

@@ -0,0 +1,94 @@
+import ctypes
+import struct
+from typing import Optional
+
+from facefusion.libraries import aom as aom_module
+from facefusion.types import AomEncoder, BitRate, Resolution
+
+
+def create(frame_resolution : Resolution, bitrate : BitRate, thread_count : int, cpu_count : int) -> Optional[AomEncoder]:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		aom_encoder = ctypes.create_string_buffer(1152)
+		aom_codec = ctypes.c_void_p.in_dll(aom_library, 'aom_codec_av1_cx_algo')
+
+		config_buffer = ctypes.create_string_buffer(1024)
+
+		if aom_library.aom_codec_enc_config_default(ctypes.byref(aom_codec), config_buffer, 1) == 0:
+			struct.pack_into('I', config_buffer, 4, thread_count)
+			struct.pack_into('I', config_buffer, 12, frame_resolution[0])
+			struct.pack_into('I', config_buffer, 16, frame_resolution[1])
+			struct.pack_into('I', config_buffer, 136, bitrate)
+			struct.pack_into('I', config_buffer, 192, 30)
+
+			if aom_library.aom_codec_enc_init_ver(aom_encoder, ctypes.byref(aom_codec), config_buffer, 0, 25) == 0:
+				aom_library.aom_codec_control(aom_encoder, 13, ctypes.c_int(cpu_count))
+				aom_library.aom_codec_control(aom_encoder, 75, ctypes.c_int(1))
+				aom_library.aom_codec_control(aom_encoder, 106, ctypes.c_int(0))
+				aom_library.aom_codec_control(aom_encoder, 122, ctypes.c_int(0))
+				aom_library.aom_codec_control(aom_encoder, 123, ctypes.c_int(0))
+				ctypes.memmove(ctypes.addressof(aom_encoder) + 128, config_buffer, 1024)
+				return aom_encoder
+
+	return None
+
+
+def encode(aom_encoder : AomEncoder, input_buffer : bytes, frame_resolution : Resolution, frame_index : int) -> bytes:
+	aom_library = aom_module.create_static_library()
+	output_buffer = bytes()
+
+	if aom_library:
+		temp_buffer = ctypes.create_string_buffer(256)
+		encode_buffer = ctypes.create_string_buffer(input_buffer)
+
+		if aom_library.aom_img_wrap(temp_buffer, 0x102, frame_resolution[0], frame_resolution[1], 1, encode_buffer) and aom_library.aom_codec_encode(aom_encoder, temp_buffer, frame_index, 1, 0, 1) == 0:
+			output_buffer = collect(aom_encoder)
+
+	return output_buffer
+
+
+def collect(aom_encoder : AomEncoder) -> bytes:
+	aom_library = aom_module.create_static_library()
+	output_parts = []
+
+	packet_cursor = ctypes.c_void_p(0)
+	packet = aom_library.aom_codec_get_cx_data(aom_encoder, ctypes.byref(packet_cursor))
+
+	while packet:
+		if ctypes.c_int.from_address(packet).value == 0:
+			buffer_pointer = ctypes.c_void_p.from_address(packet + 8).value
+			buffer_size = ctypes.c_size_t.from_address(packet + 16).value
+			output_parts.append(ctypes.string_at(buffer_pointer, buffer_size))
+
+		packet = aom_library.aom_codec_get_cx_data(aom_encoder, ctypes.byref(packet_cursor))
+
+	return bytes().join(output_parts)
+
+
+def update_resolution(aom_encoder : AomEncoder, frame_resolution : Resolution) -> bool:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		struct.pack_into('I', aom_encoder, 128 + 12, frame_resolution[0])
+		struct.pack_into('I', aom_encoder, 128 + 16, frame_resolution[1])
+		return aom_library.aom_codec_enc_config_set(aom_encoder, ctypes.cast(ctypes.addressof(aom_encoder) + 128, ctypes.c_void_p)) == 0
+
+	return False
+
+
+def update_bitrate(aom_encoder : AomEncoder, bitrate : BitRate) -> bool:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		struct.pack_into('I', aom_encoder, 128 + 136, bitrate)
+		return aom_library.aom_codec_enc_config_set(aom_encoder, ctypes.cast(ctypes.addressof(aom_encoder) + 128, ctypes.c_void_p)) == 0
+
+	return False
+
+
+def destroy(aom_encoder : AomEncoder) -> None:
+	aom_library = aom_module.create_static_library()
+
+	if aom_library:
+		aom_library.aom_codec_destroy(aom_encoder)

facefusion/codecs/opus_decoder.py

@@ -0,0 +1,36 @@
+import ctypes
+from typing import Optional
+
+from facefusion.libraries import opus as opus_module
+from facefusion.types import OpusDecoder
+
+
+def create(sample_rate : int, channel_total : int) -> Optional[OpusDecoder]:
+	opus_library = opus_module.create_static_library()
+
+	if opus_library:
+		return opus_library.opus_decoder_create(sample_rate, channel_total, ctypes.byref(ctypes.c_int(0)))
+
+	return None
+
+
+def decode(opus_decoder : OpusDecoder, input_buffer : bytes, frame_size : int, channel_total : int) -> bytes:
+	opus_library = opus_module.create_static_library()
+	output_buffer = bytes()
+
+	if opus_library:
+		input_total = len(input_buffer)
+		decode_buffer = (ctypes.c_float * (frame_size * channel_total))()
+		decode_length = opus_library.opus_decode_float(opus_decoder, input_buffer, input_total, decode_buffer, frame_size, 0)
+
+		if decode_length:
+			output_buffer = ctypes.string_at(ctypes.addressof(decode_buffer), decode_length * channel_total * ctypes.sizeof(ctypes.c_float))
+
+	return output_buffer
+
+
+def destroy(opus_decoder : OpusDecoder) -> None:
+	opus_library = opus_module.create_static_library()
+
+	if opus_library:
+		opus_library.opus_decoder_destroy(opus_decoder)

facefusion/codecs/opus_encoder.py

@@ -0,0 +1,36 @@
+import ctypes
+from typing import Optional
+
+from facefusion.libraries import opus as opus_module
+from facefusion.types import OpusEncoder
+
+
+def create(sample_rate : int, channel_total : int) -> Optional[OpusEncoder]:
+	opus_library = opus_module.create_static_library()
+
+	if opus_library:
+		return opus_library.opus_encoder_create(sample_rate, channel_total, 2049, ctypes.byref(ctypes.c_int(0)))
+
+	return None
+
+
+def encode(opus_encoder : OpusEncoder, input_buffer : bytes, frame_size : int) -> bytes:
+	opus_library = opus_module.create_static_library()
+	output_buffer = bytes()
+
+	if opus_library:
+		temp_buffer = ctypes.create_string_buffer(2048)
+		encode_buffer = ctypes.cast(ctypes.create_string_buffer(input_buffer), ctypes.POINTER(ctypes.c_float))
+		encode_length = opus_library.opus_encode_float(opus_encoder, encode_buffer, frame_size, temp_buffer, 2048)
+
+		if encode_length:
+			output_buffer = temp_buffer.raw[:encode_length]
+
+	return output_buffer
+
+
+def destroy(opus_encoder : OpusEncoder) -> None:
+	opus_library = opus_module.create_static_library()
+
+	if opus_library:
+		opus_library.opus_encoder_destroy(opus_encoder)

facefusion/codecs/vpx_decoder.py

@@ -0,0 +1,74 @@
+import ctypes
+import struct
+from typing import Optional
+
+from facefusion.libraries import vpx as vpx_module
+from facefusion.types import VpxDecoder, VpxPointer, VxpVideoCodec
+
+
+def create(video_codec : VxpVideoCodec, thread_count : int) -> Optional[VpxDecoder]:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		vpx_decoder = ctypes.create_string_buffer(64)
+		vpx_algo = 'vpx_codec_vp8_dx_algo'
+
+		if video_codec == 'vp9':
+			vpx_algo = 'vpx_codec_vp9_dx_algo'
+
+		vpx_codec = ctypes.c_void_p.in_dll(vpx_library, vpx_algo)
+		config_buffer = ctypes.create_string_buffer(128)
+
+		struct.pack_into('I', config_buffer, 0, thread_count)
+
+		if vpx_library.vpx_codec_dec_init_ver(vpx_decoder, ctypes.byref(vpx_codec), config_buffer, 0, 12) == 0:
+			return vpx_decoder
+
+	return None
+
+
+def decode(vpx_decoder : VpxDecoder, input_buffer : bytes) -> Optional[VpxPointer]:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library and input_buffer:
+		input_total = len(input_buffer)
+		temp_buffer = ctypes.create_string_buffer(input_buffer)
+
+		if vpx_library.vpx_codec_decode(vpx_decoder, temp_buffer, input_total, None, 0) == 0:
+			address = vpx_library.vpx_codec_get_frame(vpx_decoder, ctypes.byref(ctypes.c_void_p(0)))
+
+			if address:
+				frame_width = ctypes.c_uint.from_address(address + 24).value & ~1
+				frame_height = ctypes.c_uint.from_address(address + 28).value & ~1
+
+				return VpxPointer(
+					buffer = collect(address, frame_width, frame_height),
+					resolution = (frame_width, frame_height)
+				)
+
+	return None
+
+
+def collect(address : int, frame_width : int, frame_height : int) -> bytes:
+	output_parts = []
+
+	for index in range(3):
+		plane_pointer = ctypes.c_void_p.from_address(address + 48 + index * 8).value
+		stride = ctypes.c_int.from_address(address + 80 + index * 4).value
+		plane_width = frame_width >> (index > 0)
+		plane_height = frame_height >> (index > 0)
+
+		if stride == plane_width:
+			output_parts.append(ctypes.string_at(plane_pointer, plane_width * plane_height))
+		else:
+			for row in range(plane_height):
+				output_parts.append(ctypes.string_at(plane_pointer + row * stride, plane_width))
+
+	return bytes().join(output_parts)
+
+
+def destroy(vpx_decoder : VpxDecoder) -> None:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		vpx_library.vpx_codec_destroy(vpx_decoder)

facefusion/codecs/vpx_encoder.py

@@ -0,0 +1,104 @@
+import ctypes
+import struct
+from typing import Optional
+
+from facefusion.libraries import vpx as vpx_module
+from facefusion.types import BitRate, Resolution, VpxEncoder, VxpVideoCodec
+
+
+def create(video_codec : VxpVideoCodec, frame_resolution : Resolution, bitrate : BitRate, thread_count : int, cpu_count : int) -> Optional[VpxEncoder]:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		vpx_encoder = ctypes.create_string_buffer(640)
+		vpx_algo = 'vpx_codec_vp8_cx_algo'
+
+		if video_codec == 'vp9':
+			vpx_algo = 'vpx_codec_vp9_cx_algo'
+
+		vpx_codec = ctypes.c_void_p.in_dll(vpx_library, vpx_algo)
+
+		config_buffer = ctypes.create_string_buffer(512)
+
+		if vpx_library.vpx_codec_enc_config_default(ctypes.byref(vpx_codec), config_buffer, 0) == 0:
+			struct.pack_into('I', config_buffer, 4, thread_count)
+			struct.pack_into('I', config_buffer, 12, frame_resolution[0])
+			struct.pack_into('I', config_buffer, 16, frame_resolution[1])
+			struct.pack_into('I', config_buffer, 28, 1)
+			struct.pack_into('I', config_buffer, 36, 0)
+			struct.pack_into('I', config_buffer, 44, 0)
+			struct.pack_into('I', config_buffer, 72, 0)
+			struct.pack_into('I', config_buffer, 112, bitrate)
+			struct.pack_into('I', config_buffer, 116, 2)
+			struct.pack_into('I', config_buffer, 120, 50)
+			struct.pack_into('I', config_buffer, 124, 50)
+			struct.pack_into('I', config_buffer, 128, 50)
+
+			if vpx_library.vpx_codec_enc_init_ver(vpx_encoder, ctypes.byref(vpx_codec), config_buffer, 0, 39) == 0:
+				vpx_library.vpx_codec_control_(vpx_encoder, 13, ctypes.c_int(cpu_count))
+				vpx_library.vpx_codec_control_(vpx_encoder, 12, ctypes.c_int(0))
+				vpx_library.vpx_codec_control_(vpx_encoder, 27, ctypes.c_int(10))
+				ctypes.memmove(ctypes.addressof(vpx_encoder) + 64, config_buffer, 512)
+				return vpx_encoder
+
+	return None
+
+
+def encode(vpx_encoder : VpxEncoder, input_buffer : bytes, frame_resolution : Resolution, frame_index : int) -> bytes:
+	vpx_library = vpx_module.create_static_library()
+	output_buffer = bytes()
+
+	if vpx_library:
+		temp_buffer = ctypes.create_string_buffer(256)
+		encode_buffer = ctypes.create_string_buffer(input_buffer)
+
+		if vpx_library.vpx_img_wrap(temp_buffer, 0x102, frame_resolution[0], frame_resolution[1], 1, encode_buffer) and vpx_library.vpx_codec_encode(vpx_encoder, temp_buffer, frame_index, 1, 0, 1) == 0:
+			output_buffer = collect(vpx_encoder)
+
+	return output_buffer
+
+
+def collect(vpx_encoder : VpxEncoder) -> bytes:
+	vpx_library = vpx_module.create_static_library()
+	output_parts = []
+
+	packet_cursor = ctypes.c_void_p(0)
+	packet = vpx_library.vpx_codec_get_cx_data(vpx_encoder, ctypes.byref(packet_cursor))
+
+	while packet:
+		if ctypes.c_int.from_address(packet).value == 0:
+			buffer_pointer = ctypes.c_void_p.from_address(packet + 8).value
+			buffer_size = ctypes.c_size_t.from_address(packet + 16).value
+			output_parts.append(ctypes.string_at(buffer_pointer, buffer_size))
+
+		packet = vpx_library.vpx_codec_get_cx_data(vpx_encoder, ctypes.byref(packet_cursor))
+
+	return bytes().join(output_parts)
+
+
+def update_resolution(vpx_encoder : VpxEncoder, frame_resolution : Resolution) -> bool:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		struct.pack_into('I', vpx_encoder, 64 + 12, frame_resolution[0])
+		struct.pack_into('I', vpx_encoder, 64 + 16, frame_resolution[1])
+		return vpx_library.vpx_codec_enc_config_set(vpx_encoder, ctypes.cast(ctypes.addressof(vpx_encoder) + 64, ctypes.c_void_p)) == 0
+
+	return False
+
+
+def update_bitrate(vpx_encoder : VpxEncoder, bitrate : BitRate) -> bool:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		struct.pack_into('I', vpx_encoder, 64 + 112, bitrate)
+		return vpx_library.vpx_codec_enc_config_set(vpx_encoder, ctypes.cast(ctypes.addressof(vpx_encoder) + 64, ctypes.c_void_p)) == 0
+
+	return False
+
+
+def destroy(vpx_encoder : VpxEncoder) -> None:
+	vpx_library = vpx_module.create_static_library()
+
+	if vpx_library:
+		vpx_library.vpx_codec_destroy(vpx_encoder)

facefusion/common_helper.py

@@ -15,11 +15,11 @@ def is_windows() -> bool:
 
 
 def create_int_metavar(int_range : Sequence[int]) -> str:
-	return '[' + str(int_range[0]) + '..' + str(int_range[-1]) + ':' + str(calc_int_step(int_range)) + ']'
+	return '[' + str(int_range[0]) + '..' + str(int_range[-1]) + ':' + str(calculate_int_step(int_range)) + ']'
 
 
 def create_float_metavar(float_range : Sequence[float]) -> str:
-	return '[' + str(float_range[0]) + '..' + str(float_range[-1]) + ':' + str(calc_float_step(float_range)) + ']'
+	return '[' + str(float_range[0]) + '..' + str(float_range[-1]) + ':' + str(calculate_float_step(float_range)) + ']'
 
 
 def create_int_range(start : int, end : int, step : int) -> Sequence[int]:
@@ -42,11 +42,11 @@ def create_float_range(start : float, end : float, step : float) -> Sequence[flo
 	return float_range
 
 
-def calc_int_step(int_range : Sequence[int]) -> int:
+def calculate_int_step(int_range : Sequence[int]) -> int:
 	return int_range[1] - int_range[0]
 
 
-def calc_float_step(float_range : Sequence[float]) -> float:
+def calculate_float_step(float_range : Sequence[float]) -> float:
 	return round(float_range[1] - float_range[0], 2)
 
 

facefusion/content_analyser.py

@@ -4,23 +4,30 @@ from typing import List, Tuple
 import numpy
 from tqdm import tqdm
 
-from facefusion import inference_manager, state_manager, wording
+from facefusion import inference_manager, state_manager, translator
+from facefusion.common_helper import is_macos
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
 from facefusion.execution import has_execution_provider
 from facefusion.filesystem import resolve_relative_path
 from facefusion.thread_helper import conditional_thread_semaphore
 from facefusion.types import Detection, DownloadScope, DownloadSet, ExecutionProvider, Fps, InferencePool, ModelSet, VisionFrame
-from facefusion.vision import detect_video_fps, fit_frame, read_image, read_video_frame
+from facefusion.vision import detect_video_fps, fit_contain_frame, read_image, read_video_frame
 
 STREAM_COUNTER = 0
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'nsfw_1':
 		{
+			'__metadata__':
+			{
+				'vendor': 'EraX',
+				'license': 'Apache-2.0',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'content_analyser':
@@ -43,6 +50,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'nsfw_2':
 		{
+			'__metadata__':
+			{
+				'vendor': 'Marqo',
+				'license': 'Apache-2.0',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'content_analyser':
@@ -65,6 +78,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'nsfw_3':
 		{
+			'__metadata__':
+			{
+				'vendor': 'Freepik',
+				'license': 'MIT',
+				'year': 2025
+			},
 			'hashes':
 			{
 				'content_analyser':
@@ -101,7 +120,7 @@ def clear_inference_pool() -> None:
 
 
 def resolve_execution_providers() -> List[ExecutionProvider]:
-	if has_execution_provider('coreml'):
+	if is_macos() and has_execution_provider('coreml'):
 		return [ 'cpu' ]
 	return state_manager.get_item('execution_providers')
 
@@ -137,13 +156,13 @@ def analyse_frame(vision_frame : VisionFrame) -> bool:
 	return detect_nsfw(vision_frame)
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def analyse_image(image_path : str) -> bool:
 	vision_frame = read_image(image_path)
 	return analyse_frame(vision_frame)
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def analyse_video(video_path : str, trim_frame_start : int, trim_frame_end : int) -> bool:
 	video_fps = detect_video_fps(video_path)
 	frame_range = range(trim_frame_start, trim_frame_end)
@@ -151,16 +170,19 @@ def analyse_video(video_path : str, trim_frame_start : int, trim_frame_end : int
 	total = 0
 	counter = 0
 
-	with tqdm(total = len(frame_range), desc = wording.get('analysing'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
+	with tqdm(total = len(frame_range), desc = translator.get('analysing'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
 
 		for frame_number in frame_range:
 			if frame_number % int(video_fps) == 0:
 				vision_frame = read_video_frame(video_path, frame_number)
 				total += 1
+
 				if analyse_frame(vision_frame):
 					counter += 1
+
 			if counter > 0 and total > 0:
 				rate = counter / total * 100
+
 			progress.set_postfix(rate = rate)
 			progress.update()
 
@@ -196,8 +218,8 @@ def detect_with_nsfw_3(vision_frame : VisionFrame) -> bool:
 	return bool(detection_score > 10.5)
 
 
-def forward_nsfw(vision_frame : VisionFrame, nsfw_model : str) -> Detection:
-	content_analyser = get_inference_pool().get(nsfw_model)
+def forward_nsfw(vision_frame : VisionFrame, model_name : str) -> Detection:
+	content_analyser = get_inference_pool().get(model_name)
 
 	with conditional_thread_semaphore():
 		detection = content_analyser.run(None,
@@ -205,7 +227,7 @@ def forward_nsfw(vision_frame : VisionFrame, nsfw_model : str) -> Detection:
 			'input': vision_frame
 		})[0]
 
-	if nsfw_model in [ 'nsfw_2', 'nsfw_3' ]:
+	if model_name in [ 'nsfw_2', 'nsfw_3' ]:
 		return detection[0]
 
 	return detection
@@ -217,7 +239,7 @@ def prepare_detect_frame(temp_vision_frame : VisionFrame, model_name : str) -> V
 	model_mean = model_set.get('mean')
 	model_standard_deviation = model_set.get('standard_deviation')
 
-	detect_vision_frame = fit_frame(temp_vision_frame, model_size)
+	detect_vision_frame = fit_contain_frame(temp_vision_frame, model_size)
 	detect_vision_frame = detect_vision_frame[:, :, ::-1] / 255.0
 	detect_vision_frame -= model_mean
 	detect_vision_frame /= model_standard_deviation

facefusion/core.py

@@ -5,28 +5,23 @@ import signal
 import sys
 from time import time
 
-import numpy
+import uvicorn
 
-from facefusion import benchmarker, cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, hash_helper, logger, process_manager, state_manager, video_manager, voice_extractor, wording
-from facefusion.args import apply_args, collect_job_args, reduce_job_args, reduce_step_args
-from facefusion.common_helper import get_first
-from facefusion.content_analyser import analyse_image, analyse_video
+from facefusion import args_store, benchmarker, cli_helper, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, hash_helper, logger, state_manager, translator, voice_extractor
+from facefusion.apis.core import create_api
+from facefusion.args_helper import apply_args
 from facefusion.download import conditional_download_hashes, conditional_download_sources
 from facefusion.exit_helper import hard_exit, signal_exit
-from facefusion.face_analyser import get_average_face, get_many_faces, get_one_face
-from facefusion.face_selector import sort_and_filter_faces
-from facefusion.face_store import append_reference_face, clear_reference_faces, get_reference_faces
-from facefusion.ffmpeg import copy_image, extract_frames, finalize_image, merge_video, replace_audio, restore_audio
-from facefusion.filesystem import filter_audio_paths, get_file_name, is_image, is_video, resolve_file_paths, resolve_file_pattern
+from facefusion.filesystem import get_file_extension, get_file_name, resolve_file_paths, resolve_file_pattern
+from facefusion.filesystem import has_audio, has_image, has_video
 from facefusion.jobs import job_helper, job_manager, job_runner
 from facefusion.jobs.job_list import compose_job_list
-from facefusion.memory import limit_system_memory
+from facefusion.libraries import aom as aom_module, datachannel as datachannel_module, opus as opus_module, vpx as vpx_module
 from facefusion.processors.core import get_processors_modules
 from facefusion.program import create_program
 from facefusion.program_helper import validate_args
-from facefusion.temp_helper import clear_temp_directory, create_temp_directory, get_temp_file_path, move_temp_file, resolve_temp_frame_paths
-from facefusion.types import Args, ErrorCode
-from facefusion.vision import pack_resolution, read_image, read_static_images, read_video_frame, restrict_image_resolution, restrict_trim_frame, restrict_video_fps, restrict_video_resolution, unpack_resolution
+from facefusion.types import Args, ErrorCode, WorkFlow
+from facefusion.workflows import audio_to_image, image_to_image, image_to_video
 
 
 def cli() -> None:
@@ -50,51 +45,40 @@ def cli() -> None:
 
 
 def route(args : Args) -> None:
-	system_memory_limit = state_manager.get_item('system_memory_limit')
-
-	if system_memory_limit and system_memory_limit > 0:
-		limit_system_memory(system_memory_limit)
-
 	if state_manager.get_item('command') == 'force-download':
 		error_code = force_download()
-		return hard_exit(error_code)
+		hard_exit(error_code)
 
 	if state_manager.get_item('command') == 'benchmark':
 		if not common_pre_check() or not processors_pre_check() or not benchmarker.pre_check():
-			return hard_exit(2)
+			hard_exit(2)
 		benchmarker.render()
 
+	if state_manager.get_item('command') == 'api':
+		logger.info(translator.get('api_started').format(host = state_manager.get_item('api_host'), port = state_manager.get_item('api_port')), __name__)
+		uvicorn.run(create_api(), host = state_manager.get_item('api_host'), port = state_manager.get_item('api_port'))
+		hard_exit(1)
+
 	if state_manager.get_item('command') in [ 'job-list', 'job-create', 'job-submit', 'job-submit-all', 'job-delete', 'job-delete-all', 'job-add-step', 'job-remix-step', 'job-insert-step', 'job-remove-step' ]:
-		if not job_manager.init_jobs(state_manager.get_item('jobs_path')):
+		if not job_manager.init_jobs(state_manager.get_jobs_path()):
 			hard_exit(1)
 		error_code = route_job_manager(args)
 		hard_exit(error_code)
 
 	if state_manager.get_item('command') == 'run':
-		import facefusion.uis.core as ui
-
-		if not common_pre_check() or not processors_pre_check():
-			return hard_exit(2)
-		for ui_layout in ui.get_ui_layouts_modules(state_manager.get_item('ui_layouts')):
-			if not ui_layout.pre_check():
-				return hard_exit(2)
-		ui.init()
-		ui.launch()
-
-	if state_manager.get_item('command') == 'headless-run':
-		if not job_manager.init_jobs(state_manager.get_item('jobs_path')):
+		if not job_manager.init_jobs(state_manager.get_jobs_path()):
 			hard_exit(1)
-		error_core = process_headless(args)
-		hard_exit(error_core)
+		error_code = process_headless(args)
+		hard_exit(error_code)
 
 	if state_manager.get_item('command') == 'batch-run':
-		if not job_manager.init_jobs(state_manager.get_item('jobs_path')):
+		if not job_manager.init_jobs(state_manager.get_jobs_path()):
 			hard_exit(1)
-		error_core = process_batch(args)
-		hard_exit(error_core)
+		error_code = process_batch(args)
+		hard_exit(error_code)
 
 	if state_manager.get_item('command') in [ 'job-run', 'job-run-all', 'job-retry', 'job-retry-all' ]:
-		if not job_manager.init_jobs(state_manager.get_item('jobs_path')):
+		if not job_manager.init_jobs(state_manager.get_jobs_path()):
 			hard_exit(1)
 		error_code = route_job_runner()
 		hard_exit(error_code)
@@ -102,15 +86,15 @@ def route(args : Args) -> None:
 
 def pre_check() -> bool:
 	if sys.version_info < (3, 10):
-		logger.error(wording.get('python_not_supported').format(version = '3.10'), __name__)
+		logger.error(translator.get('python_not_supported').format(version = '3.10'), __name__)
 		return False
 
 	if not shutil.which('curl'):
-		logger.error(wording.get('curl_not_installed'), __name__)
+		logger.error(translator.get('curl_not_installed'), __name__)
 		return False
 
 	if not shutil.which('ffmpeg'):
-		logger.error(wording.get('ffmpeg_not_installed'), __name__)
+		logger.error(translator.get('ffmpeg_not_installed'), __name__)
 		return False
 	return True
 
@@ -118,19 +102,23 @@ def pre_check() -> bool:
 def common_pre_check() -> bool:
 	common_modules =\
 	[
+		aom_module,
+		datachannel_module,
 		content_analyser,
 		face_classifier,
 		face_detector,
 		face_landmarker,
 		face_masker,
 		face_recognizer,
-		voice_extractor
+		opus_module,
+		voice_extractor,
+		vpx_module
 	]
 
 	content_analyser_content = inspect.getsource(content_analyser).encode()
-	is_valid = hash_helper.create_hash(content_analyser_content) == 'b159fd9d'
+	content_analyser_hash = hash_helper.create_hash(content_analyser_content)
 
-	return all(module.pre_check() for module in common_modules) and is_valid
+	return all(module.pre_check() for module in common_modules) and content_analyser_hash == 'b14e7b92'
 
 
 def processors_pre_check() -> bool:
@@ -178,113 +166,113 @@ def route_job_manager(args : Args) -> ErrorCode:
 
 	if state_manager.get_item('command') == 'job-create':
 		if job_manager.create_job(state_manager.get_item('job_id')):
-			logger.info(wording.get('job_created').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('job_created').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.error(wording.get('job_not_created').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.error(translator.get('job_not_created').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-submit':
 		if job_manager.submit_job(state_manager.get_item('job_id')):
-			logger.info(wording.get('job_submitted').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('job_submitted').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.error(wording.get('job_not_submitted').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.error(translator.get('job_not_submitted').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-submit-all':
 		if job_manager.submit_jobs(state_manager.get_item('halt_on_error')):
-			logger.info(wording.get('job_all_submitted'), __name__)
+			logger.info(translator.get('job_all_submitted'), __name__)
 			return 0
-		logger.error(wording.get('job_all_not_submitted'), __name__)
+		logger.error(translator.get('job_all_not_submitted'), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-delete':
 		if job_manager.delete_job(state_manager.get_item('job_id')):
-			logger.info(wording.get('job_deleted').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('job_deleted').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.error(wording.get('job_not_deleted').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.error(translator.get('job_not_deleted').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-delete-all':
 		if job_manager.delete_jobs(state_manager.get_item('halt_on_error')):
-			logger.info(wording.get('job_all_deleted'), __name__)
+			logger.info(translator.get('job_all_deleted'), __name__)
 			return 0
-		logger.error(wording.get('job_all_not_deleted'), __name__)
+		logger.error(translator.get('job_all_not_deleted'), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-add-step':
-		step_args = reduce_step_args(args)
+		step_args = args_store.filter_step_args(args)
 
 		if job_manager.add_step(state_manager.get_item('job_id'), step_args):
-			logger.info(wording.get('job_step_added').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('job_step_added').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.error(wording.get('job_step_not_added').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.error(translator.get('job_step_not_added').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-remix-step':
-		step_args = reduce_step_args(args)
+		step_args = args_store.filter_step_args(args)
 
 		if job_manager.remix_step(state_manager.get_item('job_id'), state_manager.get_item('step_index'), step_args):
-			logger.info(wording.get('job_remix_step_added').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+			logger.info(translator.get('job_remix_step_added').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 			return 0
-		logger.error(wording.get('job_remix_step_not_added').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+		logger.error(translator.get('job_remix_step_not_added').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-insert-step':
-		step_args = reduce_step_args(args)
+		step_args = args_store.filter_step_args(args)
 
 		if job_manager.insert_step(state_manager.get_item('job_id'), state_manager.get_item('step_index'), step_args):
-			logger.info(wording.get('job_step_inserted').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+			logger.info(translator.get('job_step_inserted').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 			return 0
-		logger.error(wording.get('job_step_not_inserted').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+		logger.error(translator.get('job_step_not_inserted').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-remove-step':
 		if job_manager.remove_step(state_manager.get_item('job_id'), state_manager.get_item('step_index')):
-			logger.info(wording.get('job_step_removed').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+			logger.info(translator.get('job_step_removed').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 			return 0
-		logger.error(wording.get('job_step_not_removed').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
+		logger.error(translator.get('job_step_not_removed').format(job_id = state_manager.get_item('job_id'), step_index = state_manager.get_item('step_index')), __name__)
 		return 1
 	return 1
 
 
 def route_job_runner() -> ErrorCode:
 	if state_manager.get_item('command') == 'job-run':
-		logger.info(wording.get('running_job').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.info(translator.get('running_job').format(job_id = state_manager.get_item('job_id')), __name__)
 		if job_runner.run_job(state_manager.get_item('job_id'), process_step):
-			logger.info(wording.get('processing_job_succeed').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('processing_job_succeeded').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.info(wording.get('processing_job_failed').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.info(translator.get('processing_job_failed').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-run-all':
-		logger.info(wording.get('running_jobs'), __name__)
+		logger.info(translator.get('running_jobs'), __name__)
 		if job_runner.run_jobs(process_step, state_manager.get_item('halt_on_error')):
-			logger.info(wording.get('processing_jobs_succeed'), __name__)
+			logger.info(translator.get('processing_jobs_succeeded'), __name__)
 			return 0
-		logger.info(wording.get('processing_jobs_failed'), __name__)
+		logger.info(translator.get('processing_jobs_failed'), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-retry':
-		logger.info(wording.get('retrying_job').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.info(translator.get('retrying_job').format(job_id = state_manager.get_item('job_id')), __name__)
 		if job_runner.retry_job(state_manager.get_item('job_id'), process_step):
-			logger.info(wording.get('processing_job_succeed').format(job_id = state_manager.get_item('job_id')), __name__)
+			logger.info(translator.get('processing_job_succeeded').format(job_id = state_manager.get_item('job_id')), __name__)
 			return 0
-		logger.info(wording.get('processing_job_failed').format(job_id = state_manager.get_item('job_id')), __name__)
+		logger.info(translator.get('processing_job_failed').format(job_id = state_manager.get_item('job_id')), __name__)
 		return 1
 
 	if state_manager.get_item('command') == 'job-retry-all':
-		logger.info(wording.get('retrying_jobs'), __name__)
+		logger.info(translator.get('retrying_jobs'), __name__)
 		if job_runner.retry_jobs(process_step, state_manager.get_item('halt_on_error')):
-			logger.info(wording.get('processing_jobs_succeed'), __name__)
+			logger.info(translator.get('processing_jobs_succeeded'), __name__)
 			return 0
-		logger.info(wording.get('processing_jobs_failed'), __name__)
+		logger.info(translator.get('processing_jobs_failed'), __name__)
 		return 1
 	return 2
 
 
 def process_headless(args : Args) -> ErrorCode:
 	job_id = job_helper.suggest_job_id('headless')
-	step_args = reduce_step_args(args)
+	step_args = args_store.filter_step_args(args)
 
 	if job_manager.create_job(job_id) and job_manager.add_step(job_id, step_args) and job_manager.submit_job(job_id) and job_runner.run_job(job_id, process_step):
 		return 0
@@ -293,17 +281,21 @@ def process_headless(args : Args) -> ErrorCode:
 
 def process_batch(args : Args) -> ErrorCode:
 	job_id = job_helper.suggest_job_id('batch')
-	step_args = reduce_step_args(args)
-	job_args = reduce_job_args(args)
-	source_paths = resolve_file_pattern(job_args.get('source_pattern'))
-	target_paths = resolve_file_pattern(job_args.get('target_pattern'))
+	step_args = args_store.filter_step_args(args)
+	source_paths = resolve_file_pattern(step_args.get('source_pattern'))
+	target_paths = resolve_file_pattern(step_args.get('target_pattern'))
 
 	if job_manager.create_job(job_id):
 		if source_paths and target_paths:
 			for index, (source_path, target_path) in enumerate(itertools.product(source_paths, target_paths)):
 				step_args['source_paths'] = [ source_path ]
 				step_args['target_path'] = target_path
-				step_args['output_path'] = job_args.get('output_pattern').format(index = index)
+
+				try:
+					step_args['output_path'] = step_args.get('output_pattern').format(index = index, source_name = get_file_name(source_path), target_name = get_file_name(target_path), target_extension = get_file_extension(target_path))
+				except KeyError:
+					return 1
+
 				if not job_manager.add_step(job_id, step_args):
 					return 1
 			if job_manager.submit_job(job_id) and job_runner.run_job(job_id, process_step):
@@ -312,7 +304,12 @@ def process_batch(args : Args) -> ErrorCode:
 		if not source_paths and target_paths:
 			for index, target_path in enumerate(target_paths):
 				step_args['target_path'] = target_path
-				step_args['output_path'] = job_args.get('output_pattern').format(index = index)
+
+				try:
+					step_args['output_path'] = step_args.get('output_pattern').format(index = index, target_name = get_file_name(target_path), target_extension = get_file_extension(target_path))
+				except KeyError:
+					return 1
+
 				if not job_manager.add_step(job_id, step_args):
 					return 1
 			if job_manager.submit_job(job_id) and job_runner.run_job(job_id, process_step):
@@ -321,12 +318,13 @@ def process_batch(args : Args) -> ErrorCode:
 
 
 def process_step(job_id : str, step_index : int, step_args : Args) -> bool:
-	clear_reference_faces()
 	step_total = job_manager.count_step_total(job_id)
-	step_args.update(collect_job_args())
-	apply_args(step_args, state_manager.set_item)
+	cli_args = args_store.filter_cli_args(state_manager.get_state()) #type:ignore[arg-type]
+	args = cli_args.copy()
+	args.update(step_args)
+	apply_args(args, state_manager.set_item)
 
-	logger.info(wording.get('processing_step').format(step_current = step_index + 1, step_total = step_total), __name__)
+	logger.info(translator.get('processing_step').format(step_current = step_index + 1, step_total = step_total), __name__)
 	if common_pre_check() and processors_pre_check():
 		error_code = conditional_process()
 		return error_code == 0
@@ -336,182 +334,28 @@ def process_step(job_id : str, step_index : int, step_args : Args) -> bool:
 def conditional_process() -> ErrorCode:
 	start_time = time()
 
+	if state_manager.get_item('workflow') == 'auto':
+		state_manager.set_item('workflow', detect_workflow())
+
 	for processor_module in get_processors_modules(state_manager.get_item('processors')):
 		if not processor_module.pre_process('output'):
 			return 2
 
-	conditional_append_reference_faces()
-
-	if is_image(state_manager.get_item('target_path')):
-		return process_image(start_time)
-	if is_video(state_manager.get_item('target_path')):
-		return process_video(start_time)
+	if state_manager.get_item('workflow') == 'audio-to-image':
+		return audio_to_image.process(start_time)
+	if state_manager.get_item('workflow') == 'image-to-image':
+		return image_to_image.process(start_time)
+	if state_manager.get_item('workflow') == 'image-to-video':
+		return image_to_video.process(start_time)
 
 	return 0
 
 
-def conditional_append_reference_faces() -> None:
-	if 'reference' in state_manager.get_item('face_selector_mode') and not get_reference_faces():
-		source_frames = read_static_images(state_manager.get_item('source_paths'))
-		source_faces = get_many_faces(source_frames)
-		source_face = get_average_face(source_faces)
-		if is_video(state_manager.get_item('target_path')):
-			reference_frame = read_video_frame(state_manager.get_item('target_path'), state_manager.get_item('reference_frame_number'))
-		else:
-			reference_frame = read_image(state_manager.get_item('target_path'))
-		reference_faces = sort_and_filter_faces(get_many_faces([ reference_frame ]))
-		reference_face = get_one_face(reference_faces, state_manager.get_item('reference_face_position'))
-		append_reference_face('origin', reference_face)
+def detect_workflow() -> WorkFlow:
+	if has_video([ state_manager.get_item('target_path') ]):
+		return 'image-to-video'
 
-		if source_face and reference_face:
-			for processor_module in get_processors_modules(state_manager.get_item('processors')):
-				abstract_reference_frame = processor_module.get_reference_frame(source_face, reference_face, reference_frame)
-				if numpy.any(abstract_reference_frame):
-					abstract_reference_faces = sort_and_filter_faces(get_many_faces([ abstract_reference_frame ]))
-					abstract_reference_face = get_one_face(abstract_reference_faces, state_manager.get_item('reference_face_position'))
-					append_reference_face(processor_module.__name__, abstract_reference_face)
+	if has_audio(state_manager.get_item('source_paths')) and has_image([ state_manager.get_item('target_path') ]):
+		return 'audio-to-image'
 
-
-def process_image(start_time : float) -> ErrorCode:
-	if analyse_image(state_manager.get_item('target_path')):
-		return 3
-
-	logger.debug(wording.get('clearing_temp'), __name__)
-	clear_temp_directory(state_manager.get_item('target_path'))
-	logger.debug(wording.get('creating_temp'), __name__)
-	create_temp_directory(state_manager.get_item('target_path'))
-
-	process_manager.start()
-	temp_image_resolution = pack_resolution(restrict_image_resolution(state_manager.get_item('target_path'), unpack_resolution(state_manager.get_item('output_image_resolution'))))
-	logger.info(wording.get('copying_image').format(resolution = temp_image_resolution), __name__)
-	if copy_image(state_manager.get_item('target_path'), temp_image_resolution):
-		logger.debug(wording.get('copying_image_succeed'), __name__)
-	else:
-		logger.error(wording.get('copying_image_failed'), __name__)
-		process_manager.end()
-		return 1
-
-	temp_image_path = get_temp_file_path(state_manager.get_item('target_path'))
-	for processor_module in get_processors_modules(state_manager.get_item('processors')):
-		logger.info(wording.get('processing'), processor_module.__name__)
-		processor_module.process_image(state_manager.get_item('source_paths'), temp_image_path, temp_image_path)
-		processor_module.post_process()
-	if is_process_stopping():
-		process_manager.end()
-		return 4
-
-	logger.info(wording.get('finalizing_image').format(resolution = state_manager.get_item('output_image_resolution')), __name__)
-	if finalize_image(state_manager.get_item('target_path'), state_manager.get_item('output_path'), state_manager.get_item('output_image_resolution')):
-		logger.debug(wording.get('finalizing_image_succeed'), __name__)
-	else:
-		logger.warn(wording.get('finalizing_image_skipped'), __name__)
-
-	logger.debug(wording.get('clearing_temp'), __name__)
-	clear_temp_directory(state_manager.get_item('target_path'))
-
-	if is_image(state_manager.get_item('output_path')):
-		seconds = '{:.2f}'.format((time() - start_time) % 60)
-		logger.info(wording.get('processing_image_succeed').format(seconds = seconds), __name__)
-	else:
-		logger.error(wording.get('processing_image_failed'), __name__)
-		process_manager.end()
-		return 1
-	process_manager.end()
-	return 0
-
-
-def process_video(start_time : float) -> ErrorCode:
-	trim_frame_start, trim_frame_end = restrict_trim_frame(state_manager.get_item('target_path'), state_manager.get_item('trim_frame_start'), state_manager.get_item('trim_frame_end'))
-	if analyse_video(state_manager.get_item('target_path'), trim_frame_start, trim_frame_end):
-		return 3
-
-	logger.debug(wording.get('clearing_temp'), __name__)
-	clear_temp_directory(state_manager.get_item('target_path'))
-	logger.debug(wording.get('creating_temp'), __name__)
-	create_temp_directory(state_manager.get_item('target_path'))
-
-	process_manager.start()
-	temp_video_resolution = pack_resolution(restrict_video_resolution(state_manager.get_item('target_path'), unpack_resolution(state_manager.get_item('output_video_resolution'))))
-	temp_video_fps = restrict_video_fps(state_manager.get_item('target_path'), state_manager.get_item('output_video_fps'))
-	logger.info(wording.get('extracting_frames').format(resolution = temp_video_resolution, fps = temp_video_fps), __name__)
-	if extract_frames(state_manager.get_item('target_path'), temp_video_resolution, temp_video_fps, trim_frame_start, trim_frame_end):
-		logger.debug(wording.get('extracting_frames_succeed'), __name__)
-	else:
-		if is_process_stopping():
-			process_manager.end()
-			return 4
-		logger.error(wording.get('extracting_frames_failed'), __name__)
-		process_manager.end()
-		return 1
-
-	temp_frame_paths = resolve_temp_frame_paths(state_manager.get_item('target_path'))
-	if temp_frame_paths:
-		for processor_module in get_processors_modules(state_manager.get_item('processors')):
-			logger.info(wording.get('processing'), processor_module.__name__)
-			processor_module.process_video(state_manager.get_item('source_paths'), temp_frame_paths)
-			processor_module.post_process()
-		if is_process_stopping():
-			return 4
-	else:
-		logger.error(wording.get('temp_frames_not_found'), __name__)
-		process_manager.end()
-		return 1
-
-	logger.info(wording.get('merging_video').format(resolution = state_manager.get_item('output_video_resolution'), fps = state_manager.get_item('output_video_fps')), __name__)
-	if merge_video(state_manager.get_item('target_path'), temp_video_fps, state_manager.get_item('output_video_resolution'), state_manager.get_item('output_video_fps'), trim_frame_start, trim_frame_end):
-		logger.debug(wording.get('merging_video_succeed'), __name__)
-	else:
-		if is_process_stopping():
-			process_manager.end()
-			return 4
-		logger.error(wording.get('merging_video_failed'), __name__)
-		process_manager.end()
-		return 1
-
-	if state_manager.get_item('output_audio_volume') == 0:
-		logger.info(wording.get('skipping_audio'), __name__)
-		move_temp_file(state_manager.get_item('target_path'), state_manager.get_item('output_path'))
-	else:
-		source_audio_path = get_first(filter_audio_paths(state_manager.get_item('source_paths')))
-		if source_audio_path:
-			if replace_audio(state_manager.get_item('target_path'), source_audio_path, state_manager.get_item('output_path')):
-				video_manager.clear_video_pool()
-				logger.debug(wording.get('replacing_audio_succeed'), __name__)
-			else:
-				video_manager.clear_video_pool()
-				if is_process_stopping():
-					process_manager.end()
-					return 4
-				logger.warn(wording.get('replacing_audio_skipped'), __name__)
-				move_temp_file(state_manager.get_item('target_path'), state_manager.get_item('output_path'))
-		else:
-			if restore_audio(state_manager.get_item('target_path'), state_manager.get_item('output_path'), trim_frame_start, trim_frame_end):
-				video_manager.clear_video_pool()
-				logger.debug(wording.get('restoring_audio_succeed'), __name__)
-			else:
-				video_manager.clear_video_pool()
-				if is_process_stopping():
-					process_manager.end()
-					return 4
-				logger.warn(wording.get('restoring_audio_skipped'), __name__)
-				move_temp_file(state_manager.get_item('target_path'), state_manager.get_item('output_path'))
-
-	logger.debug(wording.get('clearing_temp'), __name__)
-	clear_temp_directory(state_manager.get_item('target_path'))
-
-	if is_video(state_manager.get_item('output_path')):
-		seconds = '{:.2f}'.format((time() - start_time))
-		logger.info(wording.get('processing_video_succeed').format(seconds = seconds), __name__)
-	else:
-		logger.error(wording.get('processing_video_failed'), __name__)
-		process_manager.end()
-		return 1
-	process_manager.end()
-	return 0
-
-
-def is_process_stopping() -> bool:
-	if process_manager.is_stopping():
-		process_manager.end()
-		logger.info(wording.get('processing_stopped'), __name__)
-	return process_manager.is_pending()
+	return 'image-to-image'

facefusion/curl_builder.py

@@ -1,27 +1,28 @@
 import itertools
 import shutil
+from typing import List
 
 from facefusion import metadata
-from facefusion.types import Commands
+from facefusion.types import Command
 
 
-def run(commands : Commands) -> Commands:
+def run(commands : List[Command]) -> List[Command]:
 	user_agent = metadata.get('name') + '/' + metadata.get('version')
 
 	return [ shutil.which('curl'), '--user-agent', user_agent, '--insecure', '--location', '--silent' ] + commands
 
 
-def chain(*commands : Commands) -> Commands:
+def chain(*commands : List[Command]) -> List[Command]:
 	return list(itertools.chain(*commands))
 
 
-def head(url : str) -> Commands:
+def head(url : str) -> List[Command]:
 	return [ '-I', url ]
 
 
-def download(url : str, download_file_path : str) -> Commands:
+def download(url : str, download_file_path : str) -> List[Command]:
 	return [ '--create-dirs', '--continue-at', '-', '--output', download_file_path, url ]
 
 
-def set_timeout(timeout : int) -> Commands:
+def set_timeout(timeout : int) -> List[Command]:
 	return [ '--connect-timeout', str(timeout) ]

facefusion/download.py

@@ -7,13 +7,13 @@ from urllib.parse import urlparse
 from tqdm import tqdm
 
 import facefusion.choices
-from facefusion import curl_builder, logger, process_manager, state_manager, wording
+from facefusion import curl_builder, logger, process_manager, state_manager, translator
 from facefusion.filesystem import get_file_name, get_file_size, is_file, remove_file
 from facefusion.hash_helper import validate_hash
-from facefusion.types import Commands, DownloadProvider, DownloadSet
+from facefusion.types import Command, DownloadProvider, DownloadSet
 
 
-def open_curl(commands : Commands) -> subprocess.Popen[bytes]:
+def open_curl(commands : List[Command]) -> subprocess.Popen[bytes]:
 	commands = curl_builder.run(commands)
 	return subprocess.Popen(commands, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
 
@@ -26,10 +26,10 @@ def conditional_download(download_directory_path : str, urls : List[str]) -> Non
 		download_size = get_static_download_size(url)
 
 		if initial_size < download_size:
-			with tqdm(total = download_size, initial = initial_size, desc = wording.get('downloading'), unit = 'B', unit_scale = True, unit_divisor = 1024, ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
+			with tqdm(total = download_size, initial = initial_size, desc = translator.get('downloading'), unit = 'B', unit_scale = True, unit_divisor = 1024, ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
 				commands = curl_builder.chain(
 					curl_builder.download(url, download_file_path),
-					curl_builder.set_timeout(10)
+					curl_builder.set_timeout(5)
 				)
 				open_curl(commands)
 				current_size = initial_size
@@ -41,7 +41,7 @@ def conditional_download(download_directory_path : str, urls : List[str]) -> Non
 						progress.update(current_size - progress.n)
 
 
-@lru_cache(maxsize = None)
+@lru_cache(maxsize = 64)
 def get_static_download_size(url : str) -> int:
 	commands = curl_builder.chain(
 		curl_builder.head(url),
@@ -59,7 +59,7 @@ def get_static_download_size(url : str) -> int:
 	return 0
 
 
-@lru_cache(maxsize = None)
+@lru_cache(maxsize = 64)
 def ping_static_url(url : str) -> bool:
 	commands = curl_builder.chain(
 		curl_builder.head(url),
@@ -87,10 +87,10 @@ def conditional_download_hashes(hash_set : DownloadSet) -> bool:
 
 	for valid_hash_path in valid_hash_paths:
 		valid_hash_file_name = get_file_name(valid_hash_path)
-		logger.debug(wording.get('validating_hash_succeed').format(hash_file_name = valid_hash_file_name), __name__)
+		logger.debug(translator.get('validating_hash_succeeded').format(hash_file_name = valid_hash_file_name), __name__)
 	for invalid_hash_path in invalid_hash_paths:
 		invalid_hash_file_name = get_file_name(invalid_hash_path)
-		logger.error(wording.get('validating_hash_failed').format(hash_file_name = invalid_hash_file_name), __name__)
+		logger.error(translator.get('validating_hash_failed').format(hash_file_name = invalid_hash_file_name), __name__)
 
 	if not invalid_hash_paths:
 		process_manager.end()
@@ -114,13 +114,13 @@ def conditional_download_sources(source_set : DownloadSet) -> bool:
 
 	for valid_source_path in valid_source_paths:
 		valid_source_file_name = get_file_name(valid_source_path)
-		logger.debug(wording.get('validating_source_succeed').format(source_file_name = valid_source_file_name), __name__)
+		logger.debug(translator.get('validating_source_succeeded').format(source_file_name = valid_source_file_name), __name__)
 	for invalid_source_path in invalid_source_paths:
 		invalid_source_file_name = get_file_name(invalid_source_path)
-		logger.error(wording.get('validating_source_failed').format(source_file_name = invalid_source_file_name), __name__)
+		logger.error(translator.get('validating_source_failed').format(source_file_name = invalid_source_file_name), __name__)
 
 		if remove_file(invalid_source_path):
-			logger.error(wording.get('deleting_corrupt_source').format(source_file_name = invalid_source_file_name), __name__)
+			logger.error(translator.get('deleting_corrupt_source').format(source_file_name = invalid_source_file_name), __name__)
 
 	if not invalid_source_paths:
 		process_manager.end()

facefusion/execution.py

@@ -28,7 +28,7 @@ def get_available_execution_providers() -> List[ExecutionProvider]:
 	return available_execution_providers
 
 
-def create_inference_session_providers(execution_device_id : str, execution_providers : List[ExecutionProvider]) -> List[InferenceSessionProvider]:
+def create_inference_session_providers(execution_device_id : int, execution_providers : List[ExecutionProvider]) -> List[InferenceSessionProvider]:
 	inference_session_providers : List[InferenceSessionProvider] = []
 
 	for execution_provider in execution_providers:
@@ -53,6 +53,12 @@ def create_inference_session_providers(execution_device_id : str, execution_prov
 			{
 				'device_id': execution_device_id
 			}))
+		if execution_provider == 'migraphx':
+			inference_session_providers.append((facefusion.choices.execution_provider_set.get(execution_provider),
+			{
+				'device_id': execution_device_id,
+				'migraphx_model_cache_dir': '.caches'
+			}))
 		if execution_provider == 'openvino':
 			inference_session_providers.append((facefusion.choices.execution_provider_set.get(execution_provider),
 			{
@@ -83,12 +89,10 @@ def resolve_cudnn_conv_algo_search() -> str:
 	return 'EXHAUSTIVE'
 
 
-def resolve_openvino_device_type(execution_device_id : str) -> str:
-	if execution_device_id == '0':
+def resolve_openvino_device_type(execution_device_id : int) -> str:
+	if execution_device_id == 0:
 		return 'GPU'
-	if execution_device_id == '∞':
-		return 'MULTI:GPU'
-	return 'GPU.' + execution_device_id
+	return 'GPU.' + str(execution_device_id)
 
 
 def run_nvidia_smi() -> subprocess.Popen[bytes]:
@@ -96,7 +100,7 @@ def run_nvidia_smi() -> subprocess.Popen[bytes]:
 	return subprocess.Popen(commands, stdout = subprocess.PIPE)
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def detect_static_execution_devices() -> List[ExecutionDevice]:
 	return detect_execution_devices()
 

facefusion/exit_helper.py

@@ -1,3 +1,4 @@
+import os
 import signal
 import sys
 from time import sleep
@@ -8,8 +9,11 @@ from facefusion.temp_helper import clear_temp_directory
 from facefusion.types import ErrorCode
 
 
+def fatal_exit(error_code : ErrorCode) -> None:
+	os._exit(error_code)
+
+
 def hard_exit(error_code : ErrorCode) -> None:
-	signal.signal(signal.SIGINT, signal.SIG_IGN)
 	sys.exit(error_code)
 
 
@@ -18,9 +22,13 @@ def signal_exit(signum : int, frame : FrameType) -> None:
 
 
 def graceful_exit(error_code : ErrorCode) -> None:
+	signal.signal(signal.SIGINT, signal.SIG_IGN)
 	process_manager.stop()
+
 	while process_manager.is_processing():
 		sleep(0.5)
-	if state_manager.get_item('target_path'):
-		clear_temp_directory(state_manager.get_item('target_path'))
+
+	if state_manager.get_item('output_path'):
+		clear_temp_directory(state_manager.get_temp_path(), state_manager.get_item('output_path'))
+
 	hard_exit(error_code)

facefusion/face_analyser.py

@@ -5,10 +5,10 @@ import numpy
 from facefusion import state_manager
 from facefusion.common_helper import get_first
 from facefusion.face_classifier import classify_face
-from facefusion.face_detector import detect_faces, detect_rotated_faces
+from facefusion.face_detector import detect_faces, detect_faces_by_angle
 from facefusion.face_helper import apply_nms, convert_to_face_landmark_5, estimate_face_angle, get_nms_threshold
 from facefusion.face_landmarker import detect_face_landmark, estimate_face_landmark_68_5
-from facefusion.face_recognizer import calc_embedding
+from facefusion.face_recognizer import calculate_face_embedding
 from facefusion.face_store import get_static_faces, set_static_faces
 from facefusion.types import BoundingBox, Face, FaceLandmark5, FaceLandmarkSet, FaceScoreSet, Score, VisionFrame
 
@@ -45,15 +45,15 @@ def create_faces(vision_frame : VisionFrame, bounding_boxes : List[BoundingBox],
 			'detector': face_score,
 			'landmarker': face_landmark_score_68
 		}
-		embedding, normed_embedding = calc_embedding(vision_frame, face_landmark_set.get('5/68'))
+		face_embedding, face_embedding_norm = calculate_face_embedding(vision_frame, face_landmark_set.get('5/68'))
 		gender, age, race = classify_face(vision_frame, face_landmark_set.get('5/68'))
 		faces.append(Face(
 			bounding_box = bounding_box,
 			score_set = face_score_set,
 			landmark_set = face_landmark_set,
 			angle = face_angle,
-			embedding = embedding,
-			normed_embedding = normed_embedding,
+			embedding = face_embedding,
+			embedding_norm = face_embedding_norm,
 			gender = gender,
 			age = age,
 			race = race
@@ -69,23 +69,23 @@ def get_one_face(faces : List[Face], position : int = 0) -> Optional[Face]:
 
 
 def get_average_face(faces : List[Face]) -> Optional[Face]:
-	embeddings = []
-	normed_embeddings = []
+	face_embeddings = []
+	face_embeddings_norm = []
 
 	if faces:
 		first_face = get_first(faces)
 
 		for face in faces:
-			embeddings.append(face.embedding)
-			normed_embeddings.append(face.normed_embedding)
+			face_embeddings.append(face.embedding)
+			face_embeddings_norm.append(face.embedding_norm)
 
 		return Face(
 			bounding_box = first_face.bounding_box,
 			score_set = first_face.score_set,
 			landmark_set = first_face.landmark_set,
 			angle = first_face.angle,
-			embedding = numpy.mean(embeddings, axis = 0),
-			normed_embedding = numpy.mean(normed_embeddings, axis = 0),
+			embedding = numpy.mean(face_embeddings, axis = 0),
+			embedding_norm = numpy.mean(face_embeddings_norm, axis = 0),
 			gender = first_face.gender,
 			age = first_face.age,
 			race = first_face.race
@@ -110,7 +110,7 @@ def get_many_faces(vision_frames : List[VisionFrame]) -> List[Face]:
 					if face_detector_angle == 0:
 						bounding_boxes, face_scores, face_landmarks_5 = detect_faces(vision_frame)
 					else:
-						bounding_boxes, face_scores, face_landmarks_5 = detect_rotated_faces(vision_frame, face_detector_angle)
+						bounding_boxes, face_scores, face_landmarks_5 = detect_faces_by_angle(vision_frame, face_detector_angle)
 					all_bounding_boxes.extend(bounding_boxes)
 					all_face_scores.extend(face_scores)
 					all_face_landmarks_5.extend(face_landmarks_5)
@@ -122,3 +122,22 @@ def get_many_faces(vision_frames : List[VisionFrame]) -> List[Face]:
 						many_faces.extend(faces)
 						set_static_faces(vision_frame, faces)
 	return many_faces
+
+
+def scale_face(target_face : Face, target_vision_frame : VisionFrame, temp_vision_frame : VisionFrame) -> Face:
+	scale_x = temp_vision_frame.shape[1] / target_vision_frame.shape[1]
+	scale_y = temp_vision_frame.shape[0] / target_vision_frame.shape[0]
+
+	bounding_box = target_face.bounding_box * [ scale_x, scale_y, scale_x, scale_y ]
+	landmark_set =\
+	{
+		'5': target_face.landmark_set.get('5') * numpy.array([ scale_x, scale_y ]),
+		'5/68': target_face.landmark_set.get('5/68') * numpy.array([ scale_x, scale_y ]),
+		'68': target_face.landmark_set.get('68') * numpy.array([ scale_x, scale_y ]),
+		'68/5': target_face.landmark_set.get('68/5') * numpy.array([ scale_x, scale_y ])
+	}
+
+	return target_face._replace(
+		bounding_box = bounding_box,
+		landmark_set = landmark_set
+	)

facefusion/face_classifier.py

@@ -11,12 +11,18 @@ from facefusion.thread_helper import conditional_thread_semaphore
 from facefusion.types import Age, DownloadScope, FaceLandmark5, Gender, InferencePool, ModelOptions, ModelSet, Race, VisionFrame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'fairface':
 		{
+			'__metadata__':
+			{
+				'vendor': 'dchen236',
+				'license': 'Non-Commercial',
+				'year': 2021
+			},
 			'hashes':
 			{
 				'face_classifier':

facefusion/face_detector.py

@@ -6,19 +6,25 @@ import numpy
 
 from facefusion import inference_manager, state_manager
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
-from facefusion.face_helper import create_rotated_matrix_and_size, create_static_anchors, distance_to_bounding_box, distance_to_face_landmark_5, normalize_bounding_box, transform_bounding_box, transform_points
+from facefusion.face_helper import create_rotation_matrix_and_size, create_static_anchors, distance_to_bounding_box, distance_to_face_landmark_5, normalize_bounding_box, transform_bounding_box, transform_points
 from facefusion.filesystem import resolve_relative_path
 from facefusion.thread_helper import thread_semaphore
-from facefusion.types import Angle, BoundingBox, Detection, DownloadScope, DownloadSet, FaceLandmark5, InferencePool, ModelSet, Score, VisionFrame
+from facefusion.types import Angle, BoundingBox, Detection, DownloadScope, DownloadSet, FaceLandmark5, InferencePool, Margin, ModelSet, Score, VisionFrame
 from facefusion.vision import restrict_frame, unpack_resolution
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'retinaface':
 		{
+			'__metadata__':
+			{
+				'vendor': 'InsightFace',
+				'license': 'Non-Commercial',
+				'year': 2020
+			},
 			'hashes':
 			{
 				'retinaface':
@@ -38,6 +44,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'scrfd':
 		{
+			'__metadata__':
+			{
+				'vendor': 'InsightFace',
+				'license': 'Non-Commercial',
+				'year': 2021
+			},
 			'hashes':
 			{
 				'scrfd':
@@ -57,6 +69,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'yolo_face':
 		{
+			'__metadata__':
+			{
+				'vendor': 'derronqi',
+				'license': 'GPL-3.0',
+				'year': 2022
+			},
 			'hashes':
 			{
 				'yolo_face':
@@ -73,6 +91,31 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 					'path': resolve_relative_path('../.assets/models/yoloface_8n.onnx')
 				}
 			}
+		},
+		'yunet':
+		{
+			'__metadata__':
+			{
+				'vendor': 'OpenCV',
+				'license': 'MIT',
+				'year': 2023
+			},
+			'hashes':
+			{
+				'yunet':
+				{
+					'url': resolve_download_url('models-3.4.0', 'yunet_2023_mar.hash'),
+					'path': resolve_relative_path('../.assets/models/yunet_2023_mar.hash')
+				}
+			},
+			'sources':
+			{
+				'yunet':
+				{
+					'url': resolve_download_url('models-3.4.0', 'yunet_2023_mar.onnx'),
+					'path': resolve_relative_path('../.assets/models/yunet_2023_mar.onnx')
+				}
+			}
 		}
 	}
 
@@ -94,7 +137,7 @@ def collect_model_downloads() -> Tuple[DownloadSet, DownloadSet]:
 	model_hash_set = {}
 	model_source_set = {}
 
-	for face_detector_model in [ 'retinaface', 'scrfd', 'yolo_face' ]:
+	for face_detector_model in [ 'retinaface', 'scrfd', 'yolo_face', 'yunet' ]:
 		if state_manager.get_item('face_detector_model') in [ 'many', face_detector_model ]:
 			model_hash_set[face_detector_model] = model_set.get(face_detector_model).get('hashes').get(face_detector_model)
 			model_source_set[face_detector_model] = model_set.get(face_detector_model).get('sources').get(face_detector_model)
@@ -109,39 +152,56 @@ def pre_check() -> bool:
 
 
 def detect_faces(vision_frame : VisionFrame) -> Tuple[List[BoundingBox], List[Score], List[FaceLandmark5]]:
+	margin_top, margin_right, margin_bottom, margin_left = prepare_margin(vision_frame)
+	margin_vision_frame = numpy.pad(vision_frame, ((margin_top, margin_bottom), (margin_left, margin_right), (0, 0)))
 	all_bounding_boxes : List[BoundingBox] = []
 	all_face_scores : List[Score] = []
 	all_face_landmarks_5 : List[FaceLandmark5] = []
 
 	if state_manager.get_item('face_detector_model') in [ 'many', 'retinaface' ]:
-		bounding_boxes, face_scores, face_landmarks_5 = detect_with_retinaface(vision_frame, state_manager.get_item('face_detector_size'))
+		bounding_boxes, face_scores, face_landmarks_5 = detect_with_retinaface(margin_vision_frame, state_manager.get_item('face_detector_size'))
 		all_bounding_boxes.extend(bounding_boxes)
 		all_face_scores.extend(face_scores)
 		all_face_landmarks_5.extend(face_landmarks_5)
 
 	if state_manager.get_item('face_detector_model') in [ 'many', 'scrfd' ]:
-		bounding_boxes, face_scores, face_landmarks_5 = detect_with_scrfd(vision_frame, state_manager.get_item('face_detector_size'))
+		bounding_boxes, face_scores, face_landmarks_5 = detect_with_scrfd(margin_vision_frame, state_manager.get_item('face_detector_size'))
 		all_bounding_boxes.extend(bounding_boxes)
 		all_face_scores.extend(face_scores)
 		all_face_landmarks_5.extend(face_landmarks_5)
 
 	if state_manager.get_item('face_detector_model') in [ 'many', 'yolo_face' ]:
-		bounding_boxes, face_scores, face_landmarks_5 = detect_with_yolo_face(vision_frame, state_manager.get_item('face_detector_size'))
+		bounding_boxes, face_scores, face_landmarks_5 = detect_with_yolo_face(margin_vision_frame, state_manager.get_item('face_detector_size'))
 		all_bounding_boxes.extend(bounding_boxes)
 		all_face_scores.extend(face_scores)
 		all_face_landmarks_5.extend(face_landmarks_5)
 
-	all_bounding_boxes = [ normalize_bounding_box(all_bounding_box) for all_bounding_box in all_bounding_boxes ]
+	if state_manager.get_item('face_detector_model') == 'yunet':
+		bounding_boxes, face_scores, face_landmarks_5 = detect_with_yunet(margin_vision_frame, state_manager.get_item('face_detector_size'))
+		all_bounding_boxes.extend(bounding_boxes)
+		all_face_scores.extend(face_scores)
+		all_face_landmarks_5.extend(face_landmarks_5)
+
+	all_bounding_boxes = [ normalize_bounding_box(all_bounding_box) - numpy.array([ margin_left, margin_top, margin_left, margin_top ]) for all_bounding_box in all_bounding_boxes ]
+	all_face_landmarks_5 = [ all_face_landmark_5 - numpy.array([ margin_left, margin_top ]) for all_face_landmark_5 in all_face_landmarks_5 ]
 	return all_bounding_boxes, all_face_scores, all_face_landmarks_5
 
 
-def detect_rotated_faces(vision_frame : VisionFrame, angle : Angle) -> Tuple[List[BoundingBox], List[Score], List[FaceLandmark5]]:
-	rotated_matrix, rotated_size = create_rotated_matrix_and_size(angle, vision_frame.shape[:2][::-1])
-	rotated_vision_frame = cv2.warpAffine(vision_frame, rotated_matrix, rotated_size)
-	rotated_inverse_matrix = cv2.invertAffineTransform(rotated_matrix)
-	bounding_boxes, face_scores, face_landmarks_5 = detect_faces(rotated_vision_frame)
-	bounding_boxes = [ transform_bounding_box(bounding_box, rotated_inverse_matrix) for bounding_box in bounding_boxes ]
-	face_landmarks_5 = [ transform_points(face_landmark_5, rotated_inverse_matrix) for face_landmark_5 in face_landmarks_5 ]
+def prepare_margin(vision_frame : VisionFrame) -> Margin:
+	margin_top = int(vision_frame.shape[0] * numpy.interp(state_manager.get_item('face_detector_margin')[0], [ 0, 100 ], [ 0, 0.5 ]))
+	margin_right = int(vision_frame.shape[1] * numpy.interp(state_manager.get_item('face_detector_margin')[1], [ 0, 100 ], [ 0, 0.5 ]))
+	margin_bottom = int(vision_frame.shape[0] * numpy.interp(state_manager.get_item('face_detector_margin')[2], [ 0, 100 ], [ 0, 0.5 ]))
+	margin_left = int(vision_frame.shape[1] * numpy.interp(state_manager.get_item('face_detector_margin')[3], [ 0, 100 ], [ 0, 0.5 ]))
+	return margin_top, margin_right, margin_bottom, margin_left
+
+
+def detect_faces_by_angle(vision_frame : VisionFrame, face_angle : Angle) -> Tuple[List[BoundingBox], List[Score], List[FaceLandmark5]]:
+	rotation_matrix, rotation_size = create_rotation_matrix_and_size(face_angle, vision_frame.shape[:2][::-1])
+	rotation_vision_frame = cv2.warpAffine(vision_frame, rotation_matrix, rotation_size)
+	rotation_inverse_matrix = cv2.invertAffineTransform(rotation_matrix)
+	bounding_boxes, face_scores, face_landmarks_5 = detect_faces(rotation_vision_frame)
+	bounding_boxes = [ transform_bounding_box(bounding_box, rotation_inverse_matrix) for bounding_box in bounding_boxes ]
+	face_landmarks_5 = [ transform_points(face_landmark_5, rotation_inverse_matrix) for face_landmark_5 in face_landmarks_5 ]
 	return bounding_boxes, face_scores, face_landmarks_5
 
 
@@ -162,7 +222,8 @@ def detect_with_retinaface(vision_frame : VisionFrame, face_detector_size : str)
 	detection = forward_with_retinaface(detect_vision_frame)
 
 	for index, feature_stride in enumerate(feature_strides):
-		keep_indices = numpy.where(detection[index] >= face_detector_score)[0]
+		face_scores_raw = detection[index]
+		keep_indices = numpy.where(face_scores_raw >= face_detector_score)[0]
 
 		if numpy.any(keep_indices):
 			stride_height = face_detector_height // feature_stride
@@ -180,7 +241,7 @@ def detect_with_retinaface(vision_frame : VisionFrame, face_detector_size : str)
 					bounding_box_raw[3] * ratio_height
 				]))
 
-			for face_score_raw in detection[index][keep_indices]:
+			for face_score_raw in face_scores_raw[keep_indices]:
 				face_scores.append(face_score_raw[0])
 
 			for face_landmark_raw_5 in distance_to_face_landmark_5(anchors, face_landmarks_5_raw)[keep_indices]:
@@ -206,7 +267,8 @@ def detect_with_scrfd(vision_frame : VisionFrame, face_detector_size : str) -> T
 	detection = forward_with_scrfd(detect_vision_frame)
 
 	for index, feature_stride in enumerate(feature_strides):
-		keep_indices = numpy.where(detection[index] >= face_detector_score)[0]
+		face_scores_raw = detection[index]
+		keep_indices = numpy.where(face_scores_raw >= face_detector_score)[0]
 
 		if numpy.any(keep_indices):
 			stride_height = face_detector_height // feature_stride
@@ -224,7 +286,7 @@ def detect_with_scrfd(vision_frame : VisionFrame, face_detector_size : str) -> T
 					bounding_box_raw[3] * ratio_height
 				]))
 
-			for face_score_raw in detection[index][keep_indices]:
+			for face_score_raw in face_scores_raw[keep_indices]:
 				face_scores.append(face_score_raw[0])
 
 			for face_landmark_raw_5 in distance_to_face_landmark_5(anchors, face_landmarks_5_raw)[keep_indices]:
@@ -271,6 +333,67 @@ def detect_with_yolo_face(vision_frame : VisionFrame, face_detector_size : str)
 	return bounding_boxes, face_scores, face_landmarks_5
 
 
+def detect_with_yunet(vision_frame : VisionFrame, face_detector_size : str) -> Tuple[List[BoundingBox], List[Score], List[FaceLandmark5]]:
+	bounding_boxes = []
+	face_scores = []
+	face_landmarks_5 = []
+	feature_strides = [ 8, 16, 32 ]
+	feature_map_channel = 3
+	anchor_total = 1
+	face_detector_score = state_manager.get_item('face_detector_score')
+	face_detector_width, face_detector_height = unpack_resolution(face_detector_size)
+	temp_vision_frame = restrict_frame(vision_frame, (face_detector_width, face_detector_height))
+	ratio_height = vision_frame.shape[0] / temp_vision_frame.shape[0]
+	ratio_width = vision_frame.shape[1] / temp_vision_frame.shape[1]
+	detect_vision_frame = prepare_detect_frame(temp_vision_frame, face_detector_size)
+	detect_vision_frame = normalize_detect_frame(detect_vision_frame, [ 0, 255 ])
+	detection = forward_with_yunet(detect_vision_frame)
+
+	for index, feature_stride in enumerate(feature_strides):
+		face_scores_raw = (detection[index] * detection[index + feature_map_channel]).reshape(-1)
+		keep_indices = numpy.where(face_scores_raw >= face_detector_score)[0]
+
+		if numpy.any(keep_indices):
+			stride_height = face_detector_height // feature_stride
+			stride_width = face_detector_width // feature_stride
+			anchors = create_static_anchors(feature_stride, anchor_total, stride_height, stride_width)
+			bounding_boxes_center = detection[index + feature_map_channel * 2].squeeze(0)[:, :2] * feature_stride + anchors
+			bounding_boxes_size = numpy.exp(detection[index + feature_map_channel * 2].squeeze(0)[:, 2:4]) * feature_stride
+			face_landmarks_5_raw = detection[index + feature_map_channel * 3].squeeze(0)
+
+			bounding_boxes_raw = numpy.stack(
+			[
+				bounding_boxes_center[:, 0] - bounding_boxes_size[:, 0] / 2,
+				bounding_boxes_center[:, 1] - bounding_boxes_size[:, 1] / 2,
+				bounding_boxes_center[:, 0] + bounding_boxes_size[:, 0] / 2,
+				bounding_boxes_center[:, 1] + bounding_boxes_size[:, 1] / 2
+			], axis = -1)
+
+			for bounding_box_raw in bounding_boxes_raw[keep_indices]:
+				bounding_boxes.append(numpy.array(
+				[
+					bounding_box_raw[0] * ratio_width,
+					bounding_box_raw[1] * ratio_height,
+					bounding_box_raw[2] * ratio_width,
+					bounding_box_raw[3] * ratio_height
+				]))
+
+			face_scores.extend(face_scores_raw[keep_indices])
+			face_landmarks_5_raw = numpy.concatenate(
+			[
+				face_landmarks_5_raw[:, [0, 1]] * feature_stride + anchors,
+				face_landmarks_5_raw[:, [2, 3]] * feature_stride + anchors,
+				face_landmarks_5_raw[:, [4, 5]] * feature_stride + anchors,
+				face_landmarks_5_raw[:, [6, 7]] * feature_stride + anchors,
+				face_landmarks_5_raw[:, [8, 9]] * feature_stride + anchors
+			], axis = -1).reshape(-1, 5, 2)
+
+			for face_landmark_raw_5 in face_landmarks_5_raw[keep_indices]:
+				face_landmarks_5.append(face_landmark_raw_5 * [ ratio_width, ratio_height ])
+
+	return bounding_boxes, face_scores, face_landmarks_5
+
+
 def forward_with_retinaface(detect_vision_frame : VisionFrame) -> Detection:
 	face_detector = get_inference_pool().get('retinaface')
 
@@ -307,6 +430,18 @@ def forward_with_yolo_face(detect_vision_frame : VisionFrame) -> Detection:
 	return detection
 
 
+def forward_with_yunet(detect_vision_frame : VisionFrame) -> Detection:
+	face_detector = get_inference_pool().get('yunet')
+
+	with thread_semaphore():
+		detection = face_detector.run(None,
+		{
+			'input': detect_vision_frame
+		})
+
+	return detection
+
+
 def prepare_detect_frame(temp_vision_frame : VisionFrame, face_detector_size : str) -> VisionFrame:
 	face_detector_width, face_detector_height = unpack_resolution(face_detector_size)
 	detect_vision_frame = numpy.zeros((face_detector_height, face_detector_width, 3))

facefusion/face_helper.py

@@ -69,8 +69,8 @@ WARP_TEMPLATE_SET : WarpTemplateSet =\
 
 
 def estimate_matrix_by_face_landmark_5(face_landmark_5 : FaceLandmark5, warp_template : WarpTemplate, crop_size : Size) -> Matrix:
-	normed_warp_template = WARP_TEMPLATE_SET.get(warp_template) * crop_size
-	affine_matrix = cv2.estimateAffinePartial2D(face_landmark_5, normed_warp_template, method = cv2.RANSAC, ransacReprojThreshold = 100)[0]
+	warp_template_norm = WARP_TEMPLATE_SET.get(warp_template) * crop_size
+	affine_matrix = cv2.estimateAffinePartial2D(face_landmark_5, warp_template_norm, method = cv2.RANSAC, ransacReprojThreshold = 100)[0]
 	return affine_matrix
 
 
@@ -98,59 +98,59 @@ def warp_face_by_translation(temp_vision_frame : VisionFrame, translation : Tran
 	return crop_vision_frame, affine_matrix
 
 
-def paste_back(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame, crop_mask : Mask, affine_matrix : Matrix) -> VisionFrame:
-	paste_bounding_box, paste_matrix = calc_paste_area(temp_vision_frame, crop_vision_frame, affine_matrix)
-	x_min, y_min, x_max, y_max = paste_bounding_box
-	paste_width = x_max - x_min
-	paste_height = y_max - y_min
-	inverse_mask = cv2.warpAffine(crop_mask, paste_matrix, (paste_width, paste_height)).clip(0, 1)
-	inverse_mask = numpy.expand_dims(inverse_mask, axis = -1)
+def paste_back(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame, crop_vision_mask : Mask, affine_matrix : Matrix) -> VisionFrame:
+	paste_bounding_box, paste_matrix = calculate_paste_area(temp_vision_frame, crop_vision_frame, affine_matrix)
+	x1, y1, x2, y2 = paste_bounding_box
+	paste_width = x2 - x1
+	paste_height = y2 - y1
+	inverse_vision_mask = cv2.warpAffine(crop_vision_mask, paste_matrix, (paste_width, paste_height)).clip(0, 1)
+	inverse_vision_mask = numpy.expand_dims(inverse_vision_mask, axis = -1)
 	inverse_vision_frame = cv2.warpAffine(crop_vision_frame, paste_matrix, (paste_width, paste_height), borderMode = cv2.BORDER_REPLICATE)
 	temp_vision_frame = temp_vision_frame.copy()
-	paste_vision_frame = temp_vision_frame[y_min:y_max, x_min:x_max]
-	paste_vision_frame = paste_vision_frame * (1 - inverse_mask) + inverse_vision_frame * inverse_mask
-	temp_vision_frame[y_min:y_max, x_min:x_max] = paste_vision_frame.astype(temp_vision_frame.dtype)
+	paste_vision_frame = temp_vision_frame[y1:y2, x1:x2]
+	paste_vision_frame = paste_vision_frame * (1 - inverse_vision_mask) + inverse_vision_frame * inverse_vision_mask
+	temp_vision_frame[y1:y2, x1:x2] = paste_vision_frame.astype(temp_vision_frame.dtype)
 	return temp_vision_frame
 
 
-def calc_paste_area(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame, affine_matrix : Matrix) -> Tuple[BoundingBox, Matrix]:
+def calculate_paste_area(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame, affine_matrix : Matrix) -> Tuple[BoundingBox, Matrix]:
 	temp_height, temp_width = temp_vision_frame.shape[:2]
 	crop_height, crop_width = crop_vision_frame.shape[:2]
 	inverse_matrix = cv2.invertAffineTransform(affine_matrix)
 	crop_points = numpy.array([ [ 0, 0 ], [ crop_width, 0 ], [ crop_width, crop_height ], [ 0, crop_height ] ])
 	paste_region_points = transform_points(crop_points, inverse_matrix)
-	min_point = numpy.floor(paste_region_points.min(axis = 0)).astype(int)
-	max_point = numpy.ceil(paste_region_points.max(axis = 0)).astype(int)
-	x_min, y_min = numpy.clip(min_point, 0, [ temp_width, temp_height ])
-	x_max, y_max = numpy.clip(max_point, 0, [ temp_width, temp_height ])
-	paste_bounding_box = numpy.array([ x_min, y_min, x_max, y_max ])
+	paste_region_point_min = numpy.floor(paste_region_points.min(axis = 0)).astype(int)
+	paste_region_point_max = numpy.ceil(paste_region_points.max(axis = 0)).astype(int)
+	x1, y1 = numpy.clip(paste_region_point_min, 0, [ temp_width, temp_height ])
+	x2, y2 = numpy.clip(paste_region_point_max, 0, [ temp_width, temp_height ])
+	paste_bounding_box = numpy.array([ x1, y1, x2, y2 ])
 	paste_matrix = inverse_matrix.copy()
-	paste_matrix[0, 2] -= x_min
-	paste_matrix[1, 2] -= y_min
+	paste_matrix[0, 2] -= x1
+	paste_matrix[1, 2] -= y1
 	return paste_bounding_box, paste_matrix
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_anchors(feature_stride : int, anchor_total : int, stride_height : int, stride_width : int) -> Anchors:
-	y, x = numpy.mgrid[:stride_height, :stride_width][::-1]
+	x, y = numpy.mgrid[:stride_width, :stride_height]
 	anchors = numpy.stack((y, x), axis = -1)
 	anchors = (anchors * feature_stride).reshape((-1, 2))
 	anchors = numpy.stack([ anchors ] * anchor_total, axis = 1).reshape((-1, 2))
 	return anchors
 
 
-def create_rotated_matrix_and_size(angle : Angle, size : Size) -> Tuple[Matrix, Size]:
-	rotated_matrix = cv2.getRotationMatrix2D((size[0] / 2, size[1] / 2), angle, 1)
-	rotated_size = numpy.dot(numpy.abs(rotated_matrix[:, :2]), size)
-	rotated_matrix[:, -1] += (rotated_size - size) * 0.5 #type:ignore[misc]
-	rotated_size = int(rotated_size[0]), int(rotated_size[1])
-	return rotated_matrix, rotated_size
+def create_rotation_matrix_and_size(angle : Angle, size : Size) -> Tuple[Matrix, Size]:
+	rotation_matrix = cv2.getRotationMatrix2D((size[0] / 2, size[1] / 2), angle, 1)
+	rotation_size = numpy.dot(numpy.abs(rotation_matrix[:, :2]), size)
+	rotation_matrix[:, -1] += (rotation_size - size) * 0.5 #type:ignore[misc]
+	rotation_size = int(rotation_size[0]), int(rotation_size[1])
+	return rotation_matrix, rotation_size
 
 
 def create_bounding_box(face_landmark_68 : FaceLandmark68) -> BoundingBox:
-	min_x, min_y = numpy.min(face_landmark_68, axis = 0)
-	max_x, max_y = numpy.max(face_landmark_68, axis = 0)
-	bounding_box = normalize_bounding_box(numpy.array([ min_x, min_y, max_x, max_y ]))
+	x1, y1 = numpy.min(face_landmark_68, axis = 0)
+	x2, y2 = numpy.max(face_landmark_68, axis = 0)
+	bounding_box = normalize_bounding_box(numpy.array([ x1, y1, x2, y2 ]))
 	return bounding_box
 
 
@@ -229,8 +229,8 @@ def estimate_face_angle(face_landmark_68 : FaceLandmark68) -> Angle:
 
 
 def apply_nms(bounding_boxes : List[BoundingBox], scores : List[Score], score_threshold : float, nms_threshold : float) -> Sequence[int]:
-	normed_bounding_boxes = [ (x1, y1, x2 - x1, y2 - y1) for (x1, y1, x2, y2) in bounding_boxes ]
-	keep_indices = cv2.dnn.NMSBoxes(normed_bounding_boxes, scores, score_threshold = score_threshold, nms_threshold = nms_threshold)
+	bounding_boxes_norm = [ (x1, y1, x2 - x1, y2 - y1) for (x1, y1, x2, y2) in bounding_boxes ]
+	keep_indices = cv2.dnn.NMSBoxes(bounding_boxes_norm, scores, score_threshold = score_threshold, nms_threshold = nms_threshold)
 	return keep_indices
 
 
@@ -246,9 +246,11 @@ def get_nms_threshold(face_detector_model : FaceDetectorModel, face_detector_ang
 	return 0.4
 
 
-def merge_matrix(matrices : List[Matrix]) -> Matrix:
-	merged_matrix = numpy.vstack([ matrices[0], [ 0, 0, 1 ] ])
-	for matrix in matrices[1:]:
-		matrix = numpy.vstack([ matrix, [ 0, 0, 1 ] ])
-		merged_matrix = numpy.dot(merged_matrix, matrix)
-	return merged_matrix[:2, :]
+def merge_matrix(temp_matrices : List[Matrix]) -> Matrix:
+	matrix = numpy.vstack([temp_matrices[0], [0, 0, 1]])
+
+	for temp_matrix in temp_matrices[1:]:
+		temp_matrix = numpy.vstack([ temp_matrix, [ 0, 0, 1 ] ])
+		matrix = numpy.dot(temp_matrix, matrix)
+
+	return matrix[:2, :]

facefusion/face_landmarker.py

@@ -6,18 +6,24 @@ import numpy
 
 from facefusion import inference_manager, state_manager
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
-from facefusion.face_helper import create_rotated_matrix_and_size, estimate_matrix_by_face_landmark_5, transform_points, warp_face_by_translation
+from facefusion.face_helper import create_rotation_matrix_and_size, estimate_matrix_by_face_landmark_5, transform_points, warp_face_by_translation
 from facefusion.filesystem import resolve_relative_path
 from facefusion.thread_helper import conditional_thread_semaphore
 from facefusion.types import Angle, BoundingBox, DownloadScope, DownloadSet, FaceLandmark5, FaceLandmark68, InferencePool, ModelSet, Prediction, Score, VisionFrame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'2dfan4':
 		{
+			'__metadata__':
+			{
+				'vendor': 'breadbread1984',
+				'license': 'MIT',
+				'year': 2018
+			},
 			'hashes':
 			{
 				'2dfan4':
@@ -38,6 +44,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'peppa_wutz':
 		{
+			'__metadata__':
+			{
+				'vendor': 'Unknown',
+				'license': 'Apache-2.0',
+				'year': 2023
+			},
 			'hashes':
 			{
 				'peppa_wutz':
@@ -58,6 +70,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'fan_68_5':
 		{
+			'__metadata__':
+			{
+				'vendor': 'FaceFusion',
+				'license': 'OpenRAIL-M',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'fan_68_5':
@@ -136,14 +154,14 @@ def detect_with_2dfan4(temp_vision_frame: VisionFrame, bounding_box: BoundingBox
 	model_size = create_static_model_set('full').get('2dfan4').get('size')
 	scale = 195 / numpy.subtract(bounding_box[2:], bounding_box[:2]).max().clip(1, None)
 	translation = (model_size[0] - numpy.add(bounding_box[2:], bounding_box[:2]) * scale) * 0.5
-	rotated_matrix, rotated_size = create_rotated_matrix_and_size(face_angle, model_size)
+	rotation_matrix, rotation_size = create_rotation_matrix_and_size(face_angle, model_size)
 	crop_vision_frame, affine_matrix = warp_face_by_translation(temp_vision_frame, translation, scale, model_size)
-	crop_vision_frame = cv2.warpAffine(crop_vision_frame, rotated_matrix, rotated_size)
+	crop_vision_frame = cv2.warpAffine(crop_vision_frame, rotation_matrix, rotation_size)
 	crop_vision_frame = conditional_optimize_contrast(crop_vision_frame)
 	crop_vision_frame = crop_vision_frame.transpose(2, 0, 1).astype(numpy.float32) / 255.0
 	face_landmark_68, face_heatmap = forward_with_2dfan4(crop_vision_frame)
 	face_landmark_68 = face_landmark_68[:, :, :2][0] / 64 * 256
-	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(rotated_matrix))
+	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(rotation_matrix))
 	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(affine_matrix))
 	face_landmark_score_68 = numpy.amax(face_heatmap, axis = (2, 3))
 	face_landmark_score_68 = numpy.mean(face_landmark_score_68)
@@ -155,15 +173,15 @@ def detect_with_peppa_wutz(temp_vision_frame : VisionFrame, bounding_box : Bound
 	model_size = create_static_model_set('full').get('peppa_wutz').get('size')
 	scale = 195 / numpy.subtract(bounding_box[2:], bounding_box[:2]).max().clip(1, None)
 	translation = (model_size[0] - numpy.add(bounding_box[2:], bounding_box[:2]) * scale) * 0.5
-	rotated_matrix, rotated_size = create_rotated_matrix_and_size(face_angle, model_size)
+	rotation_matrix, rotation_size = create_rotation_matrix_and_size(face_angle, model_size)
 	crop_vision_frame, affine_matrix = warp_face_by_translation(temp_vision_frame, translation, scale, model_size)
-	crop_vision_frame = cv2.warpAffine(crop_vision_frame, rotated_matrix, rotated_size)
+	crop_vision_frame = cv2.warpAffine(crop_vision_frame, rotation_matrix, rotation_size)
 	crop_vision_frame = conditional_optimize_contrast(crop_vision_frame)
 	crop_vision_frame = crop_vision_frame.transpose(2, 0, 1).astype(numpy.float32) / 255.0
 	crop_vision_frame = numpy.expand_dims(crop_vision_frame, axis = 0)
 	prediction = forward_with_peppa_wutz(crop_vision_frame)
 	face_landmark_68 = prediction.reshape(-1, 3)[:, :2] / 64 * model_size[0]
-	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(rotated_matrix))
+	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(rotation_matrix))
 	face_landmark_68 = transform_points(face_landmark_68, cv2.invertAffineTransform(affine_matrix))
 	face_landmark_score_68 = prediction.reshape(-1, 3)[:, 2].mean()
 	face_landmark_score_68 = numpy.interp(face_landmark_score_68, [ 0, 0.95 ], [ 0, 1 ])

facefusion/face_masker.py

@@ -12,12 +12,18 @@ from facefusion.thread_helper import conditional_thread_semaphore
 from facefusion.types import DownloadScope, DownloadSet, FaceLandmark68, FaceMaskArea, FaceMaskRegion, InferencePool, Mask, ModelSet, Padding, VisionFrame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'xseg_1':
 		{
+			'__metadata__':
+			{
+				'vendor': 'DeepFaceLab',
+				'license': 'GPL-3.0',
+				'year': 2021
+			},
 			'hashes':
 			{
 				'face_occluder':
@@ -38,6 +44,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'xseg_2':
 		{
+			'__metadata__':
+			{
+				'vendor': 'DeepFaceLab',
+				'license': 'GPL-3.0',
+				'year': 2021
+			},
 			'hashes':
 			{
 				'face_occluder':
@@ -58,6 +70,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'xseg_3':
 		{
+			'__metadata__':
+			{
+				'vendor': 'DeepFaceLab',
+				'license': 'GPL-3.0',
+				'year': 2021
+			},
 			'hashes':
 			{
 				'face_occluder':
@@ -78,6 +96,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'bisenet_resnet_18':
 		{
+			'__metadata__':
+			{
+				'vendor': 'yakhyo',
+				'license': 'MIT',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'face_parser':
@@ -98,6 +122,12 @@ def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 		},
 		'bisenet_resnet_34':
 		{
+			'__metadata__':
+			{
+				'vendor': 'yakhyo',
+				'license': 'MIT',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'face_parser':
@@ -137,7 +167,7 @@ def collect_model_downloads() -> Tuple[DownloadSet, DownloadSet]:
 	model_source_set = {}
 
 	for face_occluder_model in [ 'xseg_1', 'xseg_2', 'xseg_3' ]:
-		if state_manager.get_item('face_occluder_model') == face_occluder_model:
+		if state_manager.get_item('face_occluder_model') in [ 'many', face_occluder_model ]:
 			model_hash_set[face_occluder_model] = model_set.get(face_occluder_model).get('hashes').get('face_occluder')
 			model_source_set[face_occluder_model] = model_set.get(face_occluder_model).get('sources').get('face_occluder')
 
@@ -171,14 +201,24 @@ def create_box_mask(crop_vision_frame : VisionFrame, face_mask_blur : float, fac
 
 
 def create_occlusion_mask(crop_vision_frame : VisionFrame) -> Mask:
-	model_name = state_manager.get_item('face_occluder_model')
-	model_size = create_static_model_set('full').get(model_name).get('size')
-	prepare_vision_frame = cv2.resize(crop_vision_frame, model_size)
-	prepare_vision_frame = numpy.expand_dims(prepare_vision_frame, axis = 0).astype(numpy.float32) / 255.0
-	prepare_vision_frame = prepare_vision_frame.transpose(0, 1, 2, 3)
-	occlusion_mask = forward_occlude_face(prepare_vision_frame)
-	occlusion_mask = occlusion_mask.transpose(0, 1, 2).clip(0, 1).astype(numpy.float32)
-	occlusion_mask = cv2.resize(occlusion_mask, crop_vision_frame.shape[:2][::-1])
+	temp_masks = []
+
+	if state_manager.get_item('face_occluder_model') == 'many':
+		model_names = [ 'xseg_1', 'xseg_2', 'xseg_3' ]
+	else:
+		model_names = [ state_manager.get_item('face_occluder_model') ]
+
+	for model_name in model_names:
+		model_size = create_static_model_set('full').get(model_name).get('size')
+		prepare_vision_frame = cv2.resize(crop_vision_frame, model_size)
+		prepare_vision_frame = numpy.expand_dims(prepare_vision_frame, axis = 0).astype(numpy.float32) / 255.0
+		prepare_vision_frame = prepare_vision_frame.transpose(0, 1, 2, 3)
+		temp_mask = forward_occlude_face(prepare_vision_frame, model_name)
+		temp_mask = temp_mask.transpose(0, 1, 2).clip(0, 1).astype(numpy.float32)
+		temp_mask = cv2.resize(temp_mask, crop_vision_frame.shape[:2][::-1])
+		temp_masks.append(temp_mask)
+
+	occlusion_mask = numpy.minimum.reduce(temp_masks)
 	occlusion_mask = (cv2.GaussianBlur(occlusion_mask.clip(0, 1), (0, 0), 5).clip(0.5, 1) - 0.5) * 2
 	return occlusion_mask
 
@@ -214,8 +254,7 @@ def create_region_mask(crop_vision_frame : VisionFrame, face_mask_regions : List
 	return region_mask
 
 
-def forward_occlude_face(prepare_vision_frame : VisionFrame) -> Mask:
-	model_name = state_manager.get_item('face_occluder_model')
+def forward_occlude_face(prepare_vision_frame : VisionFrame, model_name : str) -> Mask:
 	face_occluder = get_inference_pool().get(model_name)
 
 	with conditional_thread_semaphore():

facefusion/face_recognizer.py

@@ -11,12 +11,18 @@ from facefusion.thread_helper import conditional_thread_semaphore
 from facefusion.types import DownloadScope, Embedding, FaceLandmark5, InferencePool, ModelOptions, ModelSet, VisionFrame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'arcface':
 		{
+			'__metadata__':
+			{
+				'vendor': 'InsightFace',
+				'license': 'Non-Commercial',
+				'year': 2018
+			},
 			'hashes':
 			{
 				'face_recognizer':
@@ -62,26 +68,26 @@ def pre_check() -> bool:
 	return conditional_download_hashes(model_hash_set) and conditional_download_sources(model_source_set)
 
 
-def calc_embedding(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5) -> Tuple[Embedding, Embedding]:
+def calculate_face_embedding(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5) -> Tuple[Embedding, Embedding]:
 	model_template = get_model_options().get('template')
 	model_size = get_model_options().get('size')
 	crop_vision_frame, matrix = warp_face_by_face_landmark_5(temp_vision_frame, face_landmark_5, model_template, model_size)
 	crop_vision_frame = crop_vision_frame / 127.5 - 1
 	crop_vision_frame = crop_vision_frame[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32)
 	crop_vision_frame = numpy.expand_dims(crop_vision_frame, axis = 0)
-	embedding = forward(crop_vision_frame)
-	embedding = embedding.ravel()
-	normed_embedding = embedding / numpy.linalg.norm(embedding)
-	return embedding, normed_embedding
+	face_embedding = forward(crop_vision_frame)
+	face_embedding = face_embedding.ravel()
+	face_embedding_norm = face_embedding / numpy.linalg.norm(face_embedding)
+	return face_embedding, face_embedding_norm
 
 
 def forward(crop_vision_frame : VisionFrame) -> Embedding:
 	face_recognizer = get_inference_pool().get('face_recognizer')
 
 	with conditional_thread_semaphore():
-		embedding = face_recognizer.run(None,
+		face_embedding = face_recognizer.run(None,
 		{
 			'input': crop_vision_frame
 		})[0]
 
-	return embedding
+	return face_embedding

facefusion/face_selector.py

@@ -3,31 +3,53 @@ from typing import List
 import numpy
 
 from facefusion import state_manager
-from facefusion.types import Face, FaceSelectorOrder, FaceSet, Gender, Race, Score
+from facefusion.face_analyser import get_many_faces, get_one_face
+from facefusion.types import Face, FaceSelectorOrder, Gender, Race, Score, VisionFrame
 
 
-def find_similar_faces(faces : List[Face], reference_faces : FaceSet, face_distance : float) -> List[Face]:
-	similar_faces : List[Face] = []
+def select_faces(reference_vision_frame : VisionFrame, target_vision_frame : VisionFrame) -> List[Face]:
+	target_faces = get_many_faces([ target_vision_frame ])
 
-	if faces and reference_faces:
-		for reference_set in reference_faces:
-			if not similar_faces:
-				for reference_face in reference_faces[reference_set]:
-					for face in faces:
-						if compare_faces(face, reference_face, face_distance):
-							similar_faces.append(face)
-	return similar_faces
+	if state_manager.get_item('face_selector_mode') == 'many':
+		return sort_and_filter_faces(target_faces)
+
+	if state_manager.get_item('face_selector_mode') == 'one':
+		target_face = get_one_face(sort_and_filter_faces(target_faces))
+		if target_face:
+			return [ target_face ]
+
+	if state_manager.get_item('face_selector_mode') == 'reference':
+		reference_faces = get_many_faces([ reference_vision_frame ])
+		reference_faces = sort_and_filter_faces(reference_faces)
+		reference_face = get_one_face(reference_faces, state_manager.get_item('reference_face_position'))
+		if reference_face:
+			match_faces = find_match_faces([ reference_face ], target_faces, state_manager.get_item('reference_face_distance'))
+			return match_faces
+
+	return []
+
+
+def find_match_faces(reference_faces : List[Face], target_faces : List[Face], face_distance : float) -> List[Face]:
+	match_faces : List[Face] = []
+
+	for reference_face in reference_faces:
+		if reference_face:
+			for index, target_face in enumerate(target_faces):
+				if compare_faces(target_face, reference_face, face_distance):
+					match_faces.append(target_faces[index])
+
+	return match_faces
 
 
 def compare_faces(face : Face, reference_face : Face, face_distance : float) -> bool:
-	current_face_distance = calc_face_distance(face, reference_face)
+	current_face_distance = calculate_face_distance(face, reference_face)
 	current_face_distance = float(numpy.interp(current_face_distance, [ 0, 2 ], [ 0, 1 ]))
 	return current_face_distance < face_distance
 
 
-def calc_face_distance(face : Face, reference_face : Face) -> float:
-	if hasattr(face, 'normed_embedding') and hasattr(reference_face, 'normed_embedding'):
-		return 1 - numpy.dot(face.normed_embedding, reference_face.normed_embedding)
+def calculate_face_distance(face : Face, reference_face : Face) -> float:
+	if hasattr(face, 'embedding_norm') and hasattr(reference_face, 'embedding_norm'):
+		return 1 - numpy.dot(face.embedding_norm, reference_face.embedding_norm)
 	return 0
 
 

facefusion/face_store.py

@@ -1,12 +1,11 @@
 from typing import List, Optional
 
 from facefusion.hash_helper import create_hash
-from facefusion.types import Face, FaceSet, FaceStore, VisionFrame
+from facefusion.types import Face, FaceStore, VisionFrame
 
 FACE_STORE : FaceStore =\
 {
-	'static_faces': {},
-	'reference_faces': {}
+	'static_faces': {}
 }
 
 
@@ -27,17 +26,3 @@ def set_static_faces(vision_frame : VisionFrame, faces : List[Face]) -> None:
 
 def clear_static_faces() -> None:
 	FACE_STORE['static_faces'].clear()
-
-
-def get_reference_faces() -> Optional[FaceSet]:
-	return FACE_STORE.get('reference_faces')
-
-
-def append_reference_face(name : str, face : Face) -> None:
-	if name not in FACE_STORE.get('reference_faces'):
-		FACE_STORE['reference_faces'][name] = []
-	FACE_STORE['reference_faces'][name].append(face)
-
-
-def clear_reference_faces() -> None:
-	FACE_STORE['reference_faces'].clear()

facefusion/ffmpeg.py

@@ -7,14 +7,14 @@ from typing import List, Optional, cast
 from tqdm import tqdm
 
 import facefusion.choices
-from facefusion import ffmpeg_builder, logger, process_manager, state_manager, wording
+from facefusion import ffmpeg_builder, logger, process_manager, state_manager, translator
 from facefusion.filesystem import get_file_format, remove_file
 from facefusion.temp_helper import get_temp_file_path, get_temp_frames_pattern
-from facefusion.types import AudioBuffer, AudioEncoder, Commands, EncoderSet, Fps, UpdateProgress, VideoEncoder, VideoFormat
-from facefusion.vision import detect_video_duration, detect_video_fps, predict_video_frame_total
+from facefusion.types import AudioBuffer, AudioEncoder, Command, EncoderSet, Fps, Resolution, UpdateProgress, VideoEncoder, VideoFormat
+from facefusion.vision import detect_video_duration, detect_video_fps, pack_resolution, predict_video_frame_total
 
 
-def run_ffmpeg_with_progress(commands : Commands, update_progress : UpdateProgress) -> subprocess.Popen[bytes]:
+def run_ffmpeg_with_progress(commands : List[Command], update_progress : UpdateProgress) -> subprocess.Popen[bytes]:
 	log_level = state_manager.get_item('log_level')
 	commands.extend(ffmpeg_builder.set_progress())
 	commands.extend(ffmpeg_builder.cast_stream())
@@ -23,8 +23,10 @@ def run_ffmpeg_with_progress(commands : Commands, update_progress : UpdateProgre
 
 	while process_manager.is_processing():
 		try:
-
 			while __line__ := process.stdout.readline().decode().lower():
+				if process_manager.is_stopping():
+					process.terminate()
+
 				if 'frame=' in __line__:
 					_, frame_number = __line__.split('frame=')
 					update_progress(int(frame_number))
@@ -36,8 +38,6 @@ def run_ffmpeg_with_progress(commands : Commands, update_progress : UpdateProgre
 			continue
 		return process
 
-	if process_manager.is_stopping():
-		process.terminate()
 	return process
 
 
@@ -45,7 +45,7 @@ def update_progress(progress : tqdm, frame_number : int) -> None:
 	progress.update(frame_number - progress.n)
 
 
-def run_ffmpeg(commands : Commands) -> subprocess.Popen[bytes]:
+def run_ffmpeg(commands : List[Command]) -> subprocess.Popen[bytes]:
 	log_level = state_manager.get_item('log_level')
 	commands = ffmpeg_builder.run(commands)
 	process = subprocess.Popen(commands, stderr = subprocess.PIPE, stdout = subprocess.PIPE)
@@ -61,10 +61,11 @@ def run_ffmpeg(commands : Commands) -> subprocess.Popen[bytes]:
 
 	if process_manager.is_stopping():
 		process.terminate()
+
 	return process
 
 
-def open_ffmpeg(commands : Commands) -> subprocess.Popen[bytes]:
+def open_ffmpeg(commands : List[Command]) -> subprocess.Popen[bytes]:
 	commands = ffmpeg_builder.run(commands)
 	return subprocess.Popen(commands, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
 
@@ -106,41 +107,59 @@ def get_available_encoder_set() -> EncoderSet:
 	return available_encoder_set
 
 
-def extract_frames(target_path : str, temp_video_resolution : str, temp_video_fps : Fps, trim_frame_start : int, trim_frame_end : int) -> bool:
+def extract_frames(target_path : str, output_path : str, temp_video_resolution : Resolution, temp_video_fps : Fps, trim_frame_start : int, trim_frame_end : int) -> bool:
 	extract_frame_total = predict_video_frame_total(target_path, temp_video_fps, trim_frame_start, trim_frame_end)
-	temp_frames_pattern = get_temp_frames_pattern(target_path, '%08d')
+	temp_frames_pattern = get_temp_frames_pattern(state_manager.get_temp_path(), output_path, state_manager.get_item('temp_frame_format'), '%08d')
 	commands = ffmpeg_builder.chain(
 		ffmpeg_builder.set_input(target_path),
-		ffmpeg_builder.set_media_resolution(temp_video_resolution),
+		ffmpeg_builder.set_media_resolution(pack_resolution(temp_video_resolution)),
 		ffmpeg_builder.set_frame_quality(0),
 		ffmpeg_builder.select_frame_range(trim_frame_start, trim_frame_end, temp_video_fps),
 		ffmpeg_builder.prevent_frame_drop(),
 		ffmpeg_builder.set_output(temp_frames_pattern)
 	)
 
-	with tqdm(total = extract_frame_total, desc = wording.get('extracting'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
+	with tqdm(total = extract_frame_total, desc = translator.get('extracting'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
 		process = run_ffmpeg_with_progress(commands, partial(update_progress, progress))
 		return process.returncode == 0
 
 
-def copy_image(target_path : str, temp_image_resolution : str) -> bool:
-	temp_image_path = get_temp_file_path(target_path)
+def spawn_frames(target_path : str, output_path : str, temp_video_resolution : Resolution, temp_video_fps : Fps, trim_frame_start : int, trim_frame_end : int) -> bool:
+	spawn_frame_total = trim_frame_end - trim_frame_start
+	duration = spawn_frame_total / temp_video_fps
+	temp_frames_pattern = get_temp_frames_pattern(state_manager.get_temp_path(), output_path, state_manager.get_item('temp_frame_format'), '%08d')
+	commands = ffmpeg_builder.chain(
+		ffmpeg_builder.set_loop(),
+		ffmpeg_builder.set_input(target_path),
+		ffmpeg_builder.set_video_duration(duration),
+		ffmpeg_builder.set_video_fps(temp_video_fps),
+		ffmpeg_builder.set_media_resolution(pack_resolution(temp_video_resolution)),
+		ffmpeg_builder.set_output(temp_frames_pattern)
+	)
+
+	with tqdm(total = spawn_frame_total, desc = translator.get('spawning'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
+		process = run_ffmpeg_with_progress(commands, partial(update_progress, progress))
+		return process.returncode == 0
+
+
+def copy_image(target_path : str, output_path : str, temp_image_resolution : Resolution) -> bool:
+	temp_image_path = get_temp_file_path(state_manager.get_temp_path(), output_path)
 	commands = ffmpeg_builder.chain(
 		ffmpeg_builder.set_input(target_path),
-		ffmpeg_builder.set_media_resolution(temp_image_resolution),
+		ffmpeg_builder.set_media_resolution(pack_resolution(temp_image_resolution)),
 		ffmpeg_builder.set_image_quality(target_path, 100),
 		ffmpeg_builder.force_output(temp_image_path)
 	)
 	return run_ffmpeg(commands).returncode == 0
 
 
-def finalize_image(target_path : str, output_path : str, output_image_resolution : str) -> bool:
+def finalize_image(output_path : str, output_image_resolution : Resolution) -> bool:
 	output_image_quality = state_manager.get_item('output_image_quality')
-	temp_image_path = get_temp_file_path(target_path)
+	temp_image_path = get_temp_file_path(state_manager.get_temp_path(), output_path)
 	commands = ffmpeg_builder.chain(
 		ffmpeg_builder.set_input(temp_image_path),
-		ffmpeg_builder.set_media_resolution(output_image_resolution),
-		ffmpeg_builder.set_image_quality(target_path, output_image_quality),
+		ffmpeg_builder.set_media_resolution(pack_resolution(output_image_resolution)),
+		ffmpeg_builder.set_image_quality(output_path, output_image_quality),
 		ffmpeg_builder.force_output(output_path)
 	)
 	return run_ffmpeg(commands).returncode == 0
@@ -168,8 +187,8 @@ def restore_audio(target_path : str, output_path : str, trim_frame_start : int,
 	output_audio_quality = state_manager.get_item('output_audio_quality')
 	output_audio_volume = state_manager.get_item('output_audio_volume')
 	target_video_fps = detect_video_fps(target_path)
-	temp_video_path = get_temp_file_path(target_path)
-	temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
+	temp_video_path = get_temp_file_path(state_manager.get_temp_path(), output_path)
+	temp_video_format = cast(VideoFormat, get_file_format(output_path))
 	temp_video_duration = detect_video_duration(temp_video_path)
 
 	output_audio_encoder = fix_audio_encoder(temp_video_format, output_audio_encoder)
@@ -189,12 +208,12 @@ def restore_audio(target_path : str, output_path : str, trim_frame_start : int,
 	return run_ffmpeg(commands).returncode == 0
 
 
-def replace_audio(target_path : str, audio_path : str, output_path : str) -> bool:
+def replace_audio(audio_path : str, output_path : str) -> bool:
 	output_audio_encoder = state_manager.get_item('output_audio_encoder')
 	output_audio_quality = state_manager.get_item('output_audio_quality')
 	output_audio_volume = state_manager.get_item('output_audio_volume')
-	temp_video_path = get_temp_file_path(target_path)
-	temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
+	temp_video_path = get_temp_file_path(state_manager.get_temp_path(), output_path)
+	temp_video_format = cast(VideoFormat, get_file_format(output_path))
 	temp_video_duration = detect_video_duration(temp_video_path)
 
 	output_audio_encoder = fix_audio_encoder(temp_video_format, output_audio_encoder)
@@ -211,30 +230,33 @@ def replace_audio(target_path : str, audio_path : str, output_path : str) -> boo
 	return run_ffmpeg(commands).returncode == 0
 
 
-def merge_video(target_path : str, temp_video_fps : Fps, output_video_resolution : str, output_video_fps : Fps, trim_frame_start : int, trim_frame_end : int) -> bool:
+def merge_video(target_path : str, output_path : str, temp_video_fps : Fps, output_video_resolution : Resolution, trim_frame_start : int, trim_frame_end : int) -> bool:
+	output_video_fps = state_manager.get_item('output_video_fps')
 	output_video_encoder = state_manager.get_item('output_video_encoder')
 	output_video_quality = state_manager.get_item('output_video_quality')
 	output_video_preset = state_manager.get_item('output_video_preset')
 	merge_frame_total = predict_video_frame_total(target_path, output_video_fps, trim_frame_start, trim_frame_end)
-	temp_video_path = get_temp_file_path(target_path)
-	temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
-	temp_frames_pattern = get_temp_frames_pattern(target_path, '%08d')
+	temp_video_path = get_temp_file_path(state_manager.get_temp_path(), output_path)
+	temp_video_format = cast(VideoFormat, get_file_format(output_path))
+	temp_frames_pattern = get_temp_frames_pattern(state_manager.get_temp_path(), output_path, state_manager.get_item('temp_frame_format'), '%08d')
 
 	output_video_encoder = fix_video_encoder(temp_video_format, output_video_encoder)
 	commands = ffmpeg_builder.chain(
 		ffmpeg_builder.set_input_fps(temp_video_fps),
 		ffmpeg_builder.set_input(temp_frames_pattern),
-		ffmpeg_builder.set_media_resolution(output_video_resolution),
+		ffmpeg_builder.set_media_resolution(pack_resolution(output_video_resolution)),
 		ffmpeg_builder.set_video_encoder(output_video_encoder),
 		ffmpeg_builder.set_video_quality(output_video_encoder, output_video_quality),
 		ffmpeg_builder.set_video_preset(output_video_encoder, output_video_preset),
-		ffmpeg_builder.set_video_fps(output_video_fps),
+		ffmpeg_builder.concat(
+			ffmpeg_builder.set_video_fps(output_video_fps),
+			ffmpeg_builder.keep_video_alpha(output_video_encoder)
+		),
 		ffmpeg_builder.set_pixel_format(output_video_encoder),
-		ffmpeg_builder.set_video_colorspace('bt709'),
 		ffmpeg_builder.force_output(temp_video_path)
 	)
 
-	with tqdm(total = merge_frame_total, desc = wording.get('merging'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
+	with tqdm(total = merge_frame_total, desc = translator.get('merging'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
 		process = run_ffmpeg_with_progress(commands, partial(update_progress, progress))
 		return process.returncode == 0
 
@@ -265,17 +287,19 @@ def concat_video(output_path : str, temp_output_paths : List[str]) -> bool:
 def fix_audio_encoder(video_format : VideoFormat, audio_encoder : AudioEncoder) -> AudioEncoder:
 	if video_format == 'avi' and audio_encoder == 'libopus':
 		return 'aac'
-	if video_format == 'm4v':
+	if video_format in [ 'm4v', 'mpeg', 'wmv' ]:
 		return 'aac'
 	if video_format == 'mov' and audio_encoder in [ 'flac', 'libopus' ]:
 		return 'aac'
+	if video_format == 'mxf':
+		return 'pcm_s16le'
 	if video_format == 'webm':
 		return 'libopus'
 	return audio_encoder
 
 
 def fix_video_encoder(video_format : VideoFormat, video_encoder : VideoEncoder) -> VideoEncoder:
-	if video_format == 'm4v':
+	if video_format in [ 'm4v', 'mpeg', 'mxf', 'wmv' ]:
 		return 'libx264'
 	if video_format in [ 'mkv', 'mp4' ] and video_encoder == 'rawvideo':
 		return 'libx264'

facefusion/ffmpeg_builder.py

@@ -1,54 +1,73 @@
 import itertools
 import shutil
-from typing import Optional
+from typing import List, Optional
 
 import numpy
 
 from facefusion.filesystem import get_file_format
-from facefusion.types import AudioEncoder, Commands, Duration, Fps, StreamMode, VideoEncoder, VideoPreset
+from facefusion.types import AudioEncoder, Command, CommandSet, Duration, Fps, StreamMode, VideoEncoder, VideoPreset
 
 
-def run(commands : Commands) -> Commands:
+def run(commands : List[Command]) -> List[Command]:
 	return [ shutil.which('ffmpeg'), '-loglevel', 'error' ] + commands
 
 
-def chain(*commands : Commands) -> Commands:
+def chain(*commands : List[Command]) -> List[Command]:
 	return list(itertools.chain(*commands))
 
 
-def get_encoders() -> Commands:
+def concat(*__commands__ : List[Command]) -> List[Command]:
+	commands = []
+	command_set : CommandSet = {}
+
+	for command in __commands__:
+		for argument, value in zip(command[::2], command[1::2]):
+			command_set.setdefault(argument, []).append(value)
+
+	for argument, values in command_set.items():
+		commands.append(argument)
+		commands.append(','.join(values))
+
+	return commands
+
+
+def get_encoders() -> List[Command]:
 	return [ '-encoders' ]
 
 
-def set_hardware_accelerator(value : str) -> Commands:
+def set_hardware_accelerator(value : str) -> List[Command]:
 	return [ '-hwaccel', value ]
 
 
-def set_progress() -> Commands:
+def set_progress() -> List[Command]:
 	return [ '-progress' ]
 
 
-def set_input(input_path : str) -> Commands:
+def set_input(input_path : str) -> List[Command]:
 	return [ '-i', input_path ]
 
 
-def set_input_fps(input_fps : Fps) -> Commands:
+def set_input_fps(input_fps : Fps) -> List[Command]:
 	return [ '-r', str(input_fps)]
 
 
-def set_output(output_path : str) -> Commands:
+def set_output(output_path : str) -> List[Command]:
 	return [ output_path ]
 
 
-def force_output(output_path : str) -> Commands:
+def force_output(output_path : str) -> List[Command]:
 	return [ '-y', output_path ]
 
 
-def cast_stream() -> Commands:
+def set_loop() -> List[Command]:
+	return [ '-loop', '1' ]
+
+
+def cast_stream() -> List[Command]:
 	return [ '-' ]
 
 
-def set_stream_mode(stream_mode : StreamMode) -> Commands:
+def set_stream_mode(stream_mode : StreamMode) -> List[Command]:
 	if stream_mode == 'udp':
 		return [ '-f', 'mpegts' ]
 	if stream_mode == 'v4l2':
@@ -56,25 +75,27 @@ def set_stream_mode(stream_mode : StreamMode) -> Commands:
 	return []
 
 
-def set_stream_quality(stream_quality : int) -> Commands:
+def set_stream_quality(stream_quality : int) -> List[Command]:
 	return [ '-b:v', str(stream_quality) + 'k' ]
 
 
-def unsafe_concat() -> Commands:
+def unsafe_concat() -> List[Command]:
 	return [ '-f', 'concat', '-safe', '0' ]
 
 
-def set_pixel_format(video_encoder : VideoEncoder) -> Commands:
+def set_pixel_format(video_encoder : VideoEncoder) -> List[Command]:
 	if video_encoder == 'rawvideo':
 		return [ '-pix_fmt', 'rgb24' ]
+	if video_encoder == 'libvpx-vp9':
+		return [ '-pix_fmt', 'yuva420p' ]
 	return [ '-pix_fmt', 'yuv420p' ]
 
 
-def set_frame_quality(frame_quality : int) -> Commands:
+def set_frame_quality(frame_quality : int) -> List[Command]:
 	return [ '-q:v', str(frame_quality) ]
 
 
-def select_frame_range(frame_start : int, frame_end : int, video_fps : Fps) -> Commands:
+def select_frame_range(frame_start : int, frame_end : int, video_fps : Fps) -> List[Command]:
 	if isinstance(frame_start, int) and isinstance(frame_end, int):
 		return [ '-vf', 'trim=start_frame=' + str(frame_start) + ':end_frame=' + str(frame_end) + ',fps=' + str(video_fps) ]
 	if isinstance(frame_start, int):
@@ -84,11 +105,11 @@ def select_frame_range(frame_start : int, frame_end : int, video_fps : Fps) -> C
 	return [ '-vf', 'fps=' + str(video_fps) ]
 
 
-def prevent_frame_drop() -> Commands:
+def prevent_frame_drop() -> List[Command]:
 	return [ '-vsync', '0' ]
 
 
-def select_media_range(frame_start : int, frame_end : int, media_fps : Fps) -> Commands:
+def select_media_range(frame_start : int, frame_end : int, media_fps : Fps) -> List[Command]:
 	commands = []
 
 	if isinstance(frame_start, int):
@@ -98,35 +119,35 @@ def select_media_range(frame_start : int, frame_end : int, media_fps : Fps) -> C
 	return commands
 
 
-def select_media_stream(media_stream : str) -> Commands:
+def select_media_stream(media_stream : str) -> List[Command]:
 	return [ '-map', media_stream ]
 
 
-def set_media_resolution(video_resolution : str) -> Commands:
+def set_media_resolution(video_resolution : str) -> List[Command]:
 	return [ '-s', video_resolution ]
 
 
-def set_image_quality(image_path : str, image_quality : int) -> Commands:
+def set_image_quality(image_path : str, image_quality : int) -> List[Command]:
 	if get_file_format(image_path) == 'webp':
-		image_compression = image_quality
-	else:
-		image_compression = round(31 - (image_quality * 0.31))
+		return [ '-q:v', str(image_quality) ]
+
+	image_compression = round(31 - (image_quality * 0.31))
 	return [ '-q:v', str(image_compression) ]
 
 
-def set_audio_encoder(audio_codec : str) -> Commands:
+def set_audio_encoder(audio_codec : str) -> List[Command]:
 	return [ '-c:a', audio_codec ]
 
 
-def copy_audio_encoder() -> Commands:
+def copy_audio_encoder() -> List[Command]:
 	return set_audio_encoder('copy')
 
 
-def set_audio_sample_rate(audio_sample_rate : int) -> Commands:
+def set_audio_sample_rate(audio_sample_rate : int) -> List[Command]:
 	return [ '-ar', str(audio_sample_rate) ]
 
 
-def set_audio_sample_size(audio_sample_size : int) -> Commands:
+def set_audio_sample_size(audio_sample_size : int) -> List[Command]:
 	if audio_sample_size == 16:
 		return [ '-f', 's16le' ]
 	if audio_sample_size == 32:
@@ -134,62 +155,62 @@ def set_audio_sample_size(audio_sample_size : int) -> Commands:
 	return []
 
 
-def set_audio_channel_total(audio_channel_total : int) -> Commands:
+def set_audio_channel_total(audio_channel_total : int) -> List[Command]:
 	return [ '-ac', str(audio_channel_total) ]
 
 
-def set_audio_quality(audio_encoder : AudioEncoder, audio_quality : int) -> Commands:
+def set_audio_quality(audio_encoder : AudioEncoder, audio_quality : int) -> List[Command]:
 	if audio_encoder == 'aac':
-		audio_compression = round(numpy.interp(audio_quality, [ 0, 100 ], [ 0.1, 2.0 ]), 1)
+		audio_compression = numpy.round(numpy.interp(audio_quality, [ 0, 100 ], [ 0.1, 2.0 ]), 1).astype(float).item()
 		return [ '-q:a', str(audio_compression) ]
 	if audio_encoder == 'libmp3lame':
-		audio_compression = round(numpy.interp(audio_quality, [ 0, 100 ], [ 9, 0 ]))
+		audio_compression = numpy.round(numpy.interp(audio_quality, [ 0, 100 ], [ 9, 0 ])).astype(int).item()
 		return [ '-q:a', str(audio_compression) ]
 	if audio_encoder == 'libopus':
-		audio_bit_rate = round(numpy.interp(audio_quality, [ 0, 100 ], [ 64, 256 ]))
+		audio_bit_rate = numpy.round(numpy.interp(audio_quality, [ 0, 100 ], [ 64, 256 ])).astype(int).item()
 		return [ '-b:a', str(audio_bit_rate) + 'k' ]
 	if audio_encoder == 'libvorbis':
-		audio_compression = round(numpy.interp(audio_quality, [ 0, 100 ], [ -1, 10 ]), 1)
+		audio_compression = numpy.round(numpy.interp(audio_quality, [ 0, 100 ], [ -1, 10 ]), 1).astype(float).item()
 		return [ '-q:a', str(audio_compression) ]
 	return []
 
 
-def set_audio_volume(audio_volume : int) -> Commands:
+def set_audio_volume(audio_volume : int) -> List[Command]:
 	return [ '-filter:a', 'volume=' + str(audio_volume / 100) ]
 
 
-def set_video_encoder(video_encoder : str) -> Commands:
+def set_video_encoder(video_encoder : str) -> List[Command]:
 	return [ '-c:v', video_encoder ]
 
 
-def copy_video_encoder() -> Commands:
+def copy_video_encoder() -> List[Command]:
 	return set_video_encoder('copy')
 
 
-def set_video_quality(video_encoder : VideoEncoder, video_quality : int) -> Commands:
-	if video_encoder in [ 'libx264', 'libx265' ]:
-		video_compression = round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ]))
+def set_video_quality(video_encoder : VideoEncoder, video_quality : int) -> List[Command]:
+	if video_encoder in [ 'libx264', 'libx264rgb', 'libx265' ]:
+		video_compression = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ])).astype(int).item()
 		return [ '-crf', str(video_compression) ]
 	if video_encoder == 'libvpx-vp9':
-		video_compression = round(numpy.interp(video_quality, [ 0, 100 ], [ 63, 0 ]))
+		video_compression = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 63, 0 ])).astype(int).item()
 		return [ '-crf', str(video_compression) ]
 	if video_encoder in [ 'h264_nvenc', 'hevc_nvenc' ]:
-		video_compression = round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ]))
+		video_compression = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ])).astype(int).item()
 		return [ '-cq', str(video_compression) ]
 	if video_encoder in [ 'h264_amf', 'hevc_amf' ]:
-		video_compression = round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ]))
+		video_compression = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ])).astype(int).item()
 		return [ '-qp_i', str(video_compression), '-qp_p', str(video_compression), '-qp_b', str(video_compression) ]
 	if video_encoder in [ 'h264_qsv', 'hevc_qsv' ]:
-		video_compression = round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ]))
+		video_compression = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 51, 0 ])).astype(int).item()
 		return [ '-qp', str(video_compression) ]
 	if video_encoder in [ 'h264_videotoolbox', 'hevc_videotoolbox' ]:
-		video_bit_rate = round(numpy.interp(video_quality, [ 0, 100 ], [ 1024, 50512 ]))
+		video_bit_rate = numpy.round(numpy.interp(video_quality, [ 0, 100 ], [ 1024, 50512 ])).astype(int).item()
 		return [ '-b:v', str(video_bit_rate) + 'k' ]
 	return []
 
 
-def set_video_preset(video_encoder : VideoEncoder, video_preset : VideoPreset) -> Commands:
-	if video_encoder in [ 'libx264', 'libx265' ]:
+def set_video_preset(video_encoder : VideoEncoder, video_preset : VideoPreset) -> List[Command]:
+	if video_encoder in [ 'libx264', 'libx264rgb', 'libx265' ]:
 		return [ '-preset', video_preset ]
 	if video_encoder in [ 'h264_nvenc', 'hevc_nvenc' ]:
 		return [ '-preset', map_nvenc_preset(video_preset) ]
@@ -200,23 +221,25 @@ def set_video_preset(video_encoder : VideoEncoder, video_preset : VideoPreset) -
 	return []
 
 
-def set_video_colorspace(video_colorspace : str) -> Commands:
-	return [ '-colorspace', video_colorspace ]
+def set_video_fps(video_fps : Fps) -> List[Command]:
+	return [ '-vf', 'fps=' + str(video_fps) ]
 
 
-def set_video_fps(video_fps : Fps) -> Commands:
-	return [ '-vf', 'framerate=fps=' + str(video_fps) ]
-
-
-def set_video_duration(video_duration : Duration) -> Commands:
+def set_video_duration(video_duration : Duration) -> List[Command]:
 	return [ '-t', str(video_duration) ]
 
 
-def capture_video() -> Commands:
+def keep_video_alpha(video_encoder : VideoEncoder) -> List[Command]:
+	if video_encoder == 'libvpx-vp9':
+		return [ '-vf', 'format=yuva420p' ]
+	return []
+
+
+def capture_video() -> List[Command]:
 	return [ '-f', 'rawvideo', '-pix_fmt', 'rgb24' ]
 
 
-def ignore_video_stream() -> Commands:
+def ignore_video_stream() -> List[Command]:
 	return [ '-vn' ]
 
 

facefusion/filesystem.py

@@ -36,19 +36,12 @@ def get_file_format(file_path : str) -> Optional[str]:
 			return 'jpeg'
 		if file_extension == '.tif':
 			return 'tiff'
+		if file_extension == '.mpg':
+			return 'mpeg'
 		return file_extension.lstrip('.')
 	return None
 
 
-def same_file_extension(first_file_path : str, second_file_path : str) -> bool:
-	first_file_extension = get_file_extension(first_file_path)
-	second_file_extension = get_file_extension(second_file_path)
-
-	if first_file_extension and second_file_extension:
-		return get_file_extension(first_file_path) == get_file_extension(second_file_path)
-	return False
-
-
 def is_file(file_path : str) -> bool:
 	if file_path:
 		return os.path.isfile(file_path)
@@ -99,7 +92,7 @@ def has_video(video_paths : List[str]) -> bool:
 
 def are_videos(video_paths : List[str]) -> bool:
 	if video_paths:
-		return any(map(is_video, video_paths))
+		return all(map(is_video, video_paths))
 	return False
 
 

facefusion/inference_manager.py

@@ -1,41 +1,48 @@
 import importlib
-from time import sleep
+import random
+from time import sleep, time
 from typing import List
 
 from onnxruntime import InferenceSession
 
-from facefusion import process_manager, state_manager
+from facefusion import logger, process_manager, state_manager, translator
 from facefusion.app_context import detect_app_context
-from facefusion.execution import create_inference_session_providers
-from facefusion.filesystem import is_file
+from facefusion.common_helper import is_windows
+from facefusion.execution import create_inference_session_providers, has_execution_provider
+from facefusion.exit_helper import fatal_exit
+from facefusion.filesystem import get_file_name, is_file
+from facefusion.time_helper import calculate_end_time
 from facefusion.types import DownloadSet, ExecutionProvider, InferencePool, InferencePoolSet
 
 INFERENCE_POOL_SET : InferencePoolSet =\
 {
 	'cli': {},
-	'ui': {}
+	'api': {}
 }
 
 
 def get_inference_pool(module_name : str, model_names : List[str], model_source_set : DownloadSet) -> InferencePool:
 	while process_manager.is_checking():
 		sleep(0.5)
-	execution_device_id = state_manager.get_item('execution_device_id')
+	execution_device_ids = state_manager.get_item('execution_device_ids')
 	execution_providers = resolve_execution_providers(module_name)
 	app_context = detect_app_context()
-	inference_context = get_inference_context(module_name, model_names, execution_device_id, execution_providers)
 
-	if app_context == 'cli' and INFERENCE_POOL_SET.get('ui').get(inference_context):
-		INFERENCE_POOL_SET['cli'][inference_context] = INFERENCE_POOL_SET.get('ui').get(inference_context)
-	if app_context == 'ui' and INFERENCE_POOL_SET.get('cli').get(inference_context):
-		INFERENCE_POOL_SET['ui'][inference_context] = INFERENCE_POOL_SET.get('cli').get(inference_context)
-	if not INFERENCE_POOL_SET.get(app_context).get(inference_context):
-		INFERENCE_POOL_SET[app_context][inference_context] = create_inference_pool(model_source_set, execution_device_id, execution_providers)
+	for execution_device_id in execution_device_ids:
+		inference_context = get_inference_context(module_name, model_names, execution_device_id, execution_providers)
 
-	return INFERENCE_POOL_SET.get(app_context).get(inference_context)
+		if app_context == 'cli' and INFERENCE_POOL_SET.get('api').get(inference_context):
+			INFERENCE_POOL_SET['cli'][inference_context] = INFERENCE_POOL_SET.get('api').get(inference_context)
+		if app_context == 'api' and INFERENCE_POOL_SET.get('cli').get(inference_context):
+			INFERENCE_POOL_SET['api'][inference_context] = INFERENCE_POOL_SET.get('cli').get(inference_context)
+		if not INFERENCE_POOL_SET.get(app_context).get(inference_context):
+			INFERENCE_POOL_SET[app_context][inference_context] = create_inference_pool(model_source_set, execution_device_id, execution_providers)
+
+	current_inference_context = get_inference_context(module_name, model_names, random.choice(execution_device_ids), execution_providers)
+	return INFERENCE_POOL_SET.get(app_context).get(current_inference_context)
 
 
-def create_inference_pool(model_source_set : DownloadSet, execution_device_id : str, execution_providers : List[ExecutionProvider]) -> InferencePool:
+def create_inference_pool(model_source_set : DownloadSet, execution_device_id : int, execution_providers : List[ExecutionProvider]) -> InferencePool:
 	inference_pool : InferencePool = {}
 
 	for model_name in model_source_set.keys():
@@ -47,22 +54,36 @@ def create_inference_pool(model_source_set : DownloadSet, execution_device_id :
 
 
 def clear_inference_pool(module_name : str, model_names : List[str]) -> None:
-	execution_device_id = state_manager.get_item('execution_device_id')
+	execution_device_ids = state_manager.get_item('execution_device_ids')
 	execution_providers = resolve_execution_providers(module_name)
 	app_context = detect_app_context()
-	inference_context = get_inference_context(module_name, model_names, execution_device_id, execution_providers)
 
-	if INFERENCE_POOL_SET.get(app_context).get(inference_context):
-		del INFERENCE_POOL_SET[app_context][inference_context]
+	if is_windows() and has_execution_provider('directml'):
+		INFERENCE_POOL_SET[app_context].clear()
+
+	for execution_device_id in execution_device_ids:
+		inference_context = get_inference_context(module_name, model_names, execution_device_id, execution_providers)
+		if INFERENCE_POOL_SET.get(app_context).get(inference_context):
+			del INFERENCE_POOL_SET[app_context][inference_context]
 
 
-def create_inference_session(model_path : str, execution_device_id : str, execution_providers : List[ExecutionProvider]) -> InferenceSession:
-	inference_session_providers = create_inference_session_providers(execution_device_id, execution_providers)
-	return InferenceSession(model_path, providers = inference_session_providers)
+def create_inference_session(model_path : str, execution_device_id : int, execution_providers : List[ExecutionProvider]) -> InferenceSession:
+	model_file_name = get_file_name(model_path)
+	start_time = time()
+
+	try:
+		inference_session_providers = create_inference_session_providers(execution_device_id, execution_providers)
+		inference_session = InferenceSession(model_path, providers = inference_session_providers)
+		logger.debug(translator.get('loading_model_succeeded').format(model_name = model_file_name, seconds = calculate_end_time(start_time)), __name__)
+		return inference_session
+
+	except Exception:
+		logger.error(translator.get('loading_model_failed').format(model_name = model_file_name), __name__)
+		fatal_exit(1)
 
 
-def get_inference_context(module_name : str, model_names : List[str], execution_device_id : str, execution_providers : List[ExecutionProvider]) -> str:
-	inference_context = '.'.join([ module_name ] + model_names + [ execution_device_id ] + list(execution_providers))
+def get_inference_context(module_name : str, model_names : List[str], execution_device_id : int, execution_providers : List[ExecutionProvider]) -> str:
+	inference_context = '.'.join([ module_name ] + model_names + [ str(execution_device_id) ] + list(execution_providers))
 	return inference_context
 
 

facefusion/installer.py

@@ -7,18 +7,24 @@ from argparse import ArgumentParser, HelpFormatter
 from functools import partial
 from types import FrameType
 
-from facefusion import metadata, wording
+from facefusion import metadata
 from facefusion.common_helper import is_linux, is_windows
 
+LOCALS =\
+{
+	'install_dependency': 'install the {dependency} package',
+	'skip_conda': 'skip the conda environment check',
+	'conda_not_activated': 'conda is not activated'
+}
 ONNXRUNTIME_SET =\
 {
-	'default': ('onnxruntime', '1.22.0')
+	'default': ('onnxruntime', '1.23.2')
 }
 if is_windows() or is_linux():
-	ONNXRUNTIME_SET['cuda'] = ('onnxruntime-gpu', '1.22.0')
-	ONNXRUNTIME_SET['openvino'] = ('onnxruntime-openvino', '1.22.0')
+	ONNXRUNTIME_SET['cuda'] = ('onnxruntime-gpu', '1.23.2')
+	ONNXRUNTIME_SET['openvino'] = ('onnxruntime-openvino', '1.23.0')
 if is_windows():
-	ONNXRUNTIME_SET['directml'] = ('onnxruntime-directml', '1.17.3')
+	ONNXRUNTIME_SET['directml'] = ('onnxruntime-directml', '1.23.0')
 if is_linux():
 	ONNXRUNTIME_SET['rocm'] = ('onnxruntime-rocm', '1.21.0')
 
@@ -26,8 +32,8 @@ if is_linux():
 def cli() -> None:
 	signal.signal(signal.SIGINT, signal_exit)
 	program = ArgumentParser(formatter_class = partial(HelpFormatter, max_help_position = 50))
-	program.add_argument('--onnxruntime', help = wording.get('help.install_dependency').format(dependency = 'onnxruntime'), choices = ONNXRUNTIME_SET.keys(), required = True)
-	program.add_argument('--skip-conda', help = wording.get('help.skip_conda'), action = 'store_true')
+	program.add_argument('--onnxruntime', help = LOCALS.get('install_dependency').format(dependency = 'onnxruntime'), choices = ONNXRUNTIME_SET.keys(), required = True)
+	program.add_argument('--skip-conda', help = LOCALS.get('skip_conda'), action = 'store_true')
 	program.add_argument('-v', '--version', version = metadata.get('name') + ' ' + metadata.get('version'), action = 'version')
 	run(program)
 
@@ -42,7 +48,7 @@ def run(program : ArgumentParser) -> None:
 	onnxruntime_name, onnxruntime_version = ONNXRUNTIME_SET.get(args.onnxruntime)
 
 	if not args.skip_conda and not has_conda:
-		sys.stdout.write(wording.get('conda_not_activated') + os.linesep)
+		sys.stdout.write(LOCALS.get('conda_not_activated') + os.linesep)
 		sys.exit(1)
 
 	with open('requirements.txt') as file:
@@ -92,5 +98,3 @@ def run(program : ArgumentParser) -> None:
 
 			subprocess.call([ shutil.which('conda'), 'env', 'config', 'vars', 'set', 'PATH=' + os.pathsep.join(library_paths) ])
 
-	if args.onnxruntime == 'directml':
-		subprocess.call([ shutil.which('pip'), 'install', 'numpy==1.26.4', '--force-reinstall' ])

facefusion/jobs/job_helper.py

@@ -7,10 +7,12 @@ from facefusion.filesystem import get_file_extension, get_file_name
 
 def get_step_output_path(job_id : str, step_index : int, output_path : str) -> Optional[str]:
 	if output_path:
-		output_directory_path, _ = os.path.split(output_path)
-		output_file_name = get_file_name(_)
-		output_file_extension = get_file_extension(_)
-		return os.path.join(output_directory_path, output_file_name + '-' + job_id + '-' + str(step_index) + output_file_extension)
+		output_directory_path, output_file_path = os.path.split(output_path)
+		output_file_name = get_file_name(output_file_path)
+		output_file_extension = get_file_extension(output_file_path)
+
+		if output_file_name and output_file_extension:
+			return os.path.join(output_directory_path, output_file_name + '-' + job_id + '-' + str(step_index) + output_file_extension)
 	return None
 
 

facefusion/jobs/job_list.py

@@ -1,15 +1,15 @@
 from datetime import datetime
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
 
-from facefusion.date_helper import describe_time_ago
 from facefusion.jobs import job_manager
-from facefusion.types import JobStatus, TableContents, TableHeaders
+from facefusion.time_helper import describe_time_ago
+from facefusion.types import JobStatus, TableContent, TableHeader
 
 
-def compose_job_list(job_status : JobStatus) -> Tuple[TableHeaders, TableContents]:
+def compose_job_list(job_status : JobStatus) -> Tuple[List[TableHeader], List[List[TableContent]]]:
 	jobs = job_manager.find_jobs(job_status)
-	job_headers : TableHeaders = [ 'job id', 'steps', 'date created', 'date updated', 'job status' ]
-	job_contents : TableContents = []
+	job_headers : List[TableHeader] = [ 'job id', 'steps', 'date created', 'date updated', 'job status' ]
+	job_contents : List[List[TableContent]] = []
 
 	for index, job_id in enumerate(jobs):
 		if job_manager.validate_job(job_id):

facefusion/jobs/job_manager.py

@@ -3,10 +3,10 @@ from copy import copy
 from typing import List, Optional
 
 import facefusion.choices
-from facefusion.date_helper import get_current_date_time
 from facefusion.filesystem import create_directory, get_file_name, is_directory, is_file, move_file, remove_directory, remove_file, resolve_file_pattern
 from facefusion.jobs.job_helper import get_step_output_path
 from facefusion.json import read_json, write_json
+from facefusion.time_helper import get_current_date_time
 from facefusion.types import Args, Job, JobSet, JobStatus, JobStep, JobStepStatus
 
 JOBS_PATH : Optional[str] = None

facefusion/jobs/job_store.py

@@ -1,27 +0,0 @@
-from typing import List
-
-from facefusion.types import JobStore
-
-JOB_STORE : JobStore =\
-{
-	'job_keys': [],
-	'step_keys': []
-}
-
-
-def get_job_keys() -> List[str]:
-	return JOB_STORE.get('job_keys')
-
-
-def get_step_keys() -> List[str]:
-	return JOB_STORE.get('step_keys')
-
-
-def register_job_keys(step_keys : List[str]) -> None:
-	for step_key in step_keys:
-		JOB_STORE['job_keys'].append(step_key)
-
-
-def register_step_keys(job_keys : List[str]) -> None:
-	for job_key in job_keys:
-		JOB_STORE['step_keys'].append(job_key)

facefusion/libraries/aom.py

@@ -0,0 +1,134 @@
+import ctypes
+from functools import lru_cache
+from typing import Optional
+
+from facefusion.common_helper import is_linux, is_macos, is_windows
+from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
+from facefusion.filesystem import resolve_relative_path
+from facefusion.types import LibrarySet
+
+
+@lru_cache
+def create_static_library_set() -> Optional[LibrarySet]:
+	if is_linux():
+		return\
+		{
+			'hashes':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libaom.hash'),
+					'path': resolve_relative_path('../.libraries/libaom.hash')
+				}
+			},
+			'sources':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libaom.so'),
+					'path': resolve_relative_path('../.libraries/libaom.so')
+				}
+			}
+		}
+	if is_macos():
+		return\
+		{
+			'hashes':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libaom.hash'),
+					'path': resolve_relative_path('../.libraries/libaom.hash')
+				}
+			},
+			'sources':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libaom.dylib'),
+					'path': resolve_relative_path('../.libraries/libaom.dylib')
+				}
+			}
+		}
+	if is_windows():
+		return\
+		{
+			'hashes':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/aom.hash'),
+					'path': resolve_relative_path('../.libraries/aom.hash')
+				}
+			},
+			'sources':
+			{
+				'aom':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/aom.dll'),
+					'path': resolve_relative_path('../.libraries/aom.dll')
+				}
+			}
+		}
+
+	return None
+
+
+def pre_check() -> bool:
+	library_hash_set = create_static_library_set().get('hashes')
+	library_source_set = create_static_library_set().get('sources')
+
+	return conditional_download_hashes(library_hash_set) and conditional_download_sources(library_source_set)
+
+
+@lru_cache
+def create_static_library() -> Optional[ctypes.CDLL]:
+	library_path = create_static_library_set().get('sources').get('aom').get('path')
+
+	if library_path:
+		if is_windows():
+			library = ctypes.CDLL(library_path, winmode = 0)
+		else:
+			library = ctypes.CDLL(library_path)
+
+		if library:
+			return init_ctypes(library)
+
+	return None
+
+
+def init_ctypes(library : ctypes.CDLL) -> ctypes.CDLL:
+	library.aom_codec_enc_config_default.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint ]
+	library.aom_codec_enc_config_default.restype = ctypes.c_int
+
+	library.aom_codec_enc_init_ver.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_int ]
+	library.aom_codec_enc_init_ver.restype = ctypes.c_int
+
+	library.aom_codec_encode.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_ulong, ctypes.c_long, ctypes.c_ulong ]
+	library.aom_codec_encode.restype = ctypes.c_int
+
+	library.aom_codec_get_cx_data.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p) ]
+	library.aom_codec_get_cx_data.restype = ctypes.c_void_p
+
+	library.aom_codec_enc_config_set.argtypes = [ ctypes.c_void_p, ctypes.c_void_p ]
+	library.aom_codec_enc_config_set.restype = ctypes.c_int
+
+	library.aom_codec_destroy.argtypes = [ ctypes.c_void_p ]
+	library.aom_codec_destroy.restype = ctypes.c_int
+
+	library.aom_img_wrap.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint, ctypes.c_void_p ]
+	library.aom_img_wrap.restype = ctypes.c_void_p
+
+	library.aom_codec_control.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int ]
+	library.aom_codec_control.restype = ctypes.c_int
+
+	library.aom_codec_dec_init_ver.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_int ]
+	library.aom_codec_dec_init_ver.restype = ctypes.c_int
+
+	library.aom_codec_decode.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint, ctypes.c_void_p ]
+	library.aom_codec_decode.restype = ctypes.c_int
+
+	library.aom_codec_get_frame.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p) ]
+	library.aom_codec_get_frame.restype = ctypes.c_void_p
+
+	return library

facefusion/libraries/datachannel.py

@@ -0,0 +1,294 @@
+import ctypes
+from functools import lru_cache
+from typing import Optional
+
+from facefusion.common_helper import is_linux, is_macos, is_windows
+from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
+from facefusion.filesystem import resolve_relative_path
+from facefusion.types import LibrarySet
+
+
+@lru_cache
+def create_static_library_set() -> Optional[LibrarySet]:
+	if is_linux():
+		return\
+		{
+			'hashes':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libcrypto.hash'),
+					'path': resolve_relative_path('../.libraries/libcrypto.hash')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libdatachannel_next.hash'),
+					'path': resolve_relative_path('../.libraries/libdatachannel_next.hash')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libssl.hash'),
+					'path': resolve_relative_path('../.libraries/libssl.hash')
+				}
+			},
+			'sources':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libcrypto.so'),
+					'path': resolve_relative_path('../.libraries/libcrypto.so')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libdatachannel_next.so'),
+					'path': resolve_relative_path('../.libraries/libdatachannel_next.so')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libssl.so'),
+					'path': resolve_relative_path('../.libraries/libssl.so')
+				}
+			}
+		}
+	if is_macos():
+		return\
+		{
+			'hashes':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libcrypto.hash'),
+					'path': resolve_relative_path('../.libraries/libcrypto.hash')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libdatachannel_next.hash'),
+					'path': resolve_relative_path('../.libraries/libdatachannel_next.hash')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libssl.hash'),
+					'path': resolve_relative_path('../.libraries/libssl.hash')
+				}
+			},
+			'sources':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libcrypto.dylib'),
+					'path': resolve_relative_path('../.libraries/libcrypto.dylib')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libdatachannel_next.dylib'),
+					'path': resolve_relative_path('../.libraries/libdatachannel_next.dylib')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libssl.dylib'),
+					'path': resolve_relative_path('../.libraries/libssl.dylib')
+				}
+			}
+		}
+	if is_windows():
+		return\
+		{
+			'hashes':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/crypto.hash'),
+					'path': resolve_relative_path('../.libraries/crypto.hash')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/datachannel_next.hash'),
+					'path': resolve_relative_path('../.libraries/datachannel_next.hash')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/ssl.hash'),
+					'path': resolve_relative_path('../.libraries/ssl.hash')
+				}
+			},
+			'sources':
+			{
+				'crypto':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/crypto.dll'),
+					'path': resolve_relative_path('../.libraries/crypto.dll')
+				},
+				'datachannel':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/datachannel_next.dll'),
+					'path': resolve_relative_path('../.libraries/datachannel_next.dll')
+				},
+				'ssl':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/ssl.dll'),
+					'path': resolve_relative_path('../.libraries/ssl.dll')
+				}
+			}
+		}
+
+	return None
+
+
+def pre_check() -> bool:
+	library_hash_set = create_static_library_set().get('hashes')
+	library_source_set = create_static_library_set().get('sources')
+
+	return conditional_download_hashes(library_hash_set) and conditional_download_sources(library_source_set)
+
+
+@lru_cache
+def create_static_library() -> Optional[ctypes.CDLL]:
+	crypto_source_path = create_static_library_set().get('sources').get('crypto').get('path')
+	datachannel_source_path = create_static_library_set().get('sources').get('datachannel').get('path')
+	ssl_source_path = create_static_library_set().get('sources').get('ssl').get('path')
+
+	if crypto_source_path and datachannel_source_path and ssl_source_path:
+		if is_windows():
+			ctypes.CDLL(crypto_source_path, winmode = 0)
+			ctypes.CDLL(ssl_source_path, winmode = 0)
+			library = ctypes.CDLL(datachannel_source_path, winmode = 0)
+		else:
+			ctypes.CDLL(crypto_source_path)
+			ctypes.CDLL(ssl_source_path)
+			library = ctypes.CDLL(datachannel_source_path)
+
+		if library:
+			return init_ctypes(library)
+
+	return None
+
+
+def init_ctypes(library : ctypes.CDLL) -> ctypes.CDLL:
+	library.rtcInitLogger.argtypes = [ ctypes.c_int, ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p) ]
+	library.rtcInitLogger.restype = None
+	library.rtcInitLogger(2, ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p)(0))
+
+	library.rtcCreatePeerConnection.restype = ctypes.c_int
+
+	library.rtcDeletePeerConnection.argtypes = [ ctypes.c_int ]
+	library.rtcDeletePeerConnection.restype = ctypes.c_int
+
+	library.rtcSetLocalDescription.argtypes = [ ctypes.c_int, ctypes.c_char_p ]
+	library.rtcSetLocalDescription.restype = ctypes.c_int
+
+	library.rtcSetRemoteDescription.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p ]
+	library.rtcSetRemoteDescription.restype = ctypes.c_int
+
+	library.rtcAddTrackEx.restype = ctypes.c_int
+
+	library.rtcSendMessage.argtypes = [ ctypes.c_int, ctypes.c_void_p, ctypes.c_int ]
+	library.rtcSendMessage.restype = ctypes.c_int
+
+	library.rtcSetAV1Packetizer.restype = ctypes.c_int
+	library.rtcSetVP8Packetizer.restype = ctypes.c_int
+	library.rtcSetVP9Packetizer.restype = ctypes.c_int
+
+	library.rtcChainRtcpSrReporter.argtypes = [ ctypes.c_int ]
+	library.rtcChainRtcpSrReporter.restype = ctypes.c_int
+
+	library.rtcSetTrackRtpTimestamp.argtypes = [ ctypes.c_int, ctypes.c_uint32 ]
+	library.rtcSetTrackRtpTimestamp.restype = ctypes.c_int
+
+	library.rtcIsOpen.argtypes = [ ctypes.c_int ]
+	library.rtcIsOpen.restype = ctypes.c_bool
+
+	library.rtcChainRtcpNackResponder.argtypes = [ ctypes.c_int, ctypes.c_uint ]
+	library.rtcChainRtcpNackResponder.restype = ctypes.c_int
+
+	library.rtcGetLocalDescription.argtypes = [ ctypes.c_int, ctypes.c_char_p, ctypes.c_int ]
+	library.rtcGetLocalDescription.restype = ctypes.c_int
+
+	library.rtcSetOpusPacketizer.restype = ctypes.c_int
+
+	library.rtcGetPayloadTypesForCodec.argtypes = [ ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.c_int ]
+	library.rtcGetPayloadTypesForCodec.restype = ctypes.c_int
+
+	library.rtcSetAV1Depacketizer.argtypes = [ ctypes.c_int, ctypes.c_int ]
+	library.rtcSetAV1Depacketizer.restype = ctypes.c_int
+	library.rtcSetVP8Depacketizer.restype = ctypes.c_int
+	library.rtcSetVP9Depacketizer.restype = ctypes.c_int
+	library.rtcSetOpusDepacketizer.restype = ctypes.c_int
+
+	library.rtcChainRtcpReceivingSession.argtypes = [ ctypes.c_int ]
+	library.rtcChainRtcpReceivingSession.restype = ctypes.c_int
+
+	library.rtcSetFrameCallback.argtypes = [ ctypes.c_int, ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) ]
+	library.rtcSetFrameCallback.restype = ctypes.c_int
+
+	library.rtcSetUserPointer.argtypes = [ ctypes.c_int, ctypes.c_void_p ]
+	library.rtcSetUserPointer.restype = None
+
+	library.rtcChainRembHandler.argtypes = [ ctypes.c_int, ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_uint, ctypes.c_void_p) ]
+	library.rtcChainRembHandler.restype = ctypes.c_int
+
+	library.rtcRequestBitrate.argtypes = [ ctypes.c_int, ctypes.c_uint ]
+	library.rtcRequestBitrate.restype = ctypes.c_int
+
+	library.rtcSetClosedCallback.argtypes = [ ctypes.c_int, ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_void_p) ]
+	library.rtcSetClosedCallback.restype = ctypes.c_int
+
+	return library
+
+
+def define_rtc_configuration() -> ctypes.Structure:
+	return type('RTC_CONFIGURATION', (ctypes.Structure,),
+	{
+		'_fields_':
+		[
+			('iceServers', ctypes.POINTER(ctypes.c_char_p)),
+			('iceServersCount', ctypes.c_int),
+			('proxyServer', ctypes.c_char_p),
+			('bindAddress', ctypes.c_char_p),
+			('certificateType', ctypes.c_int),
+			('iceTransportPolicy', ctypes.c_int),
+			('enableIceTcp', ctypes.c_bool),
+			('enableIceUdpMux', ctypes.c_bool),
+			('disableAutoNegotiation', ctypes.c_bool),
+			('forceMediaTransport', ctypes.c_bool),
+			('portRangeBegin', ctypes.c_ushort),
+			('portRangeEnd', ctypes.c_ushort),
+			('mtu', ctypes.c_int),
+			('maxMessageSize', ctypes.c_int)
+		]
+	})()
+
+
+def define_rtc_track_init() -> ctypes.Structure:
+	return type('RTC_TRACK_INIT', (ctypes.Structure,),
+	{
+		'_fields_':
+		[
+			('direction', ctypes.c_int),
+			('codec', ctypes.c_int),
+			('payloadType', ctypes.c_int),
+			('ssrc', ctypes.c_uint32),
+			('mid', ctypes.c_char_p),
+			('name', ctypes.c_char_p),
+			('msid', ctypes.c_char_p),
+			('trackId', ctypes.c_char_p),
+			('profile', ctypes.c_char_p)
+		]
+	})()
+
+
+def define_rtc_packetizer_init() -> ctypes.Structure:
+	return type('RTC_PACKETIZER_INIT', (ctypes.Structure,),
+	{
+		'_fields_':
+		[
+			('ssrc', ctypes.c_uint32),
+			('cname', ctypes.c_char_p),
+			('payloadType', ctypes.c_uint8),
+			('clockRate', ctypes.c_uint32),
+			('sequenceNumber', ctypes.c_uint16),
+			('timestamp', ctypes.c_uint32),
+			('maxFragmentSize', ctypes.c_uint16),
+			('nalSeparator', ctypes.c_int),
+			('obuPacketization', ctypes.c_int)
+		]
+	})()

facefusion/libraries/opus.py

@@ -0,0 +1,121 @@
+import ctypes
+from functools import lru_cache
+from typing import Optional
+
+from facefusion.common_helper import is_linux, is_macos, is_windows
+from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
+from facefusion.filesystem import resolve_relative_path
+from facefusion.types import LibrarySet
+
+
+@lru_cache
+def create_static_library_set() -> Optional[LibrarySet]:
+	if is_linux():
+		return\
+		{
+			'hashes':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libopus.hash'),
+					'path': resolve_relative_path('../.libraries/libopus.hash')
+				}
+			},
+			'sources':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libopus.so'),
+					'path': resolve_relative_path('../.libraries/libopus.so')
+				}
+			}
+		}
+
+	if is_macos():
+		return\
+		{
+			'hashes':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libopus.hash'),
+					'path': resolve_relative_path('../.libraries/libopus.hash')
+				}
+			},
+			'sources':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libopus.dylib'),
+					'path': resolve_relative_path('../.libraries/libopus.dylib')
+				}
+			}
+		}
+
+	if is_windows():
+		return\
+		{
+			'hashes':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/opus.hash'),
+					'path': resolve_relative_path('../.libraries/opus.hash')
+				}
+			},
+			'sources':
+			{
+				'opus':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/opus.dll'),
+					'path': resolve_relative_path('../.libraries/opus.dll')
+				}
+			}
+		}
+
+	return None
+
+
+def pre_check() -> bool:
+	library_hash_set = create_static_library_set().get('hashes')
+	library_source_set = create_static_library_set().get('sources')
+
+	return conditional_download_hashes(library_hash_set) and conditional_download_sources(library_source_set)
+
+
+@lru_cache
+def create_static_library() -> Optional[ctypes.CDLL]:
+	library_path = create_static_library_set().get('sources').get('opus').get('path')
+
+	if library_path:
+		if is_windows():
+			library = ctypes.CDLL(library_path, winmode = 0)
+		else:
+			library = ctypes.CDLL(library_path)
+
+		if library:
+			return init_ctypes(library)
+
+	return None
+
+
+def init_ctypes(library : ctypes.CDLL) -> ctypes.CDLL:
+	library.opus_encoder_create.argtypes = [ ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int) ]
+	library.opus_encoder_create.restype = ctypes.c_void_p
+
+	library.opus_encode_float.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_int, ctypes.c_char_p, ctypes.c_int ]
+	library.opus_encode_float.restype = ctypes.c_int
+
+	library.opus_encoder_destroy.argtypes = [ ctypes.c_void_p ]
+	library.opus_encoder_destroy.restype = None
+
+	library.opus_decoder_create.argtypes = [ ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int) ]
+	library.opus_decoder_create.restype = ctypes.c_void_p
+
+	library.opus_decode_float.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_int, ctypes.c_int ]
+	library.opus_decode_float.restype = ctypes.c_int
+
+	library.opus_decoder_destroy.argtypes = [ ctypes.c_void_p ]
+	library.opus_decoder_destroy.restype = None
+
+	return library

facefusion/libraries/vpx.py

@@ -0,0 +1,134 @@
+import ctypes
+from functools import lru_cache
+from typing import Optional
+
+from facefusion.common_helper import is_linux, is_macos, is_windows
+from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
+from facefusion.filesystem import resolve_relative_path
+from facefusion.types import LibrarySet
+
+
+@lru_cache
+def create_static_library_set() -> Optional[LibrarySet]:
+	if is_linux():
+		return\
+		{
+			'hashes':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libvpx.hash'),
+					'path': resolve_relative_path('../.libraries/libvpx.hash')
+				}
+			},
+			'sources':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'linux/libvpx.so'),
+					'path': resolve_relative_path('../.libraries/libvpx.so')
+				}
+			}
+		}
+	if is_macos():
+		return\
+		{
+			'hashes':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libvpx.hash'),
+					'path': resolve_relative_path('../.libraries/libvpx.hash')
+				}
+			},
+			'sources':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'macos/libvpx.dylib'),
+					'path': resolve_relative_path('../.libraries/libvpx.dylib')
+				}
+			}
+		}
+	if is_windows():
+		return\
+		{
+			'hashes':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/vpx.hash'),
+					'path': resolve_relative_path('../.libraries/vpx.hash')
+				}
+			},
+			'sources':
+			{
+				'vpx':
+				{
+					'url': resolve_download_url_by_provider('huggingface', 'libraries-4.0.0', 'windows/vpx.dll'),
+					'path': resolve_relative_path('../.libraries/vpx.dll')
+				}
+			}
+		}
+
+	return None
+
+
+def pre_check() -> bool:
+	library_hash_set = create_static_library_set().get('hashes')
+	library_source_set = create_static_library_set().get('sources')
+
+	return conditional_download_hashes(library_hash_set) and conditional_download_sources(library_source_set)
+
+
+@lru_cache
+def create_static_library() -> Optional[ctypes.CDLL]:
+	library_path = create_static_library_set().get('sources').get('vpx').get('path')
+
+	if library_path:
+		if is_windows():
+			library = ctypes.CDLL(library_path, winmode = 0)
+		else:
+			library = ctypes.CDLL(library_path)
+
+		if library:
+			return init_ctypes(library)
+
+	return None
+
+
+def init_ctypes(library : ctypes.CDLL) -> ctypes.CDLL:
+	library.vpx_codec_enc_config_default.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint ]
+	library.vpx_codec_enc_config_default.restype = ctypes.c_int
+
+	library.vpx_codec_enc_init_ver.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_int ]
+	library.vpx_codec_enc_init_ver.restype = ctypes.c_int
+
+	library.vpx_codec_encode.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_ulong, ctypes.c_long, ctypes.c_ulong ]
+	library.vpx_codec_encode.restype = ctypes.c_int
+
+	library.vpx_codec_get_cx_data.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p) ]
+	library.vpx_codec_get_cx_data.restype = ctypes.c_void_p
+
+	library.vpx_codec_enc_config_set.argtypes = [ ctypes.c_void_p, ctypes.c_void_p ]
+	library.vpx_codec_enc_config_set.restype = ctypes.c_int
+
+	library.vpx_codec_destroy.argtypes = [ ctypes.c_void_p ]
+	library.vpx_codec_destroy.restype = ctypes.c_int
+
+	library.vpx_img_wrap.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint, ctypes.c_void_p ]
+	library.vpx_img_wrap.restype = ctypes.c_void_p
+
+	library.vpx_codec_control_.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_int ]
+	library.vpx_codec_control_.restype = ctypes.c_int
+
+	library.vpx_codec_dec_init_ver.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long, ctypes.c_int ]
+	library.vpx_codec_dec_init_ver.restype = ctypes.c_int
+
+	library.vpx_codec_decode.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint, ctypes.c_void_p, ctypes.c_long ]
+	library.vpx_codec_decode.restype = ctypes.c_int
+
+	library.vpx_codec_get_frame.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p) ]
+	library.vpx_codec_get_frame.restype = ctypes.c_void_p
+
+	return library

facefusion/locals.py

@@ -0,0 +1,272 @@
+from facefusion.types import Locals
+
+LOCALS : Locals =\
+{
+	'en':
+	{
+		'conda_not_activated': 'conda is not activated',
+		'python_not_supported': 'python version is not supported, upgrade to {version} or higher',
+		'curl_not_installed': 'curl is not installed',
+		'ffmpeg_not_installed': 'ffmpeg is not installed',
+		'creating_temp': 'creating temporary resources',
+		'extracting_frames': 'extracting frames with a resolution of {resolution} and {fps} frames per second',
+		'extracting_frames_succeeded': 'extracting frames succeeded',
+		'extracting_frames_failed': 'extracting frames failed',
+		'spawning_frames_succeeded': 'spawning frames succeeded',
+		'spawning_frames_failed': 'spawning frames failed',
+		'analysing': 'analysing',
+		'extracting': 'extracting',
+		'spawning': 'spawning',
+		'streaming': 'streaming',
+		'processing': 'processing',
+		'merging': 'merging',
+		'downloading': 'downloading',
+		'temp_frames_not_found': 'temporary frames not found',
+		'copying_image': 'copying image with a resolution of {resolution}',
+		'copying_image_succeeded': 'copying image succeeded',
+		'copying_image_failed': 'copying image failed',
+		'finalizing_image': 'finalizing image with a resolution of {resolution}',
+		'finalizing_image_succeeded': 'finalizing image succeeded',
+		'finalizing_image_skipped': 'finalizing image skipped',
+		'merging_video': 'merging video with a resolution of {resolution} and {fps} frames per second',
+		'merging_video_succeeded': 'merging video succeeded',
+		'merging_video_failed': 'merging video failed',
+		'skipping_audio': 'skipping audio',
+		'replacing_audio_succeeded': 'replacing audio succeeded',
+		'replacing_audio_skipped': 'replacing audio skipped',
+		'restoring_audio_succeeded': 'restoring audio succeeded',
+		'restoring_audio_skipped': 'restoring audio skipped',
+		'clearing_temp': 'clearing temporary resources',
+		'processing_stopped': 'processing stopped',
+		'processing_image_succeeded': 'processing to image succeeded in {seconds} seconds',
+		'processing_image_failed': 'processing to image failed',
+		'processing_video_succeeded': 'processing to video succeeded in {seconds} seconds',
+		'processing_video_failed': 'processing to video failed',
+		'choose_image_source': 'choose an image for the source',
+		'choose_audio_source': 'choose an audio for the source',
+		'choose_video_target': 'choose a video for the target',
+		'choose_image_or_video_target': 'choose an image or video for the target',
+		'specify_image_or_video_output': 'specify the output image or video within a directory',
+		'match_target_and_output_extension': 'match the target and output extension',
+		'no_source_face_detected': 'no source face detected',
+		'processor_not_loaded': 'processor {processor} could not be loaded',
+		'processor_not_implemented': 'processor {processor} not implemented correctly',
+		'stream_not_loaded': 'stream {stream_mode} could not be loaded',
+		'stream_not_supported': 'stream not supported',
+		'api_started': 'started API on {host}:{port}',
+		'job_created': 'job {job_id} created',
+		'job_not_created': 'job {job_id} not created',
+		'job_submitted': 'job {job_id} submitted',
+		'job_not_submitted': 'job {job_id} not submitted',
+		'job_all_submitted': 'jobs submitted',
+		'job_all_not_submitted': 'jobs not submitted',
+		'job_deleted': 'job {job_id} deleted',
+		'job_not_deleted': 'job {job_id} not deleted',
+		'job_all_deleted': 'jobs deleted',
+		'job_all_not_deleted': 'jobs not deleted',
+		'job_step_added': 'step added to job {job_id}',
+		'job_step_not_added': 'step not added to job {job_id}',
+		'job_remix_step_added': 'step {step_index} remixed from job {job_id}',
+		'job_remix_step_not_added': 'step {step_index} not remixed from job {job_id}',
+		'job_step_inserted': 'step {step_index} inserted to job {job_id}',
+		'job_step_not_inserted': 'step {step_index} not inserted to job {job_id}',
+		'job_step_removed': 'step {step_index} removed from job {job_id}',
+		'job_step_not_removed': 'step {step_index} not removed from job {job_id}',
+		'running_job': 'running queued job {job_id}',
+		'running_jobs': 'running all queued jobs',
+		'retrying_job': 'retrying failed job {job_id}',
+		'retrying_jobs': 'retrying all failed jobs',
+		'processing_job_succeeded': 'processing of job {job_id} succeeded',
+		'processing_jobs_succeeded': 'processing of all jobs succeeded',
+		'processing_job_failed': 'processing of job {job_id} failed',
+		'processing_jobs_failed': 'processing of all jobs failed',
+		'processing_step': 'processing step {step_current} of {step_total}',
+		'validating_hash_succeeded': 'validating hash for {hash_file_name} succeeded',
+		'validating_hash_failed': 'validating hash for {hash_file_name} failed',
+		'validating_source_succeeded': 'validating source for {source_file_name} succeeded',
+		'validating_source_failed': 'validating source for {source_file_name} failed',
+		'deleting_corrupt_source': 'deleting corrupt source for {source_file_name}',
+		'loading_model_succeeded': 'loading model {model_name} succeeded in {seconds} seconds',
+		'loading_model_failed': 'loading model {model_name} failed',
+		'time_ago_now': 'just now',
+		'time_ago_minutes': '{minutes} minutes ago',
+		'time_ago_hours': '{hours} hours and {minutes} minutes ago',
+		'time_ago_days': '{days} days, {hours} hours and {minutes} minutes ago',
+		'point': '.',
+		'comma': ',',
+		'colon': ':',
+		'question_mark': '?',
+		'exclamation_mark': '!',
+		'help':
+		{
+			'install_dependency': 'choose the variant of {dependency} to install',
+			'skip_conda': 'skip the conda environment check',
+			'workflow': 'choose the workflow',
+			'config_path': 'choose the config file to override defaults',
+			'temp_path': 'specify the directory for the temporary resources',
+			'jobs_path': 'specify the directory to store jobs',
+			'source_paths': 'choose the image or audio paths',
+			'target_path': 'choose the image or video path',
+			'output_path': 'specify the image or video within a directory',
+			'source_pattern': 'choose the image or audio pattern',
+			'target_pattern': 'choose the image or video pattern',
+			'output_pattern': 'specify the image or video pattern',
+			'face_detector_model': 'choose the model responsible for detecting the faces',
+			'face_detector_size': 'specify the frame size provided to the face detector',
+			'face_detector_margin': 'apply top, right, bottom and left margin to the frame',
+			'face_detector_angles': 'specify the angles to rotate the frame before detecting faces',
+			'face_detector_score': 'filter the detected faces based on the confidence score',
+			'face_landmarker_model': 'choose the model responsible for detecting the face landmarks',
+			'face_landmarker_score': 'filter the detected face landmarks based on the confidence score',
+			'face_selector_mode': 'use reference based tracking or simple matching',
+			'face_selector_order': 'specify the order of the detected faces',
+			'face_selector_age_start': 'filter the detected faces based on the starting age',
+			'face_selector_age_end': 'filter the detected faces based on the ending age',
+			'face_selector_gender': 'filter the detected faces based on their gender',
+			'face_selector_race': 'filter the detected faces based on their race',
+			'reference_face_position': 'specify the position used to create the reference face',
+			'reference_face_distance': 'specify the similarity between the reference face and target face',
+			'reference_frame_number': 'specify the frame used to create the reference face',
+			'face_occluder_model': 'choose the model responsible for the occlusion mask',
+			'face_parser_model': 'choose the model responsible for the region mask',
+			'face_mask_types': 'mix and match different face mask types (choices: {choices})',
+			'face_mask_areas': 'choose the items used for the area mask (choices: {choices})',
+			'face_mask_regions': 'choose the items used for the region mask (choices: {choices})',
+			'face_mask_blur': 'specify the degree of blur applied to the box mask',
+			'face_mask_padding': 'apply top, right, bottom and left padding to the box mask',
+			'voice_extractor_model': 'choose the model responsible for extracting the voices',
+			'trim_frame_start': 'specify the starting frame of the target video',
+			'trim_frame_end': 'specify the ending frame of the target video',
+			'temp_frame_format': 'specify the temporary resources format',
+			'output_image_quality': 'specify the image quality which translates to the image compression',
+			'output_image_scale': 'specify the image scale based on the target image',
+			'output_audio_encoder': 'specify the encoder used for the audio',
+			'output_audio_quality': 'specify the audio quality which translates to the audio compression',
+			'output_audio_volume': 'specify the audio volume based on the target video',
+			'output_video_encoder': 'specify the encoder used for the video',
+			'output_video_preset': 'balance fast video processing and video file size',
+			'output_video_quality': 'specify the video quality which translates to the video compression',
+			'output_video_scale': 'specify the video scale based on the target video',
+			'output_video_fps': 'specify the video fps based on the target video',
+			'processors': 'load a single or multiple processors (choices: {choices}, ...)',
+			'background-remover-model': 'choose the model responsible for removing the background',
+			'background-remover-color': 'apply red, green blue and alpha values of the background',
+			'download_providers': 'download using different providers (choices: {choices}, ...)',
+			'download_scope': 'specify the download scope',
+			'benchmark_mode': 'choose the benchmark mode',
+			'benchmark_resolutions': 'choose the resolutions for the benchmarks (choices: {choices}, ...)',
+			'benchmark_cycle_count': 'specify the amount of cycles per benchmark',
+			'api_host': 'specify the API host',
+			'api_port': 'specify the API port',
+			'execution_device_ids': 'specify the devices used for processing',
+			'execution_providers': 'inference using different providers (choices: {choices}, ...)',
+			'execution_thread_count': 'specify the amount of parallel threads while processing',
+			'video_memory_strategy': 'balance fast processing and low VRAM usage',
+			'log_level': 'adjust the message severity displayed in the terminal',
+			'halt_on_error': 'halt the program once an error occurred',
+			'run': 'run the program',
+			'batch_run': 'run the program in batch mode',
+			'force_download': 'force automate downloads and exit',
+			'benchmark': 'benchmark the program',
+			'api': 'start the API server',
+			'job_id': 'specify the job id',
+			'job_status': 'specify the job status',
+			'step_index': 'specify the step index',
+			'job_list': 'list jobs by status',
+			'job_create': 'create a drafted job',
+			'job_submit': 'submit a drafted job to become a queued job',
+			'job_submit_all': 'submit all drafted jobs to become a queued jobs',
+			'job_delete': 'delete a drafted, queued, failed or completed job',
+			'job_delete_all': 'delete all drafted, queued, failed and completed jobs',
+			'job_add_step': 'add a step to a drafted job',
+			'job_remix_step': 'remix a previous step from a drafted job',
+			'job_insert_step': 'insert a step to a drafted job',
+			'job_remove_step': 'remove a step from a drafted job',
+			'job_run': 'run a queued job',
+			'job_run_all': 'run all queued jobs',
+			'job_retry': 'retry a failed job',
+			'job_retry_all': 'retry all failed jobs'
+		},
+		'about':
+		{
+			'fund': 'fund training server',
+			'subscribe': 'become a member',
+			'join': 'join our community'
+		},
+		'uis':
+		{
+			'apply_button': 'APPLY',
+			'benchmark_mode_dropdown': 'BENCHMARK MODE',
+			'benchmark_cycle_count_slider': 'BENCHMARK CYCLE COUNT',
+			'benchmark_resolutions_checkbox_group': 'BENCHMARK RESOLUTIONS',
+			'clear_button': 'CLEAR',
+			'common_options_checkbox_group': 'OPTIONS',
+			'download_providers_checkbox_group': 'DOWNLOAD PROVIDERS',
+			'execution_providers_checkbox_group': 'EXECUTION PROVIDERS',
+			'execution_thread_count_slider': 'EXECUTION THREAD COUNT',
+			'face_detector_angles_checkbox_group': 'FACE DETECTOR ANGLES',
+			'face_detector_model_dropdown': 'FACE DETECTOR MODEL',
+			'face_detector_margin_slider': 'FACE DETECTOR MARGIN',
+			'face_detector_score_slider': 'FACE DETECTOR SCORE',
+			'face_detector_size_dropdown': 'FACE DETECTOR SIZE',
+			'face_landmarker_model_dropdown': 'FACE LANDMARKER MODEL',
+			'face_landmarker_score_slider': 'FACE LANDMARKER SCORE',
+			'face_mask_blur_slider': 'FACE MASK BLUR',
+			'face_mask_padding_bottom_slider': 'FACE MASK PADDING BOTTOM',
+			'face_mask_padding_left_slider': 'FACE MASK PADDING LEFT',
+			'face_mask_padding_right_slider': 'FACE MASK PADDING RIGHT',
+			'face_mask_padding_top_slider': 'FACE MASK PADDING TOP',
+			'face_mask_areas_checkbox_group': 'FACE MASK AREAS',
+			'face_mask_regions_checkbox_group': 'FACE MASK REGIONS',
+			'face_mask_types_checkbox_group': 'FACE MASK TYPES',
+			'face_selector_age_range_slider': 'FACE SELECTOR AGE',
+			'face_selector_gender_dropdown': 'FACE SELECTOR GENDER',
+			'face_selector_mode_dropdown': 'FACE SELECTOR MODE',
+			'face_selector_order_dropdown': 'FACE SELECTOR ORDER',
+			'face_selector_race_dropdown': 'FACE SELECTOR RACE',
+			'face_occluder_model_dropdown': 'FACE OCCLUDER MODEL',
+			'face_parser_model_dropdown': 'FACE PARSER MODEL',
+			'voice_extractor_model_dropdown': 'VOICE EXTRACTOR MODEL',
+			'job_list_status_checkbox_group': 'JOB STATUS',
+			'job_manager_job_action_dropdown': 'JOB_ACTION',
+			'job_manager_job_id_dropdown': 'JOB ID',
+			'job_manager_step_index_dropdown': 'STEP INDEX',
+			'job_runner_job_action_dropdown': 'JOB ACTION',
+			'job_runner_job_id_dropdown': 'JOB ID',
+			'log_level_dropdown': 'LOG LEVEL',
+			'output_audio_encoder_dropdown': 'OUTPUT AUDIO ENCODER',
+			'output_audio_quality_slider': 'OUTPUT AUDIO QUALITY',
+			'output_audio_volume_slider': 'OUTPUT AUDIO VOLUME',
+			'output_image_or_video': 'OUTPUT',
+			'output_image_quality_slider': 'OUTPUT IMAGE QUALITY',
+			'output_image_scale_slider': 'OUTPUT IMAGE SCALE',
+			'output_path_textbox': 'OUTPUT PATH',
+			'output_video_encoder_dropdown': 'OUTPUT VIDEO ENCODER',
+			'output_video_fps_slider': 'OUTPUT VIDEO FPS',
+			'output_video_preset_dropdown': 'OUTPUT VIDEO PRESET',
+			'output_video_quality_slider': 'OUTPUT VIDEO QUALITY',
+			'output_video_scale_slider': 'OUTPUT VIDEO SCALE',
+			'preview_frame_slider': 'PREVIEW FRAME',
+			'preview_image': 'PREVIEW',
+			'preview_mode_dropdown': 'PREVIEW MODE',
+			'preview_resolution_dropdown': 'PREVIEW RESOLUTION',
+			'processors_checkbox_group': 'PROCESSORS',
+			'reference_face_distance_slider': 'REFERENCE FACE DISTANCE',
+			'reference_face_gallery': 'REFERENCE FACE',
+			'refresh_button': 'REFRESH',
+			'source_file': 'SOURCE',
+			'start_button': 'START',
+			'stop_button': 'STOP',
+			'target_file': 'TARGET',
+			'temp_frame_format_dropdown': 'TEMP FRAME FORMAT',
+			'terminal_textbox': 'TERMINAL',
+			'trim_frame_slider': 'TRIM FRAME',
+			'video_memory_strategy_dropdown': 'VIDEO MEMORY STRATEGY',
+			'webcam_fps_slider': 'WEBCAM FPS',
+			'webcam_image': 'WEBCAM',
+			'webcam_device_id_dropdown': 'WEBCAM DEVICE ID',
+			'webcam_mode_radio': 'WEBCAM MODE',
+			'webcam_resolution_dropdown': 'WEBCAM RESOLUTION'
+		}
+	}
+}

facefusion/media_helper.py

@@ -0,0 +1,17 @@
+from typing import Optional, Tuple
+
+
+def restrict_trim_frame(frame_total : int, trim_frame_start : Optional[int], trim_frame_end : Optional[int]) -> Tuple[int, int]:
+	if isinstance(trim_frame_start, int):
+		trim_frame_start = max(0, min(trim_frame_start, frame_total))
+	if isinstance(trim_frame_end, int):
+		trim_frame_end = max(0, min(trim_frame_end, frame_total))
+
+	if isinstance(trim_frame_start, int) and isinstance(trim_frame_end, int):
+		return trim_frame_start, trim_frame_end
+	if isinstance(trim_frame_start, int):
+		return trim_frame_start, frame_total
+	if isinstance(trim_frame_end, int):
+		return 0, trim_frame_end
+
+	return 0, frame_total

facefusion/memory.py

@@ -1,21 +0,0 @@
-from facefusion.common_helper import is_macos, is_windows
-
-if is_windows():
-	import ctypes
-else:
-	import resource
-
-
-def limit_system_memory(system_memory_limit : int = 1) -> bool:
-	if is_macos():
-		system_memory_limit = system_memory_limit * (1024 ** 6)
-	else:
-		system_memory_limit = system_memory_limit * (1024 ** 3)
-	try:
-		if is_windows():
-			ctypes.windll.kernel32.SetProcessWorkingSetSize(-1, ctypes.c_size_t(system_memory_limit), ctypes.c_size_t(system_memory_limit)) #type:ignore[attr-defined]
-		else:
-			resource.setrlimit(resource.RLIMIT_DATA, (system_memory_limit, system_memory_limit))
-		return True
-	except Exception:
-		return False

facefusion/metadata.py

@@ -4,7 +4,7 @@ METADATA =\
 {
 	'name': 'FaceFusion',
 	'description': 'Industry leading face manipulation platform',
-	'version': '3.3.1',
+	'version': 'v4',
 	'license': 'OpenRAIL-AS',
 	'author': 'Henry Ruhs',
 	'url': 'https://facefusion.io'
@@ -12,6 +12,4 @@ METADATA =\
 
 
 def get(key : str) -> Optional[str]:
-	if key in METADATA:
-		return METADATA.get(key)
-	return None
+	return METADATA.get(key)

facefusion/model_helper.py

@@ -5,7 +5,7 @@ import onnx
 from facefusion.types import ModelInitializer
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def get_static_model_initializer(model_path : str) -> ModelInitializer:
 	model = onnx.load(model_path)
 	return onnx.numpy_helper.to_array(model.graph.initializer[-1])

facefusion/normalizer.py

@@ -1,17 +1,29 @@
 from typing import List, Optional
 
-from facefusion.types import Fps, Padding
+from facefusion.types import Color, Fps, Padding
 
 
-def normalize_padding(padding : Optional[List[int]]) -> Optional[Padding]:
-	if padding and len(padding) == 1:
-		return tuple([ padding[0] ] * 4) #type:ignore[return-value]
-	if padding and len(padding) == 2:
-		return tuple([ padding[0], padding[1], padding[0], padding[1] ]) #type:ignore[return-value]
-	if padding and len(padding) == 3:
-		return tuple([ padding[0], padding[1], padding[2], padding[1] ]) #type:ignore[return-value]
-	if padding and len(padding) == 4:
-		return tuple(padding) #type:ignore[return-value]
+def normalize_color(channels : Optional[List[int]]) -> Optional[Color]:
+	if channels and len(channels) == 1:
+		return tuple([ channels[0], channels[0], channels[0], 255 ]) #type:ignore[return-value]
+	if channels and len(channels) == 2:
+		return tuple([ channels[0], channels[1], channels[0], 255 ]) #type:ignore[return-value]
+	if channels and len(channels) == 3:
+		return tuple([ channels[0], channels[1], channels[2], 255 ]) #type:ignore[return-value]
+	if channels and len(channels) == 4:
+		return tuple(channels) #type:ignore[return-value]
+	return None
+
+
+def normalize_space(spaces : Optional[List[int]]) -> Optional[Padding]:
+	if spaces and len(spaces) == 1:
+		return tuple([spaces[0]] * 4) #type:ignore[return-value]
+	if spaces and len(spaces) == 2:
+		return tuple([spaces[0], spaces[1], spaces[0], spaces[1]]) #type:ignore[return-value]
+	if spaces and len(spaces) == 3:
+		return tuple([spaces[0], spaces[1], spaces[2], spaces[1]]) #type:ignore[return-value]
+	if spaces and len(spaces) == 4:
+		return tuple(spaces) #type:ignore[return-value]
 	return None
 
 

facefusion/process_manager.py

@@ -1,6 +1,4 @@
-from typing import Generator, List
-
-from facefusion.types import ProcessState, QueuePayload
+from facefusion.types import ProcessState
 
 PROCESS_STATE : ProcessState = 'pending'
 
@@ -45,9 +43,3 @@ def stop() -> None:
 
 def end() -> None:
 	set_process_state('pending')
-
-
-def manage(queue_payloads : List[QueuePayload]) -> Generator[QueuePayload, None, None]:
-	for query_payload in queue_payloads:
-		if is_processing():
-			yield query_payload

facefusion/processors/core.py

@@ -1,15 +1,10 @@
 import importlib
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from queue import Queue
 from types import ModuleType
 from typing import Any, List
 
-from tqdm import tqdm
-
-from facefusion import logger, state_manager, wording
+from facefusion import logger, translator
 from facefusion.exit_helper import hard_exit
-from facefusion.types import ProcessFrames, QueuePayload
+
 
 PROCESSORS_METHODS =\
 [
@@ -20,26 +15,22 @@ PROCESSORS_METHODS =\
 	'pre_check',
 	'pre_process',
 	'post_process',
-	'get_reference_frame',
-	'process_frame',
-	'process_frames',
-	'process_image',
-	'process_video'
+	'process_frame'
 ]
 
 
 def load_processor_module(processor : str) -> Any:
 	try:
-		processor_module = importlib.import_module('facefusion.processors.modules.' + processor)
+		processor_module = importlib.import_module('facefusion.processors.modules.' + processor + '.core')
 		for method_name in PROCESSORS_METHODS:
 			if not hasattr(processor_module, method_name):
 				raise NotImplementedError
 	except ModuleNotFoundError as exception:
-		logger.error(wording.get('processor_not_loaded').format(processor = processor), __name__)
+		logger.error(translator.get('processor_not_loaded').format(processor = processor), __name__)
 		logger.debug(exception.msg, __name__)
 		hard_exit(1)
 	except NotImplementedError:
-		logger.error(wording.get('processor_not_implemented').format(processor = processor), __name__)
+		logger.error(translator.get('processor_not_implemented').format(processor = processor), __name__)
 		hard_exit(1)
 	return processor_module
 
@@ -51,49 +42,3 @@ def get_processors_modules(processors : List[str]) -> List[ModuleType]:
 		processor_module = load_processor_module(processor)
 		processor_modules.append(processor_module)
 	return processor_modules
-
-
-def multi_process_frames(source_paths : List[str], temp_frame_paths : List[str], process_frames : ProcessFrames) -> None:
-	queue_payloads = create_queue_payloads(temp_frame_paths)
-	with tqdm(total = len(queue_payloads), desc = wording.get('processing'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
-		progress.set_postfix(execution_providers = state_manager.get_item('execution_providers'))
-		with ThreadPoolExecutor(max_workers = state_manager.get_item('execution_thread_count')) as executor:
-			futures = []
-			queue : Queue[QueuePayload] = create_queue(queue_payloads)
-			queue_per_future = max(len(queue_payloads) // state_manager.get_item('execution_thread_count') * state_manager.get_item('execution_queue_count'), 1)
-
-			while not queue.empty():
-				future = executor.submit(process_frames, source_paths, pick_queue(queue, queue_per_future), progress.update)
-				futures.append(future)
-
-			for future_done in as_completed(futures):
-				future_done.result()
-
-
-def create_queue(queue_payloads : List[QueuePayload]) -> Queue[QueuePayload]:
-	queue : Queue[QueuePayload] = Queue()
-	for queue_payload in queue_payloads:
-		queue.put(queue_payload)
-	return queue
-
-
-def pick_queue(queue : Queue[QueuePayload], queue_per_future : int) -> List[QueuePayload]:
-	queues = []
-	for _ in range(queue_per_future):
-		if not queue.empty():
-			queues.append(queue.get())
-	return queues
-
-
-def create_queue_payloads(temp_frame_paths : List[str]) -> List[QueuePayload]:
-	queue_payloads = []
-	temp_frame_paths = sorted(temp_frame_paths, key = os.path.basename)
-
-	for frame_number, frame_path in enumerate(temp_frame_paths):
-		frame_payload : QueuePayload =\
-		{
-			'frame_number': frame_number,
-			'frame_path': frame_path
-		}
-		queue_payloads.append(frame_payload)
-	return queue_payloads

facefusion/processors/live_portrait.py

@@ -63,15 +63,15 @@ def limit_expression(expression : LivePortraitExpression) -> LivePortraitExpress
 	return numpy.clip(expression, EXPRESSION_MIN, EXPRESSION_MAX)
 
 
-def limit_euler_angles(target_pitch : LivePortraitPitch, target_yaw : LivePortraitYaw, target_roll : LivePortraitRoll, output_pitch : LivePortraitPitch, output_yaw : LivePortraitYaw, output_roll : LivePortraitRoll) -> Tuple[LivePortraitPitch, LivePortraitYaw, LivePortraitRoll]:
-	pitch_min, pitch_max, yaw_min, yaw_max, roll_min, roll_max = calc_euler_limits(target_pitch, target_yaw, target_roll)
+def limit_angle(target_pitch : LivePortraitPitch, target_yaw : LivePortraitYaw, target_roll : LivePortraitRoll, output_pitch : LivePortraitPitch, output_yaw : LivePortraitYaw, output_roll : LivePortraitRoll) -> Tuple[LivePortraitPitch, LivePortraitYaw, LivePortraitRoll]:
+	pitch_min, pitch_max, yaw_min, yaw_max, roll_min, roll_max = calculate_euler_limits(target_pitch, target_yaw, target_roll)
 	output_pitch = numpy.clip(output_pitch, pitch_min, pitch_max)
 	output_yaw = numpy.clip(output_yaw, yaw_min, yaw_max)
 	output_roll = numpy.clip(output_roll, roll_min, roll_max)
 	return output_pitch, output_yaw, output_roll
 
 
-def calc_euler_limits(pitch : LivePortraitPitch, yaw : LivePortraitYaw, roll : LivePortraitRoll) -> Tuple[float, float, float, float, float, float]:
+def calculate_euler_limits(pitch : LivePortraitPitch, yaw : LivePortraitYaw, roll : LivePortraitRoll) -> Tuple[float, float, float, float, float, float]:
 	pitch_min = -30.0
 	pitch_max = 30.0
 	yaw_min = -60.0

facefusion/processors/modules/age_modifier/choices.py

@@ -0,0 +1,8 @@
+from typing import List, Sequence
+
+from facefusion.common_helper import create_int_range
+from facefusion.processors.modules.age_modifier.types import AgeModifierModel
+
+age_modifier_models : List[AgeModifierModel] = [ 'styleganex_age' ]
+
+age_modifier_direction_range : Sequence[int] = create_int_range(-100, 100, 1)

facefusion/processors/modules/age_modifier/core.py

@@ -1,38 +1,42 @@
 from argparse import ArgumentParser
 from functools import lru_cache
-from typing import List
 
 import cv2
 import numpy
 
+import facefusion.args_store
 import facefusion.choices
 import facefusion.jobs.job_manager
-import facefusion.jobs.job_store
-import facefusion.processors.core as processors
-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording
-from facefusion.common_helper import create_int_metavar
+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, state_manager, translator, video_manager
+from facefusion.common_helper import create_int_metavar, is_macos
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
 from facefusion.execution import has_execution_provider
-from facefusion.face_analyser import get_many_faces, get_one_face
+from facefusion.face_analyser import scale_face
 from facefusion.face_helper import merge_matrix, paste_back, scale_face_landmark_5, warp_face_by_face_landmark_5
 from facefusion.face_masker import create_box_mask, create_occlusion_mask
-from facefusion.face_selector import find_similar_faces, sort_and_filter_faces
-from facefusion.face_store import get_reference_faces
-from facefusion.filesystem import in_directory, is_image, is_video, resolve_relative_path, same_file_extension
-from facefusion.processors import choices as processors_choices
-from facefusion.processors.types import AgeModifierDirection, AgeModifierInputs
+from facefusion.face_selector import select_faces
+from facefusion.filesystem import in_directory, is_image, is_video, resolve_relative_path
+from facefusion.processors.modules.age_modifier import choices as age_modifier_choices
+from facefusion.processors.modules.age_modifier.types import AgeModifierDirection, AgeModifierInputs
+from facefusion.processors.types import ProcessorOutputs
 from facefusion.program_helper import find_argument_group
 from facefusion.thread_helper import thread_semaphore
-from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, ModelOptions, ModelSet, ProcessMode, QueuePayload, UpdateProgress, VisionFrame
-from facefusion.vision import match_frame_color, read_image, read_static_image, write_image
+from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, ModelOptions, ModelSet, ProcessMode, VisionFrame
+from facefusion.vision import match_frame_color, read_static_image, read_static_video_frame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'styleganex_age':
 		{
+			'__metadata__':
+			{
+				'vendor': 'williamyang1991',
+				'license': 'S-Lab-1.0',
+				'year': 2023
+			},
 			'hashes':
 			{
 				'age_modifier':
@@ -83,9 +87,9 @@ def get_model_options() -> ModelOptions:
 def register_args(program : ArgumentParser) -> None:
 	group_processors = find_argument_group(program, 'processors')
 	if group_processors:
-		group_processors.add_argument('--age-modifier-model', help = wording.get('help.age_modifier_model'), default = config.get_str_value('processors', 'age_modifier_model', 'styleganex_age'), choices = processors_choices.age_modifier_models)
-		group_processors.add_argument('--age-modifier-direction', help = wording.get('help.age_modifier_direction'), type = int, default = config.get_int_value('processors', 'age_modifier_direction', '0'), choices = processors_choices.age_modifier_direction_range, metavar = create_int_metavar(processors_choices.age_modifier_direction_range))
-		facefusion.jobs.job_store.register_step_keys([ 'age_modifier_model', 'age_modifier_direction' ])
+		group_processors.add_argument('--age-modifier-model', help = translator.get('help.model', __package__), default = config.get_str_value('processors', 'age_modifier_model', 'styleganex_age'), choices = age_modifier_choices.age_modifier_models)
+		group_processors.add_argument('--age-modifier-direction', help = translator.get('help.direction', __package__), type = int, default = config.get_int_value('processors', 'age_modifier_direction', '0'), choices = age_modifier_choices.age_modifier_direction_range, metavar = create_int_metavar(age_modifier_choices.age_modifier_direction_range))
+		facefusion.args_store.register_args([ 'age_modifier_model', 'age_modifier_direction' ], scopes = [ 'api', 'cli' ])
 
 
 def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
@@ -102,19 +106,17 @@ def pre_check() -> bool:
 
 def pre_process(mode : ProcessMode) -> bool:
 	if mode in [ 'output', 'preview' ] and not is_image(state_manager.get_item('target_path')) and not is_video(state_manager.get_item('target_path')):
-		logger.error(wording.get('choose_image_or_video_target') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('choose_image_or_video_target') + translator.get('exclamation_mark'), __name__)
 		return False
 	if mode == 'output' and not in_directory(state_manager.get_item('output_path')):
-		logger.error(wording.get('specify_image_or_video_output') + wording.get('exclamation_mark'), __name__)
-		return False
-	if mode == 'output' and not same_file_extension(state_manager.get_item('target_path'), state_manager.get_item('output_path')):
-		logger.error(wording.get('match_target_and_output_extension') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('specify_image_or_video_output') + translator.get('exclamation_mark'), __name__)
 		return False
 	return True
 
 
 def post_process() -> None:
 	read_static_image.cache_clear()
+	read_static_video_frame.cache_clear()
 	video_manager.clear_video_pool()
 	if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:
 		clear_inference_pool()
@@ -143,8 +145,8 @@ def modify_age(target_face : Face, temp_vision_frame : VisionFrame) -> VisionFra
 
 	if 'occlusion' in state_manager.get_item('face_mask_types'):
 		occlusion_mask = create_occlusion_mask(crop_vision_frame)
-		combined_matrix = merge_matrix([ extend_affine_matrix, cv2.invertAffineTransform(affine_matrix) ])
-		occlusion_mask = cv2.warpAffine(occlusion_mask, combined_matrix, model_sizes.get('target_with_background'))
+		temp_matrix = merge_matrix([ extend_affine_matrix, cv2.invertAffineTransform(affine_matrix) ])
+		occlusion_mask = cv2.warpAffine(occlusion_mask, temp_matrix, model_sizes.get('target_with_background'))
 		crop_masks.append(occlusion_mask)
 
 	crop_vision_frame = prepare_vision_frame(crop_vision_frame)
@@ -164,7 +166,7 @@ def forward(crop_vision_frame : VisionFrame, extend_vision_frame : VisionFrame,
 	age_modifier = get_inference_pool().get('age_modifier')
 	age_modifier_inputs = {}
 
-	if has_execution_provider('coreml'):
+	if is_macos() and has_execution_provider('coreml'):
 		age_modifier.set_providers([ facefusion.choices.execution_provider_set.get('cpu') ])
 
 	for age_modifier_input in age_modifier.get_inputs():
@@ -199,56 +201,16 @@ def normalize_extend_frame(extend_vision_frame : VisionFrame) -> VisionFrame:
 	return extend_vision_frame
 
 
-def get_reference_frame(source_face : Face, target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
-	return modify_age(target_face, temp_vision_frame)
-
-
-def process_frame(inputs : AgeModifierInputs) -> VisionFrame:
-	reference_faces = inputs.get('reference_faces')
+def process_frame(inputs : AgeModifierInputs) -> ProcessorOutputs:
+	reference_vision_frame = inputs.get('reference_vision_frame')
 	target_vision_frame = inputs.get('target_vision_frame')
-	many_faces = sort_and_filter_faces(get_many_faces([ target_vision_frame ]))
+	temp_vision_frame = inputs.get('temp_vision_frame')
+	temp_vision_mask = inputs.get('temp_vision_mask')
+	target_faces = select_faces(reference_vision_frame, target_vision_frame)
 
-	if state_manager.get_item('face_selector_mode') == 'many':
-		if many_faces:
-			for target_face in many_faces:
-				target_vision_frame = modify_age(target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'one':
-		target_face = get_one_face(many_faces)
-		if target_face:
-			target_vision_frame = modify_age(target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'reference':
-		similar_faces = find_similar_faces(many_faces, reference_faces, state_manager.get_item('reference_face_distance'))
-		if similar_faces:
-			for similar_face in similar_faces:
-				target_vision_frame = modify_age(similar_face, target_vision_frame)
-	return target_vision_frame
+	if target_faces:
+		for target_face in target_faces:
+			target_face = scale_face(target_face, target_vision_frame, temp_vision_frame)
+			temp_vision_frame = modify_age(target_face, temp_vision_frame)
 
-
-def process_frames(source_path : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-
-	for queue_payload in process_manager.manage(queue_payloads):
-		target_vision_path = queue_payload['frame_path']
-		target_vision_frame = read_image(target_vision_path)
-		output_vision_frame = process_frame(
-		{
-			'reference_faces': reference_faces,
-			'target_vision_frame': target_vision_frame
-		})
-		write_image(target_vision_path, output_vision_frame)
-		update_progress(1)
-
-
-def process_image(source_path : str, target_path : str, output_path : str) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-	target_vision_frame = read_static_image(target_path)
-	output_vision_frame = process_frame(
-	{
-		'reference_faces': reference_faces,
-		'target_vision_frame': target_vision_frame
-	})
-	write_image(output_path, output_vision_frame)
-
-
-def process_video(source_paths : List[str], temp_frame_paths : List[str]) -> None:
-	processors.multi_process_frames(None, temp_frame_paths, process_frames)
+	return temp_vision_frame, temp_vision_mask

facefusion/processors/modules/age_modifier/locals.py

@@ -0,0 +1,18 @@
+from facefusion.types import Locals
+
+LOCALS : Locals =\
+{
+	'en':
+	{
+		'help':
+		{
+			'model': 'choose the model responsible for aging the face',
+			'direction': 'specify the direction in which the age should be modified'
+		},
+		'uis':
+		{
+			'direction_slider': 'AGE MODIFIER DIRECTION',
+			'model_dropdown': 'AGE MODIFIER MODEL'
+		}
+	}
+}

facefusion/processors/modules/age_modifier/types.py

@@ -0,0 +1,17 @@
+from typing import Any, Literal, TypeAlias, TypedDict
+
+from numpy.typing import NDArray
+
+from facefusion.types import Mask, VisionFrame
+
+AgeModifierInputs = TypedDict('AgeModifierInputs',
+{
+	'reference_vision_frame' : VisionFrame,
+	'target_vision_frame' : VisionFrame,
+	'temp_vision_frame' : VisionFrame,
+	'temp_vision_mask' : Mask
+})
+
+AgeModifierModel = Literal['styleganex_age']
+
+AgeModifierDirection : TypeAlias = NDArray[Any]

facefusion/processors/modules/background_remover/choices.py

@@ -0,0 +1,8 @@
+from typing import List, Sequence
+
+from facefusion.common_helper import create_int_range
+from facefusion.processors.modules.background_remover.types import BackgroundRemoverModel
+
+background_remover_models : List[BackgroundRemoverModel] = [ 'ben_2', 'birefnet_general', 'birefnet_portrait', 'isnet_general', 'modnet', 'ormbg', 'rmbg_1.4', 'rmbg_2.0', 'silueta', 'u2net_cloth', 'u2net_general', 'u2net_human', 'u2netp' ]
+
+background_remover_color_range : Sequence[int] = create_int_range(0, 255, 1)

facefusion/processors/modules/background_remover/core.py

@@ -0,0 +1,521 @@
+from argparse import ArgumentParser
+from functools import lru_cache, partial
+from typing import List, Tuple
+
+import cv2
+import numpy
+
+import facefusion.args_store
+import facefusion.jobs.job_manager
+from facefusion import config, content_analyser, inference_manager, logger, state_manager, translator, video_manager
+from facefusion.common_helper import is_macos
+from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
+from facefusion.execution import has_execution_provider
+from facefusion.filesystem import in_directory, is_image, is_video, resolve_relative_path
+from facefusion.normalizer import normalize_color
+from facefusion.processors.modules.background_remover import choices as background_remover_choices
+from facefusion.processors.modules.background_remover.types import BackgroundRemoverInputs
+from facefusion.processors.types import ProcessorOutputs
+from facefusion.program_helper import find_argument_group
+from facefusion.sanitizer import sanitize_int_range
+from facefusion.thread_helper import thread_semaphore
+from facefusion.types import ApplyStateItem, Args, DownloadScope, ExecutionProvider, InferencePool, Mask, ModelOptions, ModelSet, ProcessMode, VisionFrame
+from facefusion.vision import read_static_image, read_static_video_frame
+
+
+@lru_cache()
+def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
+	return\
+	{
+		'ben_2':
+		{
+			'__metadata__':
+			{
+				'vendor': 'PramaLLC',
+				'license': 'MIT',
+				'year': 2025
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'ben_2.hash'),
+					'path': resolve_relative_path('../.assets/models/ben_2.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'ben_2.onnx'),
+					'path': resolve_relative_path('../.assets/models/ben_2.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.0, 0.0, 0.0 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'birefnet_general':
+		{
+			'__metadata__':
+			{
+				'vendor': 'ZhengPeng7',
+				'license': 'MIT',
+				'year': 2024
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'birefnet_general.hash'),
+					'path': resolve_relative_path('../.assets/models/birefnet_general.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'birefnet_general.onnx'),
+					'path': resolve_relative_path('../.assets/models/birefnet_general.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.0, 0.0, 0.0 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'birefnet_portrait':
+		{
+			'__metadata__':
+			{
+				'vendor': 'ZhengPeng7',
+				'license': 'MIT',
+				'year': 2024
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'birefnet_portrait.hash'),
+					'path': resolve_relative_path('../.assets/models/birefnet_portrait.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'birefnet_portrait.onnx'),
+					'path': resolve_relative_path('../.assets/models/birefnet_portrait.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.0, 0.0, 0.0 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'isnet_general':
+		{
+			'__metadata__':
+			{
+				'vendor': 'xuebinqin',
+				'license': 'Apache-2.0',
+				'year': 2022
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'isnet_general.hash'),
+					'path': resolve_relative_path('../.assets/models/isnet_general.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'isnet_general.onnx'),
+					'path': resolve_relative_path('../.assets/models/isnet_general.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.5, 0.5, 0.5 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'modnet':
+		{
+			'__metadata__':
+			{
+				'vendor': 'ZHKKKe',
+				'license': 'Apache-2.0',
+				'year': 2020
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'modnet.hash'),
+					'path': resolve_relative_path('../.assets/models/modnet.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'modnet.onnx'),
+					'path': resolve_relative_path('../.assets/models/modnet.onnx')
+				}
+			},
+			'size': (512, 512),
+			'mean': [ 0.5, 0.5, 0.5 ],
+			'standard_deviation': [ 0.5, 0.5, 0.5 ]
+		},
+		'ormbg':
+		{
+			'__metadata__':
+			{
+				'vendor': 'schirrmacher',
+				'license': 'Apache-2.0',
+				'year': 2024
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'ormbg.hash'),
+					'path': resolve_relative_path('../.assets/models/ormbg.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'ormbg.onnx'),
+					'path': resolve_relative_path('../.assets/models/ormbg.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.0, 0.0, 0.0 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'rmbg_1.4':
+		{
+			'__metadata__':
+			{
+				'vendor': 'Bria',
+				'license': 'Non-Commercial',
+				'year': 2023
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'rmbg_1.4.hash'),
+					'path': resolve_relative_path('../.assets/models/rmbg_1.4.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'rmbg_1.4.onnx'),
+					'path': resolve_relative_path('../.assets/models/rmbg_1.4.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.5, 0.5, 0.5 ],
+			'standard_deviation': [ 1.0, 1.0, 1.0 ]
+		},
+		'rmbg_2.0':
+		{
+			'__metadata__':
+			{
+				'vendor': 'Bria',
+				'license': 'Non-Commercial',
+				'year': 2024
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'rmbg_2.0.hash'),
+					'path': resolve_relative_path('../.assets/models/rmbg_2.0.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'rmbg_2.0.onnx'),
+					'path': resolve_relative_path('../.assets/models/rmbg_2.0.onnx')
+				}
+			},
+			'size': (1024, 1024),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		},
+		'silueta':
+		{
+			'__metadata__':
+			{
+				'vendor': 'Kikedao',
+				'license': 'Apache-2.0',
+				'year': 2022
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'silueta.hash'),
+					'path': resolve_relative_path('../.assets/models/silueta.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'silueta.onnx'),
+					'path': resolve_relative_path('../.assets/models/silueta.onnx')
+				}
+			},
+			'size': (320, 320),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		},
+		'u2net_cloth':
+		{
+			'__metadata__':
+			{
+				'vendor': 'levindabhi',
+				'license': 'MIT',
+				'year': 2021
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_cloth.hash'),
+					'path': resolve_relative_path('../.assets/models/u2net_cloth.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_cloth.onnx'),
+					'path': resolve_relative_path('../.assets/models/u2net_cloth.onnx')
+				}
+			},
+			'size': (768, 768),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		},
+		'u2net_general':
+		{
+			'__metadata__':
+			{
+				'vendor': 'xuebinqin',
+				'license': 'Apache-2.0',
+				'year': 2020
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_general.hash'),
+					'path': resolve_relative_path('../.assets/models/u2net_general.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_general.onnx'),
+					'path': resolve_relative_path('../.assets/models/u2net_general.onnx')
+				}
+			},
+			'size': (320, 320),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		},
+		'u2net_human':
+		{
+			'__metadata__':
+			{
+				'vendor': 'xuebinqin',
+				'license': 'Apache-2.0',
+				'year': 2021
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_human.hash'),
+					'path': resolve_relative_path('../.assets/models/u2net_human.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2net_human.onnx'),
+					'path': resolve_relative_path('../.assets/models/u2net_human.onnx')
+				}
+			},
+			'size': (320, 320),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		},
+		'u2netp':
+		{
+			'__metadata__':
+			{
+				'vendor': 'xuebinqin',
+				'license': 'Apache-2.0',
+				'year': 2021
+			},
+			'hashes':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2netp.hash'),
+					'path': resolve_relative_path('../.assets/models/u2netp.hash')
+				}
+			},
+			'sources':
+			{
+				'background_remover':
+				{
+					'url': resolve_download_url('models-3.5.0', 'u2netp.onnx'),
+					'path': resolve_relative_path('../.assets/models/u2netp.onnx')
+				}
+			},
+			'size': (320, 320),
+			'mean': [ 0.485, 0.456, 0.406 ],
+			'standard_deviation': [ 0.229, 0.224, 0.225 ]
+		}
+	}
+
+
+def get_inference_pool() -> InferencePool:
+	model_names = [ state_manager.get_item('background_remover_model') ]
+	model_source_set = get_model_options().get('sources')
+
+	return inference_manager.get_inference_pool(__name__, model_names, model_source_set)
+
+
+def clear_inference_pool() -> None:
+	model_names = [ state_manager.get_item('background_remover_model') ]
+	inference_manager.clear_inference_pool(__name__, model_names)
+
+
+def resolve_execution_providers() -> List[ExecutionProvider]:
+	if is_macos() and has_execution_provider('coreml'):
+		return [ 'cpu' ]
+	return state_manager.get_item('execution_providers')
+
+
+def get_model_options() -> ModelOptions:
+	model_name = state_manager.get_item('background_remover_model')
+	return create_static_model_set('full').get(model_name)
+
+
+def register_args(program : ArgumentParser) -> None:
+	group_processors = find_argument_group(program, 'processors')
+	if group_processors:
+		group_processors.add_argument('--background-remover-model', help = translator.get('help.model', __package__), default = config.get_str_value('processors', 'background_remover_model', 'rmbg_2.0'), choices = background_remover_choices.background_remover_models)
+		group_processors.add_argument('--background-remover-color', help = translator.get('help.color', __package__), type = partial(sanitize_int_range, int_range = background_remover_choices.background_remover_color_range), default = config.get_int_list('processors', 'background_remover_color', '0 0 0 0'), nargs ='+')
+		facefusion.args_store.register_args([ 'background_remover_model', 'background_remover_color' ], scopes = [ 'api', 'cli' ])
+
+
+def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
+	apply_state_item('background_remover_model', args.get('background_remover_model'))
+	apply_state_item('background_remover_color', normalize_color(args.get('background_remover_color')))
+
+
+def pre_check() -> bool:
+	model_hash_set = get_model_options().get('hashes')
+	model_source_set = get_model_options().get('sources')
+
+	return conditional_download_hashes(model_hash_set) and conditional_download_sources(model_source_set)
+
+
+def pre_process(mode : ProcessMode) -> bool:
+	if mode in [ 'output', 'preview' ] and not is_image(state_manager.get_item('target_path')) and not is_video(state_manager.get_item('target_path')):
+		logger.error(translator.get('choose_image_or_video_target') + translator.get('exclamation_mark'), __name__)
+		return False
+	if mode == 'output' and not in_directory(state_manager.get_item('output_path')):
+		logger.error(translator.get('specify_image_or_video_output') + translator.get('exclamation_mark'), __name__)
+		return False
+	return True
+
+
+def post_process() -> None:
+	read_static_image.cache_clear()
+	read_static_video_frame.cache_clear()
+	video_manager.clear_video_pool()
+	if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:
+		clear_inference_pool()
+	if state_manager.get_item('video_memory_strategy') == 'strict':
+		content_analyser.clear_inference_pool()
+
+
+def remove_background(temp_vision_frame : VisionFrame) -> Tuple[VisionFrame, Mask]:
+	temp_vision_mask = forward(prepare_temp_frame(temp_vision_frame))
+	temp_vision_mask = normalize_vision_mask(temp_vision_mask)
+	temp_vision_mask = cv2.resize(temp_vision_mask, temp_vision_frame.shape[:2][::-1])
+	temp_vision_frame = apply_background_color(temp_vision_frame, temp_vision_mask)
+	return temp_vision_frame, temp_vision_mask
+
+
+def forward(temp_vision_frame : VisionFrame) -> VisionFrame:
+	background_remover = get_inference_pool().get('background_remover')
+	model_name = state_manager.get_item('background_remover_model')
+
+	with thread_semaphore():
+		remove_vision_frame = background_remover.run(None,
+		{
+			'input': temp_vision_frame
+		})[0]
+
+		if model_name == 'u2net_cloth':
+			remove_vision_frame = numpy.argmax(remove_vision_frame, axis = 1)
+
+	return remove_vision_frame
+
+
+def prepare_temp_frame(temp_vision_frame : VisionFrame) -> VisionFrame:
+	model_size = get_model_options().get('size')
+	model_mean = get_model_options().get('mean')
+	model_standard_deviation = get_model_options().get('standard_deviation')
+
+	temp_vision_frame = cv2.resize(temp_vision_frame, model_size)
+	temp_vision_frame = temp_vision_frame[:, :, ::-1] / 255.0
+	temp_vision_frame = (temp_vision_frame - model_mean) / model_standard_deviation
+	temp_vision_frame = temp_vision_frame.transpose(2, 0, 1)
+	temp_vision_frame = numpy.expand_dims(temp_vision_frame, axis = 0).astype(numpy.float32)
+	return temp_vision_frame
+
+
+def normalize_vision_mask(temp_vision_mask : Mask) -> Mask:
+	temp_vision_mask = numpy.squeeze(temp_vision_mask).clip(0, 1) * 255
+	temp_vision_mask = numpy.clip(temp_vision_mask, 0, 255).astype(numpy.uint8)
+	return temp_vision_mask
+
+
+def apply_background_color(temp_vision_frame : VisionFrame, temp_vision_mask : Mask) -> VisionFrame:
+	background_remover_color = state_manager.get_item('background_remover_color')
+	temp_vision_mask = temp_vision_mask.astype(numpy.float32) / 255
+	temp_vision_mask = numpy.expand_dims(temp_vision_mask, axis = 2)
+	temp_vision_mask = (1 - temp_vision_mask) * background_remover_color[-1] / 255
+	color_frame = numpy.zeros_like(temp_vision_frame)
+	color_frame[:, :, 0] = background_remover_color[2]
+	color_frame[:, :, 1] = background_remover_color[1]
+	color_frame[:, :, 2] = background_remover_color[0]
+	temp_vision_frame = temp_vision_frame * (1 - temp_vision_mask) + color_frame * temp_vision_mask
+	temp_vision_frame = temp_vision_frame.astype(numpy.uint8)
+	return temp_vision_frame
+
+
+def process_frame(inputs : BackgroundRemoverInputs) -> ProcessorOutputs:
+	temp_vision_frame = inputs.get('temp_vision_frame')
+	temp_vision_frame, temp_vision_mask = remove_background(temp_vision_frame)
+	temp_vision_mask = numpy.minimum.reduce([ temp_vision_mask, inputs.get('temp_vision_mask') ])
+	return temp_vision_frame, temp_vision_mask

facefusion/processors/modules/background_remover/locals.py

@@ -0,0 +1,21 @@
+from facefusion.types import Locals
+
+LOCALS : Locals =\
+{
+	'en':
+	{
+		'help':
+		{
+			'model': 'choose the model responsible for removing the background',
+			'color': 'apply red, green blue and alpha values to the background'
+		},
+		'uis':
+		{
+			'model_dropdown': 'BACKGROUND REMOVER MODEL',
+			'color_red_number': 'BACKGROUND COLOR RED',
+			'color_green_number': 'BACKGROUND COLOR GREEN',
+			'color_blue_number': 'BACKGROUND COLOR BLUE',
+			'color_alpha_number': 'BACKGROUND COLOR ALPHA'
+		}
+	}
+}

facefusion/processors/modules/background_remover/types.py

@@ -0,0 +1,12 @@
+from typing import Literal, TypedDict
+
+from facefusion.types import Mask, VisionFrame
+
+BackgroundRemoverInputs = TypedDict('BackgroundRemoverInputs',
+{
+	'target_vision_frame' : VisionFrame,
+	'temp_vision_frame' : VisionFrame,
+	'temp_vision_mask' : Mask
+})
+
+BackgroundRemoverModel = Literal['ben_2', 'birefnet_general', 'birefnet_portrait', 'isnet_general', 'modnet', 'ormbg', 'rmbg_1.4', 'rmbg_2.0', 'silueta', 'u2net_cloth', 'u2net_general', 'u2net_human', 'u2netp']

facefusion/processors/modules/deep_swapper/choices.py

@@ -1,10 +1,9 @@
 from typing import List, Sequence
 
-from facefusion.common_helper import create_float_range, create_int_range
+from facefusion.common_helper import create_int_range
 from facefusion.filesystem import get_file_name, resolve_file_paths, resolve_relative_path
-from facefusion.processors.types import AgeModifierModel, DeepSwapperModel, ExpressionRestorerModel, FaceDebuggerItem, FaceEditorModel, FaceEnhancerModel, FaceSwapperModel, FaceSwapperSet, FrameColorizerModel, FrameEnhancerModel, LipSyncerModel
+from facefusion.processors.modules.deep_swapper.types import DeepSwapperModel
 
-age_modifier_models : List[AgeModifierModel] = [ 'styleganex_age' ]
 deep_swapper_models : List[DeepSwapperModel] =\
 [
 	'druuzil/adam_levine_320',
@@ -174,51 +173,4 @@ if custom_model_file_paths:
 		model_id = '/'.join([ 'custom', get_file_name(model_file_path) ])
 		deep_swapper_models.append(model_id)
 
-expression_restorer_models : List[ExpressionRestorerModel] = [ 'live_portrait' ]
-face_debugger_items : List[FaceDebuggerItem] = [ 'bounding-box', 'face-landmark-5', 'face-landmark-5/68', 'face-landmark-68', 'face-landmark-68/5', 'face-mask', 'face-detector-score', 'face-landmarker-score', 'age', 'gender', 'race' ]
-face_editor_models : List[FaceEditorModel] = [ 'live_portrait' ]
-face_enhancer_models : List[FaceEnhancerModel] = [ 'codeformer', 'gfpgan_1.2', 'gfpgan_1.3', 'gfpgan_1.4', 'gpen_bfr_256', 'gpen_bfr_512', 'gpen_bfr_1024', 'gpen_bfr_2048', 'restoreformer_plus_plus' ]
-face_swapper_set : FaceSwapperSet =\
-{
-	'blendswap_256': [ '256x256', '384x384', '512x512', '768x768', '1024x1024' ],
-	'ghost_1_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'ghost_2_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'ghost_3_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'hififace_unofficial_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'hyperswap_1a_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'hyperswap_1b_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'hyperswap_1c_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'inswapper_128': [ '128x128', '256x256', '384x384', '512x512', '768x768', '1024x1024' ],
-	'inswapper_128_fp16': [ '128x128', '256x256', '384x384', '512x512', '768x768', '1024x1024' ],
-	'simswap_256': [ '256x256', '512x512', '768x768', '1024x1024' ],
-	'simswap_unofficial_512': [ '512x512', '768x768', '1024x1024' ],
-	'uniface_256': [ '256x256', '512x512', '768x768', '1024x1024' ]
-}
-face_swapper_models : List[FaceSwapperModel] = list(face_swapper_set.keys())
-frame_colorizer_models : List[FrameColorizerModel] = [ 'ddcolor', 'ddcolor_artistic', 'deoldify', 'deoldify_artistic', 'deoldify_stable' ]
-frame_colorizer_sizes : List[str] = [ '192x192', '256x256', '384x384', '512x512' ]
-frame_enhancer_models : List[FrameEnhancerModel] = [ 'clear_reality_x4', 'lsdir_x4', 'nomos8k_sc_x4', 'real_esrgan_x2', 'real_esrgan_x2_fp16', 'real_esrgan_x4', 'real_esrgan_x4_fp16', 'real_esrgan_x8', 'real_esrgan_x8_fp16', 'real_hatgan_x4', 'real_web_photo_x4', 'realistic_rescaler_x4', 'remacri_x4', 'siax_x4', 'span_kendata_x4', 'swin2_sr_x4', 'ultra_sharp_x4', 'ultra_sharp_2_x4' ]
-lip_syncer_models : List[LipSyncerModel] = [ 'edtalk_256', 'wav2lip_96', 'wav2lip_gan_96' ]
-
-age_modifier_direction_range : Sequence[int] = create_int_range(-100, 100, 1)
 deep_swapper_morph_range : Sequence[int] = create_int_range(0, 100, 1)
-expression_restorer_factor_range : Sequence[int] = create_int_range(0, 100, 1)
-face_editor_eyebrow_direction_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_eye_gaze_horizontal_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_eye_gaze_vertical_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_eye_open_ratio_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_lip_open_ratio_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_grim_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_pout_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_purse_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_smile_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_position_horizontal_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_mouth_position_vertical_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_head_pitch_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_head_yaw_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_editor_head_roll_range : Sequence[float] = create_float_range(-1.0, 1.0, 0.05)
-face_enhancer_blend_range : Sequence[int] = create_int_range(0, 100, 1)
-face_enhancer_weight_range : Sequence[float] = create_float_range(0.0, 1.0, 0.05)
-frame_colorizer_blend_range : Sequence[int] = create_int_range(0, 100, 1)
-frame_enhancer_blend_range : Sequence[int] = create_int_range(0, 100, 1)
-lip_syncer_weight_range : Sequence[float] = create_float_range(0.0, 1.0, 0.05)

facefusion/processors/modules/deep_swapper/core.py

@@ -1,32 +1,31 @@
 from argparse import ArgumentParser
 from functools import lru_cache
-from typing import List, Tuple
+from typing import Tuple
 
 import cv2
 import numpy
 from cv2.typing import Size
 
+import facefusion.args_store
 import facefusion.jobs.job_manager
-import facefusion.jobs.job_store
-import facefusion.processors.core as processors
-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording
+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, state_manager, translator, video_manager
 from facefusion.common_helper import create_int_metavar
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
-from facefusion.face_analyser import get_many_faces, get_one_face
+from facefusion.face_analyser import scale_face
 from facefusion.face_helper import paste_back, warp_face_by_face_landmark_5
 from facefusion.face_masker import create_area_mask, create_box_mask, create_occlusion_mask, create_region_mask
-from facefusion.face_selector import find_similar_faces, sort_and_filter_faces
-from facefusion.face_store import get_reference_faces
-from facefusion.filesystem import get_file_name, in_directory, is_image, is_video, resolve_file_paths, resolve_relative_path, same_file_extension
-from facefusion.processors import choices as processors_choices
-from facefusion.processors.types import DeepSwapperInputs, DeepSwapperMorph
+from facefusion.face_selector import select_faces
+from facefusion.filesystem import get_file_name, in_directory, is_image, is_video, resolve_file_paths, resolve_relative_path
+from facefusion.processors.modules.deep_swapper import choices as deep_swapper_choices
+from facefusion.processors.modules.deep_swapper.types import DeepSwapperInputs, DeepSwapperMorph
+from facefusion.processors.types import ProcessorOutputs
 from facefusion.program_helper import find_argument_group
 from facefusion.thread_helper import thread_semaphore
-from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, Mask, ModelOptions, ModelSet, ProcessMode, QueuePayload, UpdateProgress, VisionFrame
-from facefusion.vision import conditional_match_frame_color, read_image, read_static_image, write_image
+from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, Mask, ModelOptions, ModelSet, ProcessMode, VisionFrame
+from facefusion.vision import conditional_match_frame_color, read_static_image, read_static_video_frame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	model_config = []
 
@@ -277,9 +276,9 @@ def get_model_size() -> Size:
 def register_args(program : ArgumentParser) -> None:
 	group_processors = find_argument_group(program, 'processors')
 	if group_processors:
-		group_processors.add_argument('--deep-swapper-model', help = wording.get('help.deep_swapper_model'), default = config.get_str_value('processors', 'deep_swapper_model', 'iperov/elon_musk_224'), choices = processors_choices.deep_swapper_models)
-		group_processors.add_argument('--deep-swapper-morph', help = wording.get('help.deep_swapper_morph'), type = int, default = config.get_int_value('processors', 'deep_swapper_morph', '100'), choices = processors_choices.deep_swapper_morph_range, metavar = create_int_metavar(processors_choices.deep_swapper_morph_range))
-		facefusion.jobs.job_store.register_step_keys([ 'deep_swapper_model', 'deep_swapper_morph' ])
+		group_processors.add_argument('--deep-swapper-model', help = translator.get('help.model', __package__), default = config.get_str_value('processors', 'deep_swapper_model', 'iperov/elon_musk_224'), choices = deep_swapper_choices.deep_swapper_models)
+		group_processors.add_argument('--deep-swapper-morph', help = translator.get('help.morph', __package__), type = int, default = config.get_int_value('processors', 'deep_swapper_morph', '100'), choices = deep_swapper_choices.deep_swapper_morph_range, metavar = create_int_metavar(deep_swapper_choices.deep_swapper_morph_range))
+		facefusion.args_store.register_args([ 'deep_swapper_model', 'deep_swapper_morph' ], scopes = [ 'api', 'cli' ])
 
 
 def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
@@ -298,19 +297,17 @@ def pre_check() -> bool:
 
 def pre_process(mode : ProcessMode) -> bool:
 	if mode in [ 'output', 'preview' ] and not is_image(state_manager.get_item('target_path')) and not is_video(state_manager.get_item('target_path')):
-		logger.error(wording.get('choose_image_or_video_target') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('choose_image_or_video_target') + translator.get('exclamation_mark'), __name__)
 		return False
 	if mode == 'output' and not in_directory(state_manager.get_item('output_path')):
-		logger.error(wording.get('specify_image_or_video_output') + wording.get('exclamation_mark'), __name__)
-		return False
-	if mode == 'output' and not same_file_extension(state_manager.get_item('target_path'), state_manager.get_item('output_path')):
-		logger.error(wording.get('match_target_and_output_extension') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('specify_image_or_video_output') + translator.get('exclamation_mark'), __name__)
 		return False
 	return True
 
 
 def post_process() -> None:
 	read_static_image.cache_clear()
+	read_static_video_frame.cache_clear()
 	video_manager.clear_video_pool()
 	if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:
 		clear_inference_pool()
@@ -409,56 +406,16 @@ def prepare_crop_mask(crop_source_mask : Mask, crop_target_mask : Mask) -> Mask:
 	return crop_mask
 
 
-def get_reference_frame(source_face : Face, target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
-	return swap_face(target_face, temp_vision_frame)
-
-
-def process_frame(inputs : DeepSwapperInputs) -> VisionFrame:
-	reference_faces = inputs.get('reference_faces')
+def process_frame(inputs : DeepSwapperInputs) -> ProcessorOutputs:
+	reference_vision_frame = inputs.get('reference_vision_frame')
 	target_vision_frame = inputs.get('target_vision_frame')
-	many_faces = sort_and_filter_faces(get_many_faces([ target_vision_frame ]))
+	temp_vision_frame = inputs.get('temp_vision_frame')
+	temp_vision_mask = inputs.get('temp_vision_mask')
+	target_faces = select_faces(reference_vision_frame, target_vision_frame)
 
-	if state_manager.get_item('face_selector_mode') == 'many':
-		if many_faces:
-			for target_face in many_faces:
-				target_vision_frame = swap_face(target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'one':
-		target_face = get_one_face(many_faces)
-		if target_face:
-			target_vision_frame = swap_face(target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'reference':
-		similar_faces = find_similar_faces(many_faces, reference_faces, state_manager.get_item('reference_face_distance'))
-		if similar_faces:
-			for similar_face in similar_faces:
-				target_vision_frame = swap_face(similar_face, target_vision_frame)
-	return target_vision_frame
+	if target_faces:
+		for target_face in target_faces:
+			target_face = scale_face(target_face, target_vision_frame, temp_vision_frame)
+			temp_vision_frame = swap_face(target_face, temp_vision_frame)
 
-
-def process_frames(source_path : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-
-	for queue_payload in process_manager.manage(queue_payloads):
-		target_vision_path = queue_payload['frame_path']
-		target_vision_frame = read_image(target_vision_path)
-		output_vision_frame = process_frame(
-		{
-			'reference_faces': reference_faces,
-			'target_vision_frame': target_vision_frame
-		})
-		write_image(target_vision_path, output_vision_frame)
-		update_progress(1)
-
-
-def process_image(source_path : str, target_path : str, output_path : str) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-	target_vision_frame = read_static_image(target_path)
-	output_vision_frame = process_frame(
-	{
-		'reference_faces': reference_faces,
-		'target_vision_frame': target_vision_frame
-	})
-	write_image(output_path, output_vision_frame)
-
-
-def process_video(source_paths : List[str], temp_frame_paths : List[str]) -> None:
-	processors.multi_process_frames(None, temp_frame_paths, process_frames)
+	return temp_vision_frame, temp_vision_mask

facefusion/processors/modules/deep_swapper/locals.py

@@ -0,0 +1,18 @@
+from facefusion.types import Locals
+
+LOCALS : Locals =\
+{
+	'en':
+	{
+		'help':
+		{
+			'model': 'choose the model responsible for swapping the face',
+			'morph': 'morph between source face and target faces'
+		},
+		'uis':
+		{
+			'model_dropdown': 'DEEP SWAPPER MODEL',
+			'morph_slider': 'DEEP SWAPPER MORPH'
+		}
+	}
+}

facefusion/processors/modules/deep_swapper/types.py

@@ -0,0 +1,17 @@
+from typing import Any, TypeAlias, TypedDict
+
+from numpy.typing import NDArray
+
+from facefusion.types import Mask, VisionFrame
+
+DeepSwapperInputs = TypedDict('DeepSwapperInputs',
+{
+	'reference_vision_frame' : VisionFrame,
+	'target_vision_frame' : VisionFrame,
+	'temp_vision_frame' : VisionFrame,
+	'temp_vision_mask' : Mask
+})
+
+DeepSwapperModel : TypeAlias = str
+
+DeepSwapperMorph : TypeAlias = NDArray[Any]

facefusion/processors/modules/expression_restorer/choices.py

@@ -0,0 +1,10 @@
+from typing import List, Sequence
+
+from facefusion.common_helper import create_int_range
+from facefusion.processors.modules.expression_restorer.types import ExpressionRestorerArea, ExpressionRestorerModel
+
+expression_restorer_models : List[ExpressionRestorerModel] = [ 'live_portrait' ]
+
+expression_restorer_areas : List[ExpressionRestorerArea] = [ 'upper-face', 'lower-face' ]
+
+expression_restorer_factor_range : Sequence[int] = create_int_range(0, 100, 1)

facefusion/processors/modules/expression_restorer/core.py

@@ -1,37 +1,42 @@
 from argparse import ArgumentParser
 from functools import lru_cache
-from typing import List, Tuple
+from typing import Tuple
 
 import cv2
 import numpy
 
+import facefusion.args_store
 import facefusion.jobs.job_manager
-import facefusion.jobs.job_store
-import facefusion.processors.core as processors
-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wording
+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, state_manager, translator, video_manager
 from facefusion.common_helper import create_int_metavar
 from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url
-from facefusion.face_analyser import get_many_faces, get_one_face
+from facefusion.face_analyser import scale_face
 from facefusion.face_helper import paste_back, warp_face_by_face_landmark_5
 from facefusion.face_masker import create_box_mask, create_occlusion_mask
-from facefusion.face_selector import find_similar_faces, sort_and_filter_faces
-from facefusion.face_store import get_reference_faces
-from facefusion.filesystem import in_directory, is_image, is_video, resolve_relative_path, same_file_extension
-from facefusion.processors import choices as processors_choices
+from facefusion.face_selector import select_faces
+from facefusion.filesystem import in_directory, is_image, is_video, resolve_relative_path
 from facefusion.processors.live_portrait import create_rotation, limit_expression
-from facefusion.processors.types import ExpressionRestorerInputs, LivePortraitExpression, LivePortraitFeatureVolume, LivePortraitMotionPoints, LivePortraitPitch, LivePortraitRoll, LivePortraitScale, LivePortraitTranslation, LivePortraitYaw
+from facefusion.processors.modules.expression_restorer import choices as expression_restorer_choices
+from facefusion.processors.modules.expression_restorer.types import ExpressionRestorerInputs
+from facefusion.processors.types import LivePortraitExpression, LivePortraitFeatureVolume, LivePortraitMotionPoints, LivePortraitPitch, LivePortraitRoll, LivePortraitScale, LivePortraitTranslation, LivePortraitYaw, ProcessorOutputs
 from facefusion.program_helper import find_argument_group
 from facefusion.thread_helper import conditional_thread_semaphore, thread_semaphore
-from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, ModelOptions, ModelSet, ProcessMode, QueuePayload, UpdateProgress, VisionFrame
-from facefusion.vision import read_image, read_static_image, read_video_frame, write_image
+from facefusion.types import ApplyStateItem, Args, DownloadScope, Face, InferencePool, ModelOptions, ModelSet, ProcessMode, VisionFrame
+from facefusion.vision import read_static_image, read_static_video_frame
 
 
-@lru_cache(maxsize = None)
+@lru_cache()
 def create_static_model_set(download_scope : DownloadScope) -> ModelSet:
 	return\
 	{
 		'live_portrait':
 		{
+			'__metadata__':
+			{
+				'vendor': 'KwaiVGI',
+				'license': 'MIT',
+				'year': 2024
+			},
 			'hashes':
 			{
 				'feature_extractor':
@@ -94,14 +99,16 @@ def get_model_options() -> ModelOptions:
 def register_args(program : ArgumentParser) -> None:
 	group_processors = find_argument_group(program, 'processors')
 	if group_processors:
-		group_processors.add_argument('--expression-restorer-model', help = wording.get('help.expression_restorer_model'), default = config.get_str_value('processors', 'expression_restorer_model', 'live_portrait'), choices = processors_choices.expression_restorer_models)
-		group_processors.add_argument('--expression-restorer-factor', help = wording.get('help.expression_restorer_factor'), type = int, default = config.get_int_value('processors', 'expression_restorer_factor', '80'), choices = processors_choices.expression_restorer_factor_range, metavar = create_int_metavar(processors_choices.expression_restorer_factor_range))
-		facefusion.jobs.job_store.register_step_keys([ 'expression_restorer_model', 'expression_restorer_factor' ])
+		group_processors.add_argument('--expression-restorer-model', help = translator.get('help.model', __package__), default = config.get_str_value('processors', 'expression_restorer_model', 'live_portrait'), choices = expression_restorer_choices.expression_restorer_models)
+		group_processors.add_argument('--expression-restorer-factor', help = translator.get('help.factor', __package__), type = int, default = config.get_int_value('processors', 'expression_restorer_factor', '80'), choices = expression_restorer_choices.expression_restorer_factor_range, metavar = create_int_metavar(expression_restorer_choices.expression_restorer_factor_range))
+		group_processors.add_argument('--expression-restorer-areas', help = translator.get('help.areas', __package__).format(choices = ', '.join(expression_restorer_choices.expression_restorer_areas)), default = config.get_str_list('processors', 'expression_restorer_areas', ' '.join(expression_restorer_choices.expression_restorer_areas)), choices = expression_restorer_choices.expression_restorer_areas, nargs ='+', metavar ='EXPRESSION_RESTORER_AREAS')
+		facefusion.args_store.register_args([ 'expression_restorer_model', 'expression_restorer_factor', 'expression_restorer_areas' ], scopes = [ 'api', 'cli' ])
 
 
 def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
 	apply_state_item('expression_restorer_model', args.get('expression_restorer_model'))
 	apply_state_item('expression_restorer_factor', args.get('expression_restorer_factor'))
+	apply_state_item('expression_restorer_areas', args.get('expression_restorer_areas'))
 
 
 def pre_check() -> bool:
@@ -113,22 +120,20 @@ def pre_check() -> bool:
 
 def pre_process(mode : ProcessMode) -> bool:
 	if mode == 'stream':
-		logger.error(wording.get('stream_not_supported') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('stream_not_supported') + translator.get('exclamation_mark'), __name__)
 		return False
 	if mode in [ 'output', 'preview' ] and not is_image(state_manager.get_item('target_path')) and not is_video(state_manager.get_item('target_path')):
-		logger.error(wording.get('choose_image_or_video_target') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('choose_image_or_video_target') + translator.get('exclamation_mark'), __name__)
 		return False
 	if mode == 'output' and not in_directory(state_manager.get_item('output_path')):
-		logger.error(wording.get('specify_image_or_video_output') + wording.get('exclamation_mark'), __name__)
-		return False
-	if mode == 'output' and not same_file_extension(state_manager.get_item('target_path'), state_manager.get_item('output_path')):
-		logger.error(wording.get('match_target_and_output_extension') + wording.get('exclamation_mark'), __name__)
+		logger.error(translator.get('specify_image_or_video_output') + translator.get('exclamation_mark'), __name__)
 		return False
 	return True
 
 
 def post_process() -> None:
 	read_static_image.cache_clear()
+	read_static_video_frame.cache_clear()
 	video_manager.clear_video_pool()
 	if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:
 		clear_inference_pool()
@@ -141,46 +146,58 @@ def post_process() -> None:
 		face_recognizer.clear_inference_pool()
 
 
-def restore_expression(source_vision_frame : VisionFrame, target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
+def restore_expression(target_face : Face, target_vision_frame : VisionFrame, temp_vision_frame : VisionFrame) -> VisionFrame:
 	model_template = get_model_options().get('template')
 	model_size = get_model_options().get('size')
 	expression_restorer_factor = float(numpy.interp(float(state_manager.get_item('expression_restorer_factor')), [ 0, 100 ], [ 0, 1.2 ]))
-	source_vision_frame = cv2.resize(source_vision_frame, temp_vision_frame.shape[:2][::-1])
-	source_crop_vision_frame, _ = warp_face_by_face_landmark_5(source_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
-	target_crop_vision_frame, affine_matrix = warp_face_by_face_landmark_5(temp_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
-	box_mask = create_box_mask(target_crop_vision_frame, state_manager.get_item('face_mask_blur'), (0, 0, 0, 0))
+	target_crop_vision_frame, _ = warp_face_by_face_landmark_5(target_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
+	temp_crop_vision_frame, affine_matrix = warp_face_by_face_landmark_5(temp_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
+	box_mask = create_box_mask(temp_crop_vision_frame, state_manager.get_item('face_mask_blur'), (0, 0, 0, 0))
 	crop_masks =\
 	[
 		box_mask
 	]
 
 	if 'occlusion' in state_manager.get_item('face_mask_types'):
-		occlusion_mask = create_occlusion_mask(target_crop_vision_frame)
+		occlusion_mask = create_occlusion_mask(temp_crop_vision_frame)
 		crop_masks.append(occlusion_mask)
 
-	source_crop_vision_frame = prepare_crop_frame(source_crop_vision_frame)
 	target_crop_vision_frame = prepare_crop_frame(target_crop_vision_frame)
-	target_crop_vision_frame = apply_restore(source_crop_vision_frame, target_crop_vision_frame, expression_restorer_factor)
-	target_crop_vision_frame = normalize_crop_frame(target_crop_vision_frame)
+	temp_crop_vision_frame = prepare_crop_frame(temp_crop_vision_frame)
+	temp_crop_vision_frame = apply_restore(target_crop_vision_frame, temp_crop_vision_frame, expression_restorer_factor)
+	temp_crop_vision_frame = normalize_crop_frame(temp_crop_vision_frame)
 	crop_mask = numpy.minimum.reduce(crop_masks).clip(0, 1)
-	temp_vision_frame = paste_back(temp_vision_frame, target_crop_vision_frame, crop_mask, affine_matrix)
-	return temp_vision_frame
+	paste_vision_frame = paste_back(temp_vision_frame, temp_crop_vision_frame, crop_mask, affine_matrix)
+	return paste_vision_frame
 
 
-def apply_restore(source_crop_vision_frame : VisionFrame, target_crop_vision_frame : VisionFrame, expression_restorer_factor : float) -> VisionFrame:
-	feature_volume = forward_extract_feature(target_crop_vision_frame)
-	source_expression = forward_extract_motion(source_crop_vision_frame)[5]
-	pitch, yaw, roll, scale, translation, target_expression, motion_points = forward_extract_motion(target_crop_vision_frame)
+def apply_restore(target_crop_vision_frame : VisionFrame, temp_crop_vision_frame : VisionFrame, expression_restorer_factor : float) -> VisionFrame:
+	feature_volume = forward_extract_feature(temp_crop_vision_frame)
+	target_expression = forward_extract_motion(target_crop_vision_frame)[5]
+	pitch, yaw, roll, scale, translation, temp_expression, motion_points = forward_extract_motion(temp_crop_vision_frame)
 	rotation = create_rotation(pitch, yaw, roll)
-	source_expression[:, [ 0, 4, 5, 8, 9 ]] = target_expression[:, [ 0, 4, 5, 8, 9 ]]
-	source_expression = source_expression * expression_restorer_factor + target_expression * (1 - expression_restorer_factor)
-	source_expression = limit_expression(source_expression)
-	source_motion_points = scale * (motion_points @ rotation.T + source_expression) + translation
+	target_expression = restrict_expression_areas(temp_expression, target_expression)
+	target_expression = target_expression * expression_restorer_factor + temp_expression * (1 - expression_restorer_factor)
+	target_expression = limit_expression(target_expression)
 	target_motion_points = scale * (motion_points @ rotation.T + target_expression) + translation
-	crop_vision_frame = forward_generate_frame(feature_volume, source_motion_points, target_motion_points)
+	temp_motion_points = scale * (motion_points @ rotation.T + temp_expression) + translation
+	crop_vision_frame = forward_generate_frame(feature_volume, target_motion_points, temp_motion_points)
 	return crop_vision_frame
 
 
+def restrict_expression_areas(temp_expression : LivePortraitExpression, target_expression : LivePortraitExpression) -> LivePortraitExpression:
+	expression_restorer_areas = state_manager.get_item('expression_restorer_areas')
+
+	if 'upper-face' not in expression_restorer_areas:
+		target_expression[:, [1, 2, 6, 10, 11, 12, 13, 15, 16]] = temp_expression[:, [1, 2, 6, 10, 11, 12, 13, 15, 16]]
+
+	if 'lower-face' not in expression_restorer_areas:
+		target_expression[:, [3, 7, 14, 17, 18, 19, 20]] = temp_expression[:, [3, 7, 14, 17, 18, 19, 20]]
+
+	target_expression[:, [0, 4, 5, 8, 9]] = temp_expression[:, [0, 4, 5, 8, 9]]
+	return target_expression
+
+
 def forward_extract_feature(crop_vision_frame : VisionFrame) -> LivePortraitFeatureVolume:
 	feature_extractor = get_inference_pool().get('feature_extractor')
 
@@ -205,15 +222,15 @@ def forward_extract_motion(crop_vision_frame : VisionFrame) -> Tuple[LivePortrai
 	return pitch, yaw, roll, scale, translation, expression, motion_points
 
 
-def forward_generate_frame(feature_volume : LivePortraitFeatureVolume, source_motion_points : LivePortraitMotionPoints, target_motion_points : LivePortraitMotionPoints) -> VisionFrame:
+def forward_generate_frame(feature_volume : LivePortraitFeatureVolume, target_motion_points : LivePortraitMotionPoints, temp_motion_points : LivePortraitMotionPoints) -> VisionFrame:
 	generator = get_inference_pool().get('generator')
 
 	with thread_semaphore():
 		crop_vision_frame = generator.run(None,
 		{
 			'feature_volume': feature_volume,
-			'source': source_motion_points,
-			'target': target_motion_points
+			'source': target_motion_points,
+			'target': temp_motion_points
 		})[0][0]
 
 	return crop_vision_frame
@@ -235,64 +252,16 @@ def normalize_crop_frame(crop_vision_frame : VisionFrame) -> VisionFrame:
 	return crop_vision_frame
 
 
-def get_reference_frame(source_face : Face, target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
-	pass
-
-
-def process_frame(inputs : ExpressionRestorerInputs) -> VisionFrame:
-	reference_faces = inputs.get('reference_faces')
-	source_vision_frame = inputs.get('source_vision_frame')
+def process_frame(inputs : ExpressionRestorerInputs) -> ProcessorOutputs:
+	reference_vision_frame = inputs.get('reference_vision_frame')
 	target_vision_frame = inputs.get('target_vision_frame')
-	many_faces = sort_and_filter_faces(get_many_faces([ target_vision_frame ]))
+	temp_vision_frame = inputs.get('temp_vision_frame')
+	temp_vision_mask = inputs.get('temp_vision_mask')
+	target_faces = select_faces(reference_vision_frame, target_vision_frame)
 
-	if state_manager.get_item('face_selector_mode') == 'many':
-		if many_faces:
-			for target_face in many_faces:
-				target_vision_frame = restore_expression(source_vision_frame, target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'one':
-		target_face = get_one_face(many_faces)
-		if target_face:
-			target_vision_frame = restore_expression(source_vision_frame, target_face, target_vision_frame)
-	if state_manager.get_item('face_selector_mode') == 'reference':
-		similar_faces = find_similar_faces(many_faces, reference_faces, state_manager.get_item('reference_face_distance'))
-		if similar_faces:
-			for similar_face in similar_faces:
-				target_vision_frame = restore_expression(source_vision_frame, similar_face, target_vision_frame)
-	return target_vision_frame
+	if target_faces:
+		for target_face in target_faces:
+			target_face = scale_face(target_face, target_vision_frame, temp_vision_frame)
+			temp_vision_frame = restore_expression(target_face, target_vision_frame, temp_vision_frame)
 
-
-def process_frames(source_path : List[str], queue_payloads : List[QueuePayload], update_progress : UpdateProgress) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-
-	for queue_payload in process_manager.manage(queue_payloads):
-		frame_number = queue_payload.get('frame_number')
-		if state_manager.get_item('trim_frame_start'):
-			frame_number += state_manager.get_item('trim_frame_start')
-		source_vision_frame = read_video_frame(state_manager.get_item('target_path'), frame_number)
-		target_vision_path = queue_payload.get('frame_path')
-		target_vision_frame = read_image(target_vision_path)
-		output_vision_frame = process_frame(
-		{
-			'reference_faces': reference_faces,
-			'source_vision_frame': source_vision_frame,
-			'target_vision_frame': target_vision_frame
-		})
-		write_image(target_vision_path, output_vision_frame)
-		update_progress(1)
-
-
-def process_image(source_path : str, target_path : str, output_path : str) -> None:
-	reference_faces = get_reference_faces() if 'reference' in state_manager.get_item('face_selector_mode') else None
-	source_vision_frame = read_static_image(state_manager.get_item('target_path'))
-	target_vision_frame = read_static_image(target_path)
-	output_vision_frame = process_frame(
-	{
-		'reference_faces': reference_faces,
-		'source_vision_frame': source_vision_frame,
-		'target_vision_frame': target_vision_frame
-	})
-	write_image(output_path, output_vision_frame)
-
-
-def process_video(source_paths : List[str], temp_frame_paths : List[str]) -> None:
-	processors.multi_process_frames(None, temp_frame_paths, process_frames)
+	return temp_vision_frame, temp_vision_mask