mirror of
https://github.com/facefusion/facefusion.git
synced 2026-07-27 12:30:55 +02:00
New Video Manager (#1191)
* tiny adjustment for tests * address the review on the video manager Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * introduce the stream strategy for the video workflow (#1192) * introduce the stream strategy for the video workflow Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * address the review on the stream strategy Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> * annotate the changes for review Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * annotate the new tests for review Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * match the temp pixel format help to the locale style Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * question the set_input_seek naming Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * question the reader and writer keys Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * drop the review annotations from the encoder mapping tests Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * drop the review annotations from the thread count tests Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * drop the review annotations from the ui files Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * capture the open review questions as annotations Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a * drop the settled annotations from the types Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Tbcd6VWCiU4BQP1gywPr2a --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Henry Ruhs
@@ -55,6 +55,7 @@ voice_extractor_model =
|
||||
trim_frame_start =
|
||||
trim_frame_end =
|
||||
temp_frame_format =
|
||||
temp_pixel_format =
|
||||
|
||||
[frame_distribution]
|
||||
target_frame_amount =
|
||||
|
||||
@@ -47,6 +47,7 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
|
||||
apply_state_item('trim_frame_start', args.get('trim_frame_start'))
|
||||
apply_state_item('trim_frame_end', args.get('trim_frame_end'))
|
||||
apply_state_item('temp_frame_format', args.get('temp_frame_format'))
|
||||
apply_state_item('temp_pixel_format', args.get('temp_pixel_format'))
|
||||
apply_state_item('target_frame_amount', args.get('target_frame_amount'))
|
||||
apply_state_item('output_image_quality', args.get('output_image_quality'))
|
||||
apply_state_item('output_image_scale', args.get('output_image_scale'))
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
from typing import List, Sequence, get_args
|
||||
|
||||
from facefusion.common_helper import create_float_range, create_int_range
|
||||
from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, BenchmarkMode, BenchmarkResolution, BenchmarkSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorGender, FaceSelectorMode, FaceSelectorOrder, FaceSelectorRace, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, UiWorkflow, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, VoiceExtractorModel, WorkflowMode, WorkflowStrategy
|
||||
from facefusion.types import Angle, AudioEncoder, AudioFormat, AudioTypeSet, BenchmarkMode, BenchmarkResolution, BenchmarkSet, DownloadProvider, DownloadProviderSet, DownloadScope, EncoderSet, ExecutionProvider, ExecutionProviderSet, FaceDetectorModel, FaceDetectorSet, FaceLandmarkerModel, FaceMaskArea, FaceMaskAreaSet, FaceMaskRegion, FaceMaskRegionSet, FaceMaskType, FaceOccluderModel, FaceParserModel, FaceSelectorGender, FaceSelectorMode, FaceSelectorOrder, FaceSelectorRace, Gender, ImageFormat, ImageTypeSet, JobStatus, LogLevel, LogLevelSet, Race, Score, TempFrameFormat, TempPixelFormat, UiWorkflow, VideoEncoder, VideoFormat, VideoMemoryStrategy, VideoPreset, VideoTypeSet, VoiceExtractorModel, WorkflowMode, WorkflowStrategy
|
||||
|
||||
face_detector_set : FaceDetectorSet =\
|
||||
{
|
||||
@@ -83,6 +83,8 @@ audio_formats : List[AudioFormat] = list(get_args(AudioFormat))
|
||||
image_formats : List[ImageFormat] = list(get_args(ImageFormat))
|
||||
video_formats : List[VideoFormat] = list(get_args(VideoFormat))
|
||||
temp_frame_formats : List[TempFrameFormat] = list(get_args(TempFrameFormat))
|
||||
#todo: needs review - [config] [critical: low] temp pixel format choices for the stream pipe
|
||||
temp_pixel_formats : List[TempPixelFormat] = list(get_args(TempPixelFormat))
|
||||
|
||||
output_audio_encoders : List[AudioEncoder] = list(get_args(AudioEncoder))
|
||||
output_video_encoders : List[VideoEncoder] = list(get_args(VideoEncoder))
|
||||
|
||||
+61
-12
@@ -7,11 +7,10 @@ from typing import List, Optional, cast
|
||||
from tqdm import tqdm
|
||||
|
||||
import facefusion.choices
|
||||
from facefusion import ffmpeg_builder, ffprobe, logger, process_manager, state_manager, translator
|
||||
from facefusion import ffmpeg_builder, ffprobe, logger, process_manager, state_manager, translator, vision
|
||||
from facefusion.filesystem import get_file_format, remove_file
|
||||
from facefusion.temp_helper import get_temp_file_path, get_temp_frame_pattern
|
||||
from facefusion.types import AudioBuffer, AudioEncoder, Command, EncoderSet, Fps, Resolution, UpdateProgress, VideoEncoder, VideoFormat
|
||||
from facefusion.vision import detect_video_duration, detect_video_fps, pack_resolution, predict_video_frame_total
|
||||
from facefusion.types import AudioBuffer, AudioEncoder, Command, EncoderSet, Fps, Resolution, UpdateProgress, VideoEncoder, VideoFormat, VideoMetadata
|
||||
|
||||
|
||||
def run_ffmpeg_with_progress(commands : List[Command], update_progress : UpdateProgress) -> subprocess.Popen[bytes]:
|
||||
@@ -70,6 +69,56 @@ def open_ffmpeg(commands : List[Command]) -> subprocess.Popen[bytes]:
|
||||
return subprocess.Popen(commands, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: medium] raw pipe reader with seek based start, spawn moved from the POC video_manager into ffmpeg
|
||||
#todo: question if this does too much
|
||||
def create_video_reader(video_path : str, frame_position : int, video_metadata : VideoMetadata) -> subprocess.Popen[bytes]:
|
||||
commands = ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_input_seek(frame_position / video_metadata.get('fps')),
|
||||
ffmpeg_builder.set_input(video_path),
|
||||
ffmpeg_builder.set_filter_thread_count(4),
|
||||
ffmpeg_builder.restrict_color_transfer(video_metadata.get('color_transfer')),
|
||||
ffmpeg_builder.prevent_frame_drop(),
|
||||
ffmpeg_builder.enforce_pixel_format('bgr24'),
|
||||
ffmpeg_builder.set_output_format('rawvideo'),
|
||||
ffmpeg_builder.cast_stream()
|
||||
)
|
||||
return open_ffmpeg(commands)
|
||||
|
||||
|
||||
#todo: needs review - [encoding] [critical: high] pipe writer always converts and tags bt709, temp_pixel_format drives the pipe channels, encoder_thread_count hardcoded to 16
|
||||
#todo: question if this does too much
|
||||
def create_video_writer(target_path : str, temp_video_fps : Fps, temp_video_resolution : Resolution, output_video_resolution : Resolution, output_video_fps : Fps) -> subprocess.Popen[bytes]:
|
||||
output_video_encoder = state_manager.get_item('output_video_encoder')
|
||||
output_video_quality = state_manager.get_item('output_video_quality')
|
||||
output_video_preset = state_manager.get_item('output_video_preset')
|
||||
temp_video_path = get_temp_file_path(target_path)
|
||||
temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
|
||||
output_video_encoder = fix_video_encoder(temp_video_format, output_video_encoder)
|
||||
encoder_thread_count = 16
|
||||
|
||||
commands = ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_output_format('rawvideo'),
|
||||
ffmpeg_builder.enforce_pixel_format(state_manager.get_item('temp_pixel_format')),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(temp_video_resolution)),
|
||||
ffmpeg_builder.set_input_fps(temp_video_fps),
|
||||
ffmpeg_builder.set_input('pipe:0'),
|
||||
ffmpeg_builder.set_filter_thread_count(4),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(output_video_resolution)),
|
||||
ffmpeg_builder.set_video_encoder(output_video_encoder),
|
||||
ffmpeg_builder.set_global_thread_count(encoder_thread_count),
|
||||
ffmpeg_builder.set_video_tag(output_video_encoder, temp_video_format),
|
||||
ffmpeg_builder.set_video_quality(output_video_encoder, output_video_quality),
|
||||
ffmpeg_builder.set_video_preset(output_video_encoder, output_video_preset),
|
||||
ffmpeg_builder.concat(
|
||||
ffmpeg_builder.set_video_fps(output_video_fps),
|
||||
ffmpeg_builder.convert_color_space('bt709')
|
||||
),
|
||||
ffmpeg_builder.set_pixel_format(output_video_encoder),
|
||||
ffmpeg_builder.force_output(temp_video_path)
|
||||
)
|
||||
return open_ffmpeg(commands)
|
||||
|
||||
|
||||
def log_debug(process : subprocess.Popen[bytes]) -> None:
|
||||
_, stderr = process.communicate()
|
||||
errors = stderr.decode().split(os.linesep)
|
||||
@@ -109,11 +158,11 @@ def get_available_encoder_set() -> EncoderSet:
|
||||
|
||||
def extract_frames(target_path : str, temp_video_resolution : Resolution, temp_video_fps : Fps, trim_frame_start : int, trim_frame_end : int) -> bool:
|
||||
color_transfer = ffprobe.extract_static_video_metadata(target_path).get('color_transfer')
|
||||
extract_frame_total = predict_video_frame_total(target_path, temp_video_fps, trim_frame_start, trim_frame_end)
|
||||
extract_frame_total = vision.predict_video_frame_total(target_path, temp_video_fps, trim_frame_start, trim_frame_end)
|
||||
temp_frame_pattern = get_temp_frame_pattern(target_path, '%08d')
|
||||
commands = ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_input(target_path),
|
||||
ffmpeg_builder.set_media_resolution(pack_resolution(temp_video_resolution)),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(temp_video_resolution)),
|
||||
ffmpeg_builder.set_frame_quality(0),
|
||||
ffmpeg_builder.enforce_pixel_format('rgb24'),
|
||||
ffmpeg_builder.concat(
|
||||
@@ -134,7 +183,7 @@ def copy_image(target_path : str, temp_image_resolution : Resolution) -> bool:
|
||||
temp_image_path = get_temp_file_path(target_path)
|
||||
commands = ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_input(target_path),
|
||||
ffmpeg_builder.set_media_resolution(pack_resolution(temp_image_resolution)),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(temp_image_resolution)),
|
||||
ffmpeg_builder.set_image_quality(target_path, 100),
|
||||
ffmpeg_builder.force_output(temp_image_path)
|
||||
)
|
||||
@@ -146,7 +195,7 @@ def finalize_image(target_path : str, output_path : str, output_image_resolution
|
||||
temp_image_path = get_temp_file_path(target_path)
|
||||
commands = ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_input(temp_image_path),
|
||||
ffmpeg_builder.set_media_resolution(pack_resolution(output_image_resolution)),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(output_image_resolution)),
|
||||
ffmpeg_builder.set_image_quality(target_path, output_image_quality),
|
||||
ffmpeg_builder.force_output(output_path)
|
||||
)
|
||||
@@ -174,10 +223,10 @@ def restore_audio(target_path : str, output_path : str, trim_frame_start : int,
|
||||
output_audio_encoder = state_manager.get_item('output_audio_encoder')
|
||||
output_audio_quality = state_manager.get_item('output_audio_quality')
|
||||
output_audio_volume = state_manager.get_item('output_audio_volume')
|
||||
target_video_fps = detect_video_fps(target_path)
|
||||
target_video_fps = vision.detect_video_fps(target_path)
|
||||
temp_video_path = get_temp_file_path(target_path)
|
||||
temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
|
||||
temp_video_duration = detect_video_duration(temp_video_path)
|
||||
temp_video_duration = vision.detect_video_duration(temp_video_path)
|
||||
output_video_format = cast(VideoFormat, get_file_format(output_path))
|
||||
|
||||
output_audio_encoder = fix_audio_encoder(temp_video_format, output_audio_encoder)
|
||||
@@ -204,7 +253,7 @@ def replace_audio(target_path : str, audio_path : str, output_path : str) -> boo
|
||||
output_audio_volume = state_manager.get_item('output_audio_volume')
|
||||
temp_video_path = get_temp_file_path(target_path)
|
||||
temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
|
||||
temp_video_duration = detect_video_duration(temp_video_path)
|
||||
temp_video_duration = vision.detect_video_duration(temp_video_path)
|
||||
output_video_format = cast(VideoFormat, get_file_format(output_path))
|
||||
|
||||
output_audio_encoder = fix_audio_encoder(temp_video_format, output_audio_encoder)
|
||||
@@ -226,7 +275,7 @@ def merge_video(target_path : str, temp_video_fps : Fps, output_video_resolution
|
||||
output_video_encoder = state_manager.get_item('output_video_encoder')
|
||||
output_video_quality = state_manager.get_item('output_video_quality')
|
||||
output_video_preset = state_manager.get_item('output_video_preset')
|
||||
merge_frame_total = predict_video_frame_total(target_path, output_video_fps, trim_frame_start, trim_frame_end)
|
||||
merge_frame_total = vision.predict_video_frame_total(target_path, output_video_fps, trim_frame_start, trim_frame_end)
|
||||
temp_video_path = get_temp_file_path(target_path)
|
||||
temp_video_format = cast(VideoFormat, get_file_format(temp_video_path))
|
||||
temp_frame_pattern = get_temp_frame_pattern(target_path, '%08d')
|
||||
@@ -236,7 +285,7 @@ def merge_video(target_path : str, temp_video_fps : Fps, output_video_resolution
|
||||
ffmpeg_builder.set_input_fps(temp_video_fps),
|
||||
ffmpeg_builder.set_start_number(trim_frame_start),
|
||||
ffmpeg_builder.set_input(temp_frame_pattern),
|
||||
ffmpeg_builder.set_media_resolution(pack_resolution(output_video_resolution)),
|
||||
ffmpeg_builder.set_media_resolution(vision.pack_resolution(output_video_resolution)),
|
||||
ffmpeg_builder.set_video_encoder(output_video_encoder),
|
||||
ffmpeg_builder.set_video_tag(output_video_encoder, temp_video_format),
|
||||
ffmpeg_builder.set_video_quality(output_video_encoder, output_video_quality),
|
||||
|
||||
@@ -83,6 +83,18 @@ def unsafe_concat() -> List[Command]:
|
||||
return [ '-f', 'concat', '-safe', '0' ]
|
||||
|
||||
|
||||
#todo: needs review - [seeking] [critical: low] seek based start for the pipe reader
|
||||
#todo: question to rename set_input_seek to just set_input
|
||||
def set_input_seek(seek_time : float) -> List[Command]:
|
||||
return [ '-ss', str(seek_time) ]
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: low] raw output format for pipe io
|
||||
#todo: question if set_output_format is needed
|
||||
def set_output_format(output_format : str) -> List[Command]:
|
||||
return [ '-f', output_format ]
|
||||
|
||||
|
||||
def enforce_pixel_format(pixel_format : str) -> List[Command]:
|
||||
return [ '-pix_fmt', pixel_format ]
|
||||
|
||||
@@ -193,6 +205,16 @@ def set_audio_volume(audio_volume : int) -> List[Command]:
|
||||
return [ '-filter:a', 'volume=' + str(audio_volume / 100) ]
|
||||
|
||||
|
||||
#todo: needs review - [encoding] [critical: low] explicit ffmpeg thread cap
|
||||
def set_global_thread_count(thread_count : int) -> List[Command]:
|
||||
return [ '-threads', str(thread_count) ]
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: low] threads the filter graph, new against the POC
|
||||
def set_filter_thread_count(thread_count : int) -> List[Command]:
|
||||
return [ '-filter_threads', str(thread_count) ]
|
||||
|
||||
|
||||
def set_video_encoder(video_encoder : str) -> List[Command]:
|
||||
return [ '-c:v', video_encoder ]
|
||||
|
||||
|
||||
@@ -137,6 +137,7 @@ LOCALES : Locales =\
|
||||
'trim_frame_start': 'specify the starting frame of the target video',
|
||||
'trim_frame_end': 'specify the ending frame of the target video',
|
||||
'temp_frame_format': 'specify the temporary resources format',
|
||||
'temp_pixel_format': 'specify the temporary pixel format',
|
||||
'target_frame_amount': 'specify the amount of target frames forwarded to the processor',
|
||||
'output_image_quality': 'specify the image quality which translates to the image compression',
|
||||
'output_image_scale': 'specify the image scale based on the target image',
|
||||
@@ -203,6 +204,8 @@ LOCALES : Locales =\
|
||||
'clear_button': 'CLEAR',
|
||||
'download_providers_checkbox_group': 'DOWNLOAD PROVIDERS',
|
||||
'execution_providers_checkbox_group': 'EXECUTION PROVIDERS',
|
||||
'workflow_mode_dropdown': 'WORKFLOW MODE',
|
||||
'workflow_strategy_dropdown': 'WORKFLOW STRATEGY',
|
||||
'execution_thread_count_slider': 'EXECUTION THREAD COUNT',
|
||||
'face_detector_angles_checkbox_group': 'FACE DETECTOR ANGLES',
|
||||
'face_detector_model_dropdown': 'FACE DETECTOR MODEL',
|
||||
|
||||
@@ -178,7 +178,9 @@ def create_frame_extraction_program() -> ArgumentParser:
|
||||
group_frame_extraction.add_argument('--trim-frame-start', help = translator.get('help.trim_frame_start'), type = int, default = facefusion.config.get_int_value('frame_extraction', 'trim_frame_start'))
|
||||
group_frame_extraction.add_argument('--trim-frame-end', help = translator.get('help.trim_frame_end'), type = int, default = facefusion.config.get_int_value('frame_extraction', 'trim_frame_end'))
|
||||
group_frame_extraction.add_argument('--temp-frame-format', help = translator.get('help.temp_frame_format'), default = config.get_str_value('frame_extraction', 'temp_frame_format', 'png'), choices = facefusion.choices.temp_frame_formats)
|
||||
job_store.register_step_keys([ 'trim_frame_start', 'trim_frame_end', 'temp_frame_format' ])
|
||||
#todo: needs review - [streaming] [critical: low] new temp pixel format option, bgra enables alpha through the stream pipe
|
||||
group_frame_extraction.add_argument('--temp-pixel-format', help = translator.get('help.temp_pixel_format'), default = config.get_str_value('frame_extraction', 'temp_pixel_format', 'bgr24'), choices = facefusion.choices.temp_pixel_formats)
|
||||
job_store.register_step_keys([ 'trim_frame_start', 'trim_frame_end', 'temp_frame_format', 'temp_pixel_format' ])
|
||||
return program
|
||||
|
||||
|
||||
|
||||
+29
-8
@@ -1,3 +1,4 @@
|
||||
import subprocess
|
||||
from collections import namedtuple
|
||||
from threading import Lock
|
||||
from typing import Any, Callable, Dict, List, Literal, NotRequired, Optional, Tuple, TypeAlias, TypedDict
|
||||
@@ -64,16 +65,9 @@ Locales : TypeAlias = Dict[Language, Dict[str, Any]]
|
||||
LocalePoolSet : TypeAlias = Dict[str, Locales]
|
||||
|
||||
WorkflowMode = Literal['auto', 'image-to-image', 'image-to-video']
|
||||
WorkflowStrategy = Literal['disk']
|
||||
WorkflowStrategy = Literal['disk', 'stream']
|
||||
|
||||
VideoCaptureSet : TypeAlias = Dict[str, cv2.VideoCapture]
|
||||
VideoWriterSet : TypeAlias = Dict[str, cv2.VideoWriter]
|
||||
CameraCaptureSet : TypeAlias = Dict[str, cv2.VideoCapture]
|
||||
VideoPoolSet = TypedDict('VideoPoolSet',
|
||||
{
|
||||
'capture' : VideoCaptureSet,
|
||||
'writer' : VideoWriterSet
|
||||
})
|
||||
CameraPoolSet = TypedDict('CameraPoolSet',
|
||||
{
|
||||
'capture' : CameraCaptureSet
|
||||
@@ -106,6 +100,7 @@ Fps : TypeAlias = float
|
||||
Duration : TypeAlias = float
|
||||
|
||||
Buffer : TypeAlias = bytes
|
||||
VisionFrameSet : TypeAlias = Dict[int, VisionFrame]
|
||||
Color : TypeAlias = Tuple[int, int, int, int]
|
||||
Padding : TypeAlias = Tuple[int, int, int, int]
|
||||
Margin : TypeAlias = Tuple[int, int, int, int]
|
||||
@@ -128,6 +123,29 @@ VideoMetadata = TypedDict('VideoMetadata',
|
||||
'bit_rate' : BitRate,
|
||||
'color_transfer' : ColorTransfer
|
||||
})
|
||||
#todo: needs review - [types] [critical: low] reader and writer state around ffmpeg processes, metadata embedded whole
|
||||
#todo: question if the body of VideoReader and VideoWriter needs all the keys
|
||||
VideoReader = TypedDict('VideoReader',
|
||||
{
|
||||
'process' : subprocess.Popen[bytes],
|
||||
'file_path' : str,
|
||||
'metadata' : VideoMetadata,
|
||||
'position' : int,
|
||||
'frame_set' : VisionFrameSet
|
||||
})
|
||||
VideoReaderSet : TypeAlias = Dict[str, VideoReader]
|
||||
VideoWriter = TypedDict('VideoWriter',
|
||||
{
|
||||
'process' : subprocess.Popen[bytes],
|
||||
'file_path' : str,
|
||||
'metadata' : VideoMetadata
|
||||
})
|
||||
VideoWriterSet : TypeAlias = Dict[str, VideoWriter]
|
||||
VideoPoolSet = TypedDict('VideoPoolSet',
|
||||
{
|
||||
'reader' : VideoReaderSet,
|
||||
'writer' : VideoWriterSet
|
||||
})
|
||||
|
||||
ProcessState = Literal['checking', 'processing', 'stopping', 'pending']
|
||||
Args : TypeAlias = Dict[str, Any]
|
||||
@@ -169,6 +187,7 @@ AudioFormat = Literal['flac', 'm4a', 'mp3', 'ogg', 'opus', 'wav']
|
||||
ImageFormat = Literal['bmp', 'jpeg', 'png', 'tiff', 'webp']
|
||||
VideoFormat = Literal['avi', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mxf', 'webm', 'wmv']
|
||||
TempFrameFormat = Literal['bmp', 'jpeg', 'png', 'tiff']
|
||||
TempPixelFormat = Literal['bgr24', 'bgra']
|
||||
AudioTypeSet : TypeAlias = Dict[AudioFormat, str]
|
||||
ImageTypeSet : TypeAlias = Dict[ImageFormat, str]
|
||||
VideoTypeSet : TypeAlias = Dict[VideoFormat, str]
|
||||
@@ -341,6 +360,7 @@ StateKey = Literal\
|
||||
'trim_frame_start',
|
||||
'trim_frame_end',
|
||||
'temp_frame_format',
|
||||
'temp_pixel_format',
|
||||
'target_frame_amount',
|
||||
'output_image_quality',
|
||||
'output_image_scale',
|
||||
@@ -413,6 +433,7 @@ State = TypedDict('State',
|
||||
'trim_frame_start' : int,
|
||||
'trim_frame_end' : int,
|
||||
'temp_frame_format' : TempFrameFormat,
|
||||
'temp_pixel_format' : TempPixelFormat,
|
||||
'target_frame_amount' : int,
|
||||
'output_image_quality' : int,
|
||||
'output_image_scale' : Scale,
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
from typing import Optional
|
||||
|
||||
import gradio
|
||||
|
||||
import facefusion.choices
|
||||
from facefusion import state_manager, translator
|
||||
from facefusion.types import WorkflowMode, WorkflowStrategy
|
||||
|
||||
WORKFLOW_MODE_DROPDOWN : Optional[gradio.Dropdown] = None
|
||||
WORKFLOW_STRATEGY_DROPDOWN : Optional[gradio.Dropdown] = None
|
||||
|
||||
|
||||
def render() -> None:
|
||||
global WORKFLOW_MODE_DROPDOWN
|
||||
global WORKFLOW_STRATEGY_DROPDOWN
|
||||
|
||||
WORKFLOW_MODE_DROPDOWN = gradio.Dropdown(
|
||||
label = translator.get('uis.workflow_mode_dropdown'),
|
||||
choices = facefusion.choices.workflow_modes,
|
||||
value = state_manager.get_item('workflow_mode')
|
||||
)
|
||||
WORKFLOW_STRATEGY_DROPDOWN = gradio.Dropdown(
|
||||
label = translator.get('uis.workflow_strategy_dropdown'),
|
||||
choices = facefusion.choices.workflow_strategies,
|
||||
value = state_manager.get_item('workflow_strategy')
|
||||
)
|
||||
|
||||
|
||||
def listen() -> None:
|
||||
WORKFLOW_MODE_DROPDOWN.change(update_workflow_mode, inputs = WORKFLOW_MODE_DROPDOWN)
|
||||
WORKFLOW_STRATEGY_DROPDOWN.change(update_workflow_strategy, inputs = WORKFLOW_STRATEGY_DROPDOWN)
|
||||
|
||||
|
||||
def update_workflow_mode(workflow_mode : WorkflowMode) -> None:
|
||||
state_manager.set_item('workflow_mode', workflow_mode)
|
||||
|
||||
|
||||
def update_workflow_strategy(workflow_strategy : WorkflowStrategy) -> None:
|
||||
state_manager.set_item('workflow_strategy', workflow_strategy)
|
||||
@@ -1,7 +1,7 @@
|
||||
import gradio
|
||||
|
||||
from facefusion import state_manager
|
||||
from facefusion.uis.components import about, age_modifier_options, background_remover_options, deep_swapper_options, download, execution, execution_thread_count, expression_restorer_options, face_debugger_options, face_detector, face_editor_options, face_enhancer_options, face_landmarker, face_masker, face_selector, face_swapper_options, face_tracker, frame_colorizer_options, frame_enhancer_options, instant_runner, job_manager, job_runner, lip_syncer_options, memory, output, output_options, preview, preview_options, processors, source, target, temp_frame, terminal, trim_frame, ui_workflow, voice_extractor
|
||||
from facefusion.uis.components import about, age_modifier_options, background_remover_options, deep_swapper_options, download, execution, execution_thread_count, expression_restorer_options, face_debugger_options, face_detector, face_editor_options, face_enhancer_options, face_landmarker, face_masker, face_selector, face_swapper_options, face_tracker, frame_colorizer_options, frame_enhancer_options, instant_runner, job_manager, job_runner, lip_syncer_options, memory, output, output_options, preview, preview_options, processors, source, target, temp_frame, terminal, trim_frame, ui_workflow, voice_extractor, workflow
|
||||
|
||||
|
||||
def pre_check() -> bool:
|
||||
@@ -40,6 +40,8 @@ def render() -> gradio.Blocks:
|
||||
lip_syncer_options.render()
|
||||
with gradio.Blocks():
|
||||
voice_extractor.render()
|
||||
with gradio.Blocks():
|
||||
workflow.render()
|
||||
with gradio.Blocks():
|
||||
execution.render()
|
||||
execution_thread_count.render()
|
||||
@@ -119,6 +121,7 @@ def listen() -> None:
|
||||
face_detector.listen()
|
||||
face_landmarker.listen()
|
||||
voice_extractor.listen()
|
||||
workflow.listen()
|
||||
|
||||
|
||||
def run(ui : gradio.Blocks) -> None:
|
||||
|
||||
+102
-25
@@ -1,46 +1,123 @@
|
||||
import cv2
|
||||
from typing import Optional
|
||||
|
||||
from facefusion.types import VideoPoolSet
|
||||
import numpy
|
||||
|
||||
from facefusion import ffmpeg, ffprobe
|
||||
from facefusion.types import Fps, Resolution, VideoMetadata, VideoPoolSet, VideoReader, VideoWriter, VisionFrame, VisionFrameSet
|
||||
|
||||
VIDEO_POOL_SET : VideoPoolSet =\
|
||||
{
|
||||
'capture': {},
|
||||
'reader': {},
|
||||
'writer': {}
|
||||
}
|
||||
|
||||
|
||||
def get_video_capture(video_path : str) -> cv2.VideoCapture:
|
||||
if video_path not in VIDEO_POOL_SET.get('capture'):
|
||||
video_capture = cv2.VideoCapture(video_path)
|
||||
#todo: needs review - [lifecycle] [critical: medium] pooled reader per path is created without lock and only reaped via clear_video_pool
|
||||
def get_reader(video_path : str) -> VideoReader:
|
||||
if video_path not in VIDEO_POOL_SET.get('reader'):
|
||||
video_metadata = ffprobe.extract_static_video_metadata(video_path)
|
||||
|
||||
if video_capture.isOpened():
|
||||
VIDEO_POOL_SET['capture'][video_path] = video_capture
|
||||
VIDEO_POOL_SET['reader'][video_path] =\
|
||||
{
|
||||
'process': ffmpeg.create_video_reader(video_path, 0, video_metadata),
|
||||
'file_path': video_path,
|
||||
'metadata': video_metadata,
|
||||
'position': 0,
|
||||
'frame_set': {}
|
||||
}
|
||||
|
||||
return VIDEO_POOL_SET.get('capture').get(video_path)
|
||||
return VIDEO_POOL_SET.get('reader').get(video_path)
|
||||
|
||||
|
||||
def conditional_set_video_frame_position(video_capture : cv2.VideoCapture, frame_position : int) -> bool:
|
||||
if not video_capture.get(cv2.CAP_PROP_POS_FRAMES) == frame_position:
|
||||
return video_capture.set(cv2.CAP_PROP_POS_FRAMES, frame_position)
|
||||
return True
|
||||
#todo: needs review - [seeking] [critical: high] forward skip up to 128 frames by draining the pipe, everything else refreshes the process
|
||||
def conditional_set_video_reader_position(video_reader : VideoReader, frame_position : int) -> None:
|
||||
skip_margin = 128
|
||||
skip_total = frame_position - video_reader.get('position')
|
||||
|
||||
if skip_total > 0 and skip_total <= skip_margin:
|
||||
for _ in range(skip_total):
|
||||
read_video_reader_frame(video_reader)
|
||||
|
||||
if not video_reader.get('position') == frame_position:
|
||||
refresh_video_reader(video_reader, frame_position)
|
||||
|
||||
|
||||
def get_video_writer(video_path : str) -> cv2.VideoWriter:
|
||||
if video_path not in VIDEO_POOL_SET.get('writer'):
|
||||
video_writer = cv2.VideoWriter()
|
||||
|
||||
if video_writer.isOpened():
|
||||
VIDEO_POOL_SET['writer'][video_path] = video_writer
|
||||
|
||||
return VIDEO_POOL_SET.get('writer').get(video_path)
|
||||
#todo: needs review - [seeking] [critical: high] kill and respawn per out of order seek, frequent refreshes stack up on random access
|
||||
def refresh_video_reader(video_reader : VideoReader, frame_position : int) -> None:
|
||||
video_reader.get('process').kill()
|
||||
video_reader.get('process').wait()
|
||||
video_reader['process'] = ffmpeg.create_video_reader(video_reader.get('file_path'), frame_position, video_reader.get('metadata'))
|
||||
video_reader['position'] = frame_position
|
||||
video_reader['frame_set'].clear()
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: high] partial pipe read returns none and desyncs position from the actual stream
|
||||
def read_video_reader_frame(video_reader : VideoReader) -> Optional[VisionFrame]:
|
||||
width, height = video_reader.get('metadata').get('resolution')
|
||||
frame_size = width * height * 3
|
||||
frame_buffer = video_reader.get('process').stdout.read(frame_size)
|
||||
|
||||
if len(frame_buffer) == frame_size:
|
||||
video_reader['position'] = video_reader.get('position') + 1
|
||||
return numpy.frombuffer(frame_buffer, numpy.uint8).reshape(height, width, 3)
|
||||
return None
|
||||
|
||||
|
||||
#todo: needs review - [memory] [critical: high] frame_set keeps decoded frames in ram, eviction only trims below frame_start minus buffer_margin
|
||||
def read_video_reader_window(video_reader : VideoReader, frame_start : int, frame_end : int) -> VisionFrameSet:
|
||||
frame_set = video_reader.get('frame_set')
|
||||
buffer_margin = 16
|
||||
|
||||
if frame_start not in frame_set and (frame_start < video_reader.get('position') or frame_start > video_reader.get('position') + buffer_margin):
|
||||
refresh_video_reader(video_reader, frame_start)
|
||||
|
||||
for frame_number in range(video_reader.get('position'), frame_end + 1):
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
|
||||
if numpy.any(vision_frame):
|
||||
frame_set[frame_number] = vision_frame
|
||||
|
||||
for frame_number in list(frame_set):
|
||||
if frame_number < frame_start - buffer_margin:
|
||||
del frame_set[frame_number]
|
||||
|
||||
return frame_set
|
||||
|
||||
|
||||
#todo: needs review - [lifecycle] [critical: low] pooled writer keyed by target_path, metadata forwarded by the caller
|
||||
def get_writer(target_path : str, video_metadata : VideoMetadata, temp_video_fps : Fps, temp_video_resolution : Resolution, output_video_resolution : Resolution, output_video_fps : Fps) -> VideoWriter:
|
||||
if target_path not in VIDEO_POOL_SET.get('writer'):
|
||||
VIDEO_POOL_SET['writer'][target_path] =\
|
||||
{
|
||||
'process': ffmpeg.create_video_writer(target_path, temp_video_fps, temp_video_resolution, output_video_resolution, output_video_fps),
|
||||
'file_path': target_path,
|
||||
'metadata': video_metadata
|
||||
}
|
||||
|
||||
return VIDEO_POOL_SET.get('writer').get(target_path)
|
||||
|
||||
|
||||
#todo: needs review - [encoding] [critical: medium] blocking stdin write without backpressure or broken pipe handling
|
||||
def write_video_writer_frame(video_writer : VideoWriter, vision_frame : VisionFrame) -> None:
|
||||
video_writer.get('process').stdin.write(vision_frame.tobytes())
|
||||
|
||||
|
||||
#todo: needs review - [encoding] [critical: medium] encoder failures only surface via returncode at close time
|
||||
def close_video_writer(video_writer : VideoWriter) -> bool:
|
||||
video_writer.get('process').stdin.close()
|
||||
video_writer.get('process').wait()
|
||||
return video_writer.get('process').returncode == 0
|
||||
|
||||
|
||||
#todo: needs review - [lifecycle] [critical: high] kill over terminate, sigterm deadlocks while the pipe is full
|
||||
def clear_video_pool() -> None:
|
||||
for video_capture in VIDEO_POOL_SET.get('capture').values():
|
||||
video_capture.release()
|
||||
for video_reader in VIDEO_POOL_SET.get('reader').values():
|
||||
video_reader.get('process').kill()
|
||||
video_reader.get('process').wait()
|
||||
|
||||
for video_writer in VIDEO_POOL_SET.get('writer').values():
|
||||
video_writer.release()
|
||||
video_writer.get('process').kill()
|
||||
video_writer.get('process').wait()
|
||||
|
||||
VIDEO_POOL_SET['capture'].clear()
|
||||
VIDEO_POOL_SET['reader'].clear()
|
||||
VIDEO_POOL_SET['writer'].clear()
|
||||
|
||||
+32
-27
@@ -1,17 +1,16 @@
|
||||
import math
|
||||
from functools import lru_cache
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import cv2
|
||||
import numpy
|
||||
from cv2.typing import Size
|
||||
|
||||
from facefusion import ffprobe
|
||||
from facefusion import ffprobe, video_manager
|
||||
from facefusion.common_helper import is_windows
|
||||
from facefusion.filesystem import get_file_extension, is_image, is_video
|
||||
from facefusion.thread_helper import thread_lock, thread_semaphore
|
||||
from facefusion.types import ColorMode, Duration, Fps, Mask, Orientation, Resolution, Scale, VisionFrame
|
||||
from facefusion.video_manager import conditional_set_video_frame_position, get_video_capture
|
||||
from facefusion.types import ColorMode, Duration, Fps, Mask, Orientation, Resolution, Scale, VisionFrame, VisionFrameSet
|
||||
|
||||
|
||||
def read_static_images(image_paths : List[str], color_mode : ColorMode = 'rgb') -> List[VisionFrame]:
|
||||
@@ -76,63 +75,69 @@ def read_static_video_frame(video_path : str, frame_number : int = 0) -> Optiona
|
||||
return read_video_frame(video_path, frame_number)
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: medium] cv2 capture replaced by the pooled ffmpeg reader, position clamped to the frame_total estimate
|
||||
def read_video_frame(video_path : str, frame_number : int = 0) -> Optional[VisionFrame]:
|
||||
if is_video(video_path):
|
||||
video_capture = get_video_capture(video_path)
|
||||
video_reader = video_manager.get_reader(video_path)
|
||||
|
||||
if video_capture and video_capture.isOpened():
|
||||
video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
video_frame_position = min(video_frame_total, frame_number)
|
||||
if video_reader:
|
||||
video_frame_position = min(video_reader.get('metadata').get('frame_total'), frame_number)
|
||||
|
||||
with thread_semaphore():
|
||||
conditional_set_video_frame_position(video_capture, video_frame_position)
|
||||
has_vision_frame, vision_frame = video_capture.read()
|
||||
|
||||
if has_vision_frame:
|
||||
return vision_frame
|
||||
video_manager.conditional_set_video_reader_position(video_reader, video_frame_position)
|
||||
return video_manager.read_video_reader_frame(video_reader)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize = 2)
|
||||
def read_static_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> Dict[int, VisionFrame]:
|
||||
def read_static_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> VisionFrameSet:
|
||||
return read_video_chunk(video_path, chunk_number, chunk_size)
|
||||
|
||||
|
||||
def read_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> Dict[int, VisionFrame]:
|
||||
#todo: needs review - [decoding] [critical: medium] chunk end clamped by the ffprobe frame_total estimate
|
||||
def read_video_chunk(video_path : str, chunk_number : int, chunk_size : int) -> VisionFrameSet:
|
||||
video_frame_chunk = {}
|
||||
|
||||
if is_video(video_path) and chunk_number > -1:
|
||||
video_capture = get_video_capture(video_path)
|
||||
video_reader = video_manager.get_reader(video_path)
|
||||
|
||||
if video_capture and video_capture.isOpened():
|
||||
video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
if video_reader:
|
||||
video_frame_position = chunk_number * chunk_size
|
||||
video_frame_end = video_frame_position + chunk_size
|
||||
|
||||
if video_reader.get('metadata').get('frame_total') > 0:
|
||||
video_frame_end = min(video_frame_end, video_reader.get('metadata').get('frame_total'))
|
||||
|
||||
with thread_semaphore():
|
||||
conditional_set_video_frame_position(video_capture, video_frame_position)
|
||||
video_manager.conditional_set_video_reader_position(video_reader, video_frame_position)
|
||||
|
||||
for frame_number in range(video_frame_position, min(video_frame_position + chunk_size, video_frame_total)):
|
||||
has_vision_frame, vision_frame = video_capture.read()
|
||||
for frame_number in range(video_frame_position, video_frame_end):
|
||||
vision_frame = video_manager.read_video_reader_frame(video_reader)
|
||||
|
||||
if has_vision_frame:
|
||||
if numpy.any(vision_frame):
|
||||
video_frame_chunk[frame_number] = vision_frame
|
||||
|
||||
return video_frame_chunk
|
||||
|
||||
|
||||
#todo: needs review - [decoding] [critical: high] window read replaces the chunk cache, out of range frames fall back to empty vision frames
|
||||
#todo: question to restore the chunk_size approach over the window read
|
||||
def select_video_frames(video_path : str, frame_number : int = 0, frame_offset : int = 2) -> List[VisionFrame]:
|
||||
vision_frames = []
|
||||
chunk_size = frame_offset * 2 + 1
|
||||
frame_start = frame_number - frame_offset
|
||||
frame_end = frame_number + frame_offset
|
||||
|
||||
if is_video(video_path):
|
||||
with thread_lock():
|
||||
for current_number in range(frame_number - frame_offset, frame_number + frame_offset + 1):
|
||||
video_frame_chunk = read_static_video_chunk(video_path, current_number // chunk_size, chunk_size)
|
||||
video_reader = video_manager.get_reader(video_path)
|
||||
frame_set = video_manager.read_video_reader_window(video_reader, max(frame_start, 0), frame_end)
|
||||
|
||||
for frame_number in range(frame_start, frame_end + 1):
|
||||
vision_frame = create_empty_vision_frame()
|
||||
|
||||
if current_number in video_frame_chunk:
|
||||
vision_frame = video_frame_chunk.get(current_number)
|
||||
if frame_number in frame_set:
|
||||
vision_frame = frame_set.get(frame_number)
|
||||
|
||||
vision_frames.append(vision_frame)
|
||||
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
|
||||
from functools import partial
|
||||
from typing import List, Tuple
|
||||
|
||||
import cv2
|
||||
import numpy
|
||||
from tqdm import tqdm
|
||||
|
||||
from facefusion import content_analyser, ffmpeg, logger, process_manager, state_manager, translator, video_manager
|
||||
from facefusion import content_analyser, ffmpeg, ffprobe, logger, process_manager, state_manager, translator, video_manager
|
||||
from facefusion.audio import create_empty_audio_frame, get_audio_frame, get_voice_frame
|
||||
from facefusion.common_helper import get_first
|
||||
from facefusion.common_helper import get_first, get_middle
|
||||
from facefusion.filesystem import filter_audio_paths, is_video
|
||||
from facefusion.processors.core import get_processors_modules
|
||||
from facefusion.temp_helper import move_temp_file, resolve_temp_frame_set
|
||||
from facefusion.time_helper import calculate_end_time
|
||||
from facefusion.types import ErrorCode
|
||||
from facefusion.types import ErrorCode, FrameSet, Resolution, VideoWriter, VisionFrame
|
||||
from facefusion.vision import conditional_merge_vision_mask, detect_video_resolution, extract_vision_mask, pack_resolution, read_static_image, read_static_images, read_static_video_frame, restrict_trim_frame, restrict_video_fps, restrict_video_resolution, scale_resolution, select_video_frames, write_image
|
||||
from facefusion.workflows.core import clear, is_process_stopping, setup
|
||||
|
||||
@@ -32,6 +34,10 @@ def process(start_time : float) -> ErrorCode:
|
||||
merge_frames
|
||||
])
|
||||
|
||||
#todo: needs review - [workflow] [critical: medium] stream appends its single task, disk keeps extract process merge
|
||||
if state_manager.get_item('workflow_strategy') == 'stream':
|
||||
tasks.append(process_stream_frames)
|
||||
|
||||
tasks.extend(
|
||||
[
|
||||
restore_audio,
|
||||
@@ -76,6 +82,7 @@ def extract_frames() -> ErrorCode:
|
||||
return 0
|
||||
|
||||
|
||||
#todo: needs review - [workflow] [critical: medium] warms the reference frame before the executor, neighbors come from temp frames
|
||||
def process_frames() -> ErrorCode:
|
||||
temp_frame_set = resolve_temp_frame_set(state_manager.get_item('target_path'))
|
||||
|
||||
@@ -83,11 +90,13 @@ def process_frames() -> ErrorCode:
|
||||
with tqdm(total = len(temp_frame_set), desc = translator.get('processing'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
|
||||
progress.set_postfix(execution_providers = state_manager.get_item('execution_providers'))
|
||||
|
||||
read_static_video_frame(state_manager.get_item('target_path'), state_manager.get_item('reference_frame_number'))
|
||||
|
||||
with ThreadPoolExecutor(max_workers = state_manager.get_item('execution_thread_count')) as executor:
|
||||
futures = []
|
||||
|
||||
for frame_number, temp_frame_path in temp_frame_set.items():
|
||||
future = executor.submit(process_temp_frame, temp_frame_path, frame_number)
|
||||
future = executor.submit(process_disk_frame, temp_frame_path, frame_number, temp_frame_set)
|
||||
futures.append(future)
|
||||
|
||||
for future in as_completed(futures):
|
||||
@@ -157,14 +166,13 @@ def restore_audio() -> ErrorCode:
|
||||
return 0
|
||||
|
||||
|
||||
def process_temp_frame(temp_frame_path : str, frame_number : int) -> bool:
|
||||
#todo: needs review - [workflow] [critical: medium] shared processing path for disk and stream, receives vision frames instead of paths
|
||||
def process_target_frame(frame_number : int, target_vision_frames : List[VisionFrame], temp_vision_frame : VisionFrame) -> VisionFrame:
|
||||
trim_frame_start, _ = restrict_trim_frame(state_manager.get_item('target_path'), state_manager.get_item('trim_frame_start'), state_manager.get_item('trim_frame_end'))
|
||||
reference_vision_frame = read_static_video_frame(state_manager.get_item('target_path'), state_manager.get_item('reference_frame_number'))
|
||||
source_vision_frames = read_static_images(state_manager.get_item('source_paths'))
|
||||
source_audio_path = get_first(filter_audio_paths(state_manager.get_item('source_paths')))
|
||||
target_vision_frames = select_video_frames(state_manager.get_item('target_path'), frame_number, state_manager.get_item('target_frame_amount'))
|
||||
temp_video_fps = restrict_video_fps(state_manager.get_item('target_path'), state_manager.get_item('output_video_fps'))
|
||||
temp_vision_frame = read_static_image(temp_frame_path, 'rgba')
|
||||
temp_vision_mask = extract_vision_mask(temp_vision_frame)
|
||||
|
||||
source_audio_frame = get_audio_frame(source_audio_path, temp_video_fps, frame_number - trim_frame_start)
|
||||
@@ -187,10 +195,105 @@ def process_temp_frame(temp_frame_path : str, frame_number : int) -> bool:
|
||||
'temp_vision_mask': temp_vision_mask
|
||||
})
|
||||
|
||||
temp_vision_frame = conditional_merge_vision_mask(temp_vision_frame, temp_vision_mask)
|
||||
return conditional_merge_vision_mask(temp_vision_frame, temp_vision_mask)
|
||||
|
||||
|
||||
#todo: needs review - [correctness] [critical: high] missing neighbor frames resolve to none paths and rely on read_static_image returning none
|
||||
def resolve_temp_vision_frames(frame_number : int, frame_amount : int, temp_frame_set : FrameSet) -> List[VisionFrame]:
|
||||
temp_vision_frames = []
|
||||
frame_range = range(frame_number - frame_amount, frame_number + frame_amount + 1)
|
||||
|
||||
for temp_frame_number in frame_range:
|
||||
temp_vision_frames.append(read_static_image(temp_frame_set.get(temp_frame_number)))
|
||||
|
||||
return temp_vision_frames
|
||||
|
||||
|
||||
#todo: needs review - [workflow] [critical: low] disk variant reads temp frame and neighbors from disk and writes back in place
|
||||
def process_disk_frame(temp_frame_path : str, frame_number : int, temp_frame_set : FrameSet) -> bool:
|
||||
target_vision_frames = resolve_temp_vision_frames(frame_number, state_manager.get_item('target_frame_amount'), temp_frame_set)
|
||||
temp_vision_frame = read_static_image(temp_frame_path, 'rgba')
|
||||
temp_vision_frame = process_target_frame(frame_number, target_vision_frames, temp_vision_frame)
|
||||
return write_image(temp_frame_path, temp_vision_frame)
|
||||
|
||||
|
||||
#todo: needs review - [memory] [critical: high] frame look ahead bound by a hardcoded 3gb budget
|
||||
def calculate_frame_look_ahead(temp_video_resolution : Resolution) -> int:
|
||||
width, height = temp_video_resolution
|
||||
frame_memory_budget = 3 * 1024 ** 3
|
||||
frame_memory_usage = width * height * 4 * 6
|
||||
return min(state_manager.get_item('execution_thread_count') * 2, max(2, frame_memory_budget // frame_memory_usage))
|
||||
|
||||
|
||||
#todo: needs review - [streaming] [critical: high] middle frame resized to temp resolution before processing, channels follow temp_pixel_format to match the writer pipe
|
||||
def process_stream_frame(frame_number : int, temp_video_resolution : Resolution) -> Tuple[int, VisionFrame]:
|
||||
target_vision_frames = select_video_frames(state_manager.get_item('target_path'), frame_number, state_manager.get_item('target_frame_amount'))
|
||||
target_vision_frame = get_middle(target_vision_frames)
|
||||
temp_vision_frame = target_vision_frame.copy()
|
||||
|
||||
if not (target_vision_frame.shape[1], target_vision_frame.shape[0]) == temp_video_resolution:
|
||||
temp_vision_frame = cv2.resize(target_vision_frame, temp_video_resolution)
|
||||
temp_vision_frame = process_target_frame(frame_number, target_vision_frames, temp_vision_frame)
|
||||
|
||||
if state_manager.get_item('temp_pixel_format') == 'bgra':
|
||||
temp_vision_frame = cv2.cvtColor(temp_vision_frame, cv2.COLOR_BGR2BGRA)
|
||||
|
||||
if state_manager.get_item('temp_pixel_format') == 'bgr24':
|
||||
temp_vision_frame = temp_vision_frame[:, :, :3]
|
||||
|
||||
return frame_number, numpy.ascontiguousarray(temp_vision_frame)
|
||||
|
||||
|
||||
#todo: needs review - [streaming] [critical: high] ordered submit and drain keeps writes in order, failed close stops the process manager
|
||||
def process_stream_frames() -> ErrorCode:
|
||||
trim_frame_start, trim_frame_end = restrict_trim_frame(state_manager.get_item('target_path'), state_manager.get_item('trim_frame_start'), state_manager.get_item('trim_frame_end'))
|
||||
output_video_resolution = scale_resolution(detect_video_resolution(state_manager.get_item('target_path')), state_manager.get_item('output_video_scale'))
|
||||
temp_video_resolution = restrict_video_resolution(state_manager.get_item('target_path'), output_video_resolution)
|
||||
temp_video_fps = restrict_video_fps(state_manager.get_item('target_path'), state_manager.get_item('output_video_fps'))
|
||||
frame_range = range(trim_frame_start, trim_frame_end)
|
||||
|
||||
if frame_range:
|
||||
video_metadata = ffprobe.extract_static_video_metadata(state_manager.get_item('target_path'))
|
||||
video_writer = video_manager.get_writer(state_manager.get_item('target_path'), video_metadata, temp_video_fps, temp_video_resolution, output_video_resolution, state_manager.get_item('output_video_fps'))
|
||||
frame_look_ahead = calculate_frame_look_ahead(temp_video_resolution)
|
||||
|
||||
with tqdm(total = len(frame_range), desc = translator.get('processing'), unit = 'frame', ascii = ' =', disable = state_manager.get_item('log_level') in [ 'warn', 'error' ]) as progress:
|
||||
progress.set_postfix(execution_providers = state_manager.get_item('execution_providers'))
|
||||
|
||||
read_static_video_frame(state_manager.get_item('target_path'), state_manager.get_item('reference_frame_number'))
|
||||
|
||||
with ThreadPoolExecutor(max_workers = state_manager.get_item('execution_thread_count')) as executor:
|
||||
futures = []
|
||||
|
||||
for frame_number in frame_range:
|
||||
futures.append(executor.submit(process_stream_frame, frame_number, temp_video_resolution))
|
||||
write_stream_frames(video_writer, futures, frame_look_ahead, progress)
|
||||
|
||||
write_stream_frames(video_writer, futures, 1, progress)
|
||||
|
||||
if not video_manager.close_video_writer(video_writer):
|
||||
process_manager.stop()
|
||||
|
||||
for processor_module in get_processors_modules(state_manager.get_item('processors')):
|
||||
processor_module.post_process()
|
||||
|
||||
if is_process_stopping():
|
||||
return 4
|
||||
else:
|
||||
logger.error(translator.get('temp_frames_not_found'), __name__)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
#todo: needs review - [streaming] [critical: medium] drains futures down to the look ahead, stopping skips writes without cancelling
|
||||
def write_stream_frames(video_writer : VideoWriter, futures : List[Future[Tuple[int, VisionFrame]]], frame_look_ahead : int, progress : tqdm) -> None:
|
||||
for _ in range(len(futures) - frame_look_ahead + 1):
|
||||
if not is_process_stopping():
|
||||
_, temp_vision_frame = futures.pop(0).result()
|
||||
video_manager.write_video_writer_frame(video_writer, temp_vision_frame)
|
||||
progress.update()
|
||||
|
||||
|
||||
def finalize_video(start_time : float) -> ErrorCode:
|
||||
if is_video(state_manager.get_item('output_path')):
|
||||
logger.info(translator.get('processing_video_succeeded').format(seconds = calculate_end_time(start_time)), __name__)
|
||||
|
||||
+27
-1
@@ -6,7 +6,7 @@ import pytest
|
||||
import facefusion.ffmpeg
|
||||
from facefusion import ffmpeg, ffmpeg_builder, process_manager, state_manager
|
||||
from facefusion.download import conditional_download
|
||||
from facefusion.ffmpeg import concat_video, extract_frames, merge_video, read_audio_buffer, replace_audio, restore_audio
|
||||
from facefusion.ffmpeg import concat_video, extract_frames, fix_audio_encoder, fix_video_encoder, merge_video, read_audio_buffer, replace_audio, restore_audio
|
||||
from facefusion.ffprobe import extract_video_metadata
|
||||
from facefusion.filesystem import copy_file
|
||||
from facefusion.temp_helper import clear_temp_directory, create_temp_directory, get_temp_file_path, resolve_temp_frame_set
|
||||
@@ -236,3 +236,29 @@ def test_replace_audio() -> None:
|
||||
clear_temp_directory(target_path)
|
||||
|
||||
state_manager.init_item('output_audio_encoder', 'aac')
|
||||
|
||||
|
||||
def test_fix_audio_encoder() -> None:
|
||||
assert fix_audio_encoder('avi', 'libopus') == 'aac'
|
||||
assert fix_audio_encoder('m4v', 'libopus') == 'aac'
|
||||
assert fix_audio_encoder('mpeg', 'libopus') == 'aac'
|
||||
assert fix_audio_encoder('wmv', 'libopus') == 'aac'
|
||||
assert fix_audio_encoder('mov', 'flac') == 'aac'
|
||||
assert fix_audio_encoder('mov', 'libopus') == 'aac'
|
||||
assert fix_audio_encoder('mxf', 'libopus') == 'pcm_s16le'
|
||||
assert fix_audio_encoder('webm', 'aac') == 'libopus'
|
||||
assert fix_audio_encoder('mp4', 'aac') == 'aac'
|
||||
assert fix_audio_encoder('avi', 'aac') == 'aac'
|
||||
|
||||
|
||||
def test_fix_video_encoder() -> None:
|
||||
assert fix_video_encoder('m4v', 'libx265') == 'libx264'
|
||||
assert fix_video_encoder('mpeg', 'libx265') == 'libx264'
|
||||
assert fix_video_encoder('mxf', 'libx265') == 'libx264'
|
||||
assert fix_video_encoder('wmv', 'libx265') == 'libx264'
|
||||
assert fix_video_encoder('mkv', 'rawvideo') == 'libx264'
|
||||
assert fix_video_encoder('mp4', 'rawvideo') == 'libx264'
|
||||
assert fix_video_encoder('mov', 'libvpx-vp9') == 'libx264'
|
||||
assert fix_video_encoder('webm', 'libx264') == 'libvpx-vp9'
|
||||
assert fix_video_encoder('mp4', 'libx265') == 'libx265'
|
||||
assert fix_video_encoder('avi', 'rawvideo') == 'rawvideo'
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from shutil import which
|
||||
|
||||
from facefusion import ffmpeg_builder
|
||||
from facefusion.ffmpeg_builder import chain, concat, convert_color_space, keep_video_alpha, restrict_color_transfer, run, select_frame_range, set_audio_quality, set_audio_sample_size, set_faststart, set_stream_mode, set_video_encoder, set_video_fps, set_video_quality, set_video_tag
|
||||
from facefusion.ffmpeg_builder import chain, concat, convert_color_space, keep_video_alpha, restrict_color_transfer, run, select_frame_range, set_audio_quality, set_audio_sample_size, set_faststart, set_filter_thread_count, set_global_thread_count, set_input_seek, set_output_format, set_stream_mode, set_video_encoder, set_video_fps, set_video_quality, set_video_tag
|
||||
|
||||
|
||||
def test_run() -> None:
|
||||
@@ -41,6 +41,19 @@ def test_set_stream_mode() -> None:
|
||||
assert set_stream_mode('v4l2') == [ '-f', 'v4l2' ]
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_set_input_seek() -> None:
|
||||
assert set_input_seek(0.0) == [ '-ss', '0.0' ]
|
||||
assert set_input_seek(4.2) == [ '-ss', '4.2' ]
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_set_output_format() -> None:
|
||||
assert set_output_format('rawvideo') == [ '-f', 'rawvideo' ]
|
||||
|
||||
|
||||
def test_select_frame_range() -> None:
|
||||
assert select_frame_range(0, None, 30) == [ '-vf', 'trim=start_frame=0,fps=30' ]
|
||||
assert select_frame_range(None, 100, 30) == [ '-vf', 'trim=end_frame=100,fps=30' ]
|
||||
@@ -83,6 +96,16 @@ def test_set_audio_quality() -> None:
|
||||
assert set_audio_quality('flac', 100) == []
|
||||
|
||||
|
||||
def test_set_global_thread_count() -> None:
|
||||
assert set_global_thread_count(8) == [ '-threads', '8' ]
|
||||
assert set_global_thread_count(16) == [ '-threads', '16' ]
|
||||
|
||||
|
||||
def test_set_filter_thread_count() -> None:
|
||||
assert set_filter_thread_count(8) == [ '-filter_threads', '8' ]
|
||||
assert set_filter_thread_count(16) == [ '-filter_threads', '16' ]
|
||||
|
||||
|
||||
def test_set_faststart() -> None:
|
||||
assert set_faststart('m4v') == [ '-movflags', '+faststart' ]
|
||||
assert set_faststart('mov') == [ '-movflags', '+faststart' ]
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import tempfile
|
||||
|
||||
import numpy
|
||||
import pytest
|
||||
|
||||
from facefusion import ffmpeg, ffmpeg_builder, process_manager, state_manager
|
||||
from facefusion.download import conditional_download
|
||||
from facefusion.ffprobe import extract_video_metadata
|
||||
from facefusion.temp_helper import create_temp_directory, get_temp_file_path
|
||||
from facefusion.video_manager import clear_video_pool, close_video_writer, conditional_set_video_reader_position, get_reader, get_writer, read_video_reader_frame, read_video_reader_window, refresh_video_reader, write_video_writer_frame
|
||||
from .helper import get_test_example_file, get_test_examples_directory
|
||||
|
||||
|
||||
@pytest.fixture(scope = 'module', autouse = True)
|
||||
def before_all() -> None:
|
||||
process_manager.start()
|
||||
conditional_download(get_test_examples_directory(),
|
||||
[
|
||||
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/target-240p.mp4'
|
||||
])
|
||||
|
||||
for video_fps in [ 25, 30 ]:
|
||||
ffmpeg.run_ffmpeg(
|
||||
ffmpeg_builder.chain(
|
||||
ffmpeg_builder.set_input(get_test_example_file('target-240p.mp4')),
|
||||
ffmpeg_builder.set_video_fps(video_fps),
|
||||
ffmpeg_builder.set_output(get_test_example_file('target-240p-' + str(video_fps) + 'fps.mp4'))
|
||||
)
|
||||
)
|
||||
|
||||
state_manager.init_item('temp_path', tempfile.gettempdir())
|
||||
state_manager.init_item('temp_frame_format', 'png')
|
||||
state_manager.init_item('output_video_encoder', 'libx264')
|
||||
state_manager.init_item('output_video_quality', 80)
|
||||
state_manager.init_item('output_video_preset', 'veryfast')
|
||||
state_manager.init_item('temp_pixel_format', 'bgr24')
|
||||
|
||||
|
||||
@pytest.fixture(scope = 'function', autouse = True)
|
||||
def before_each() -> None:
|
||||
clear_video_pool()
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_get_reader() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
video_metadata = video_reader.get('metadata')
|
||||
|
||||
assert video_metadata.get('resolution') == (426, 226)
|
||||
assert video_metadata.get('fps') == 25.0
|
||||
assert video_metadata.get('frame_total') == 270
|
||||
assert video_reader.get('position') == 0
|
||||
assert get_reader(get_test_example_file('target-240p-25fps.mp4')) is video_reader
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_conditional_set_video_reader_position() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
|
||||
conditional_set_video_reader_position(video_reader, 50)
|
||||
|
||||
assert video_reader.get('position') == 50
|
||||
|
||||
conditional_set_video_reader_position(video_reader, 10)
|
||||
|
||||
assert video_reader.get('position') == 10
|
||||
|
||||
conditional_set_video_reader_position(video_reader, 200)
|
||||
|
||||
assert video_reader.get('position') == 200
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_refresh_video_reader() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
sequential_frames = {}
|
||||
|
||||
for frame_number in range(30):
|
||||
sequential_frames[frame_number] = read_video_reader_frame(video_reader)
|
||||
|
||||
for frame_number in [ 5, 17, 29 ]:
|
||||
refresh_video_reader(video_reader, frame_number)
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
|
||||
assert numpy.array_equal(vision_frame, sequential_frames.get(frame_number)) is True
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_read_video_reader_frame() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
|
||||
assert vision_frame.shape == (226, 426, 3)
|
||||
assert video_reader.get('position') == 1
|
||||
|
||||
conditional_set_video_reader_position(video_reader, 269)
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
|
||||
assert vision_frame.shape == (226, 426, 3)
|
||||
assert read_video_reader_frame(video_reader) is None
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_read_video_reader_window() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
frame_set = read_video_reader_window(video_reader, 0, 4)
|
||||
|
||||
assert sorted(frame_set) == [ 0, 1, 2, 3, 4 ]
|
||||
|
||||
frame_set = read_video_reader_window(video_reader, 100, 104)
|
||||
|
||||
assert sorted(frame_set) == [ 100, 101, 102, 103, 104 ]
|
||||
|
||||
frame_set = read_video_reader_window(video_reader, 268, 275)
|
||||
|
||||
assert sorted(frame_set) == [ 268, 269 ]
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_evict_video_reader_buffer() -> None:
|
||||
video_reader = get_reader(get_test_example_file('target-240p-25fps.mp4'))
|
||||
read_video_reader_window(video_reader, 0, 4)
|
||||
frame_set = read_video_reader_window(video_reader, 21, 25)
|
||||
|
||||
assert min(frame_set) == 5
|
||||
assert max(frame_set) == 25
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_get_writer() -> None:
|
||||
target_path = get_test_example_file('target-240p-25fps.mp4')
|
||||
video_metadata = extract_video_metadata(target_path)
|
||||
create_temp_directory(target_path)
|
||||
video_writer = get_writer(target_path, video_metadata, 25.0, (426, 226), (426, 226), 25.0)
|
||||
|
||||
assert get_writer(target_path, video_metadata, 25.0, (426, 226), (426, 226), 25.0) is video_writer
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_write_video_writer_frame() -> None:
|
||||
target_path = get_test_example_file('target-240p-25fps.mp4')
|
||||
create_temp_directory(target_path)
|
||||
video_reader = get_reader(target_path)
|
||||
video_writer = get_writer(target_path, video_reader.get('metadata'), 25.0, (426, 226), (426, 226), 25.0)
|
||||
|
||||
for frame_number in range(25):
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
write_video_writer_frame(video_writer, vision_frame)
|
||||
|
||||
assert close_video_writer(video_writer) is True
|
||||
|
||||
video_metadata = extract_video_metadata(get_temp_file_path(target_path))
|
||||
|
||||
assert video_metadata.get('duration') == 1.0
|
||||
assert video_metadata.get('frame_total') == 25
|
||||
assert video_metadata.get('fps') == 25.0
|
||||
assert video_metadata.get('resolution') == (426, 226)
|
||||
assert video_metadata.get('color_transfer') == 'bt709'
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_close_video_writer() -> None:
|
||||
target_path = get_test_example_file('target-240p-30fps.mp4')
|
||||
create_temp_directory(target_path)
|
||||
video_reader = get_reader(target_path)
|
||||
video_writer = get_writer(target_path, video_reader.get('metadata'), 30.0, (426, 226), (426, 226), 30.0)
|
||||
vision_frame = read_video_reader_frame(video_reader)
|
||||
write_video_writer_frame(video_writer, vision_frame)
|
||||
|
||||
assert close_video_writer(video_writer) is True
|
||||
|
||||
|
||||
#todo: needs review - [testing] question if the assertions are good
|
||||
#todo: run mutation testing, strip down to the minimum, test with real data
|
||||
def test_clear_video_pool() -> None:
|
||||
target_path = get_test_example_file('target-240p-25fps.mp4')
|
||||
create_temp_directory(target_path)
|
||||
video_reader = get_reader(target_path)
|
||||
video_writer = get_writer(target_path, video_reader.get('metadata'), 25.0, (426, 226), (426, 226), 25.0)
|
||||
clear_video_pool()
|
||||
|
||||
assert video_reader.get('process').returncode == -9
|
||||
assert video_writer.get('process').returncode == -9
|
||||
Reference in New Issue
Block a user