mirror of
https://github.com/hacksider/Deep-Live-Cam.git
synced 2026-08-27 19:20:26 +02:00
Rendering optimization
The PNG encode/decode alone was consuming significant CPU time per frame. This is eliminated entirely.
This commit is contained in:
Kenneth Estanislao
@@ -1,12 +1,16 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import importlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import ModuleType
|
||||
from typing import Any, List, Callable
|
||||
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
import modules
|
||||
import modules.globals
|
||||
import modules.globals
|
||||
|
||||
FRAME_PROCESSORS_MODULES: List[ModuleType] = []
|
||||
FRAME_PROCESSORS_INTERFACE = [
|
||||
@@ -107,3 +111,265 @@ def process_video(source_path: str, frame_paths: list[str], process_frames: Call
|
||||
with tqdm(total=total, desc='Processing', unit='frame', dynamic_ncols=True, bar_format=progress_bar_format) as progress:
|
||||
progress.set_postfix({'execution_providers': modules.globals.execution_providers, 'execution_threads': modules.globals.execution_threads, 'max_memory': modules.globals.max_memory})
|
||||
multi_process_frame(source_path, frame_paths, process_frames, progress)
|
||||
|
||||
|
||||
def process_video_in_memory(source_path: str, target_path: str, fps: float) -> bool:
|
||||
"""Process video frames in-memory using FFmpeg pipes, eliminating disk I/O.
|
||||
|
||||
Reads raw frames from the source video via an FFmpeg decoder pipe, runs each
|
||||
frame through all active frame processors sequentially, and writes the
|
||||
result directly to an FFmpeg encoder pipe. This avoids extracting frames to
|
||||
PNG on disk, which is the biggest I/O bottleneck in the disk-based pipeline.
|
||||
|
||||
Returns True on success, False on failure (caller should fall back to the
|
||||
disk-based pipeline).
|
||||
"""
|
||||
import cv2
|
||||
from modules.face_analyser import get_one_face
|
||||
from modules.utilities import (
|
||||
get_video_dimensions,
|
||||
estimate_frame_count,
|
||||
get_temp_output_path,
|
||||
)
|
||||
|
||||
temp_output_path = get_temp_output_path(target_path)
|
||||
|
||||
# --- Pre-load source face (needed by face_swapper in simple mode) ---
|
||||
source_face = None
|
||||
if source_path and os.path.exists(source_path):
|
||||
source_img = cv2.imread(source_path)
|
||||
if source_img is not None:
|
||||
source_face = get_one_face(source_img)
|
||||
del source_img
|
||||
if source_face is None:
|
||||
print("[DLC.CORE] Warning: No face detected in source image. "
|
||||
"Face swapping will be skipped.")
|
||||
|
||||
# --- Collect frame processors & reset per-video state ---
|
||||
frame_processors = get_frame_processors_modules(modules.globals.frame_processors)
|
||||
for fp in frame_processors:
|
||||
if hasattr(fp, 'PREVIOUS_FRAME_RESULT'):
|
||||
fp.PREVIOUS_FRAME_RESULT = None
|
||||
|
||||
# --- Video metadata ---
|
||||
try:
|
||||
width, height = get_video_dimensions(target_path)
|
||||
except Exception as e:
|
||||
print(f"[DLC.CORE] Failed to get video dimensions: {e}")
|
||||
return False
|
||||
|
||||
total_frames = estimate_frame_count(target_path, fps)
|
||||
frame_size = width * height * 3
|
||||
|
||||
# --- Build encoder arguments ---
|
||||
encoder = modules.globals.video_encoder
|
||||
encoder_options: List[str] = []
|
||||
is_hw_encoder = False
|
||||
|
||||
if 'CUDAExecutionProvider' in modules.globals.execution_providers:
|
||||
if encoder == 'libx264':
|
||||
encoder = 'h264_nvenc'
|
||||
is_hw_encoder = True
|
||||
encoder_options = [
|
||||
'-preset', 'p4', '-tune', 'hq', '-rc', 'vbr',
|
||||
'-cq', str(modules.globals.video_quality), '-b:v', '0',
|
||||
]
|
||||
elif encoder == 'libx265':
|
||||
encoder = 'hevc_nvenc'
|
||||
is_hw_encoder = True
|
||||
encoder_options = [
|
||||
'-preset', 'p4', '-tune', 'hq', '-rc', 'vbr',
|
||||
'-cq', str(modules.globals.video_quality), '-b:v', '0',
|
||||
]
|
||||
elif 'DmlExecutionProvider' in modules.globals.execution_providers:
|
||||
if encoder == 'libx264':
|
||||
encoder = 'h264_amf'
|
||||
is_hw_encoder = True
|
||||
encoder_options = [
|
||||
'-quality', 'quality', '-rc', 'vbr_latency',
|
||||
'-qp_i', str(modules.globals.video_quality),
|
||||
'-qp_p', str(modules.globals.video_quality),
|
||||
]
|
||||
elif encoder == 'libx265':
|
||||
encoder = 'hevc_amf'
|
||||
is_hw_encoder = True
|
||||
encoder_options = [
|
||||
'-quality', 'quality', '-rc', 'vbr_latency',
|
||||
'-qp_i', str(modules.globals.video_quality),
|
||||
'-qp_p', str(modules.globals.video_quality),
|
||||
]
|
||||
|
||||
if not is_hw_encoder:
|
||||
if encoder == 'libx264':
|
||||
encoder_options = [
|
||||
'-preset', 'medium',
|
||||
'-crf', str(modules.globals.video_quality),
|
||||
'-tune', 'film',
|
||||
]
|
||||
elif encoder == 'libx265':
|
||||
encoder_options = [
|
||||
'-preset', 'medium',
|
||||
'-crf', str(modules.globals.video_quality),
|
||||
'-x265-params', 'log-level=error',
|
||||
]
|
||||
elif encoder == 'libvpx-vp9':
|
||||
encoder_options = [
|
||||
'-crf', str(modules.globals.video_quality),
|
||||
'-b:v', '0', '-cpu-used', '2',
|
||||
]
|
||||
|
||||
# --- Attempt pipeline (hw encoder first, then sw fallback) ---
|
||||
encoders_to_try = [(encoder, encoder_options)]
|
||||
if is_hw_encoder:
|
||||
# Software fallback
|
||||
sw_encoder = 'libx264'
|
||||
sw_options = [
|
||||
'-preset', 'medium',
|
||||
'-crf', str(modules.globals.video_quality),
|
||||
'-tune', 'film',
|
||||
]
|
||||
encoders_to_try.append((sw_encoder, sw_options))
|
||||
|
||||
for attempt, (enc, enc_opts) in enumerate(encoders_to_try):
|
||||
# Reset interpolation state on retry
|
||||
if attempt > 0:
|
||||
for fp in frame_processors:
|
||||
if hasattr(fp, 'PREVIOUS_FRAME_RESULT'):
|
||||
fp.PREVIOUS_FRAME_RESULT = None
|
||||
|
||||
success = _run_pipe_pipeline(
|
||||
target_path, temp_output_path, fps,
|
||||
source_face, frame_processors,
|
||||
width, height, frame_size, total_frames,
|
||||
enc, enc_opts,
|
||||
)
|
||||
if success:
|
||||
return True
|
||||
|
||||
if attempt == 0 and is_hw_encoder:
|
||||
print(f"[DLC.CORE] Hardware encoder '{enc}' failed, "
|
||||
f"retrying with software encoder...")
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _run_pipe_pipeline(
|
||||
target_path: str,
|
||||
temp_output_path: str,
|
||||
fps: float,
|
||||
source_face: Any,
|
||||
frame_processors: List[Any],
|
||||
width: int,
|
||||
height: int,
|
||||
frame_size: int,
|
||||
total_frames: int,
|
||||
encoder: str,
|
||||
encoder_options: List[str],
|
||||
) -> bool:
|
||||
"""Run the FFmpeg-pipe read → process → encode pipeline once."""
|
||||
|
||||
# --- Reader: decode source video to raw BGR24 on stdout ---
|
||||
reader_cmd = [
|
||||
'ffmpeg', '-hide_banner',
|
||||
'-hwaccel', 'auto',
|
||||
'-i', target_path,
|
||||
'-f', 'rawvideo',
|
||||
'-pix_fmt', 'bgr24',
|
||||
'-v', 'error',
|
||||
'-',
|
||||
]
|
||||
|
||||
# --- Writer: encode raw BGR24 from stdin ---
|
||||
writer_cmd = [
|
||||
'ffmpeg', '-hide_banner',
|
||||
'-f', 'rawvideo',
|
||||
'-pix_fmt', 'bgr24',
|
||||
'-s', f'{width}x{height}',
|
||||
'-r', str(fps),
|
||||
'-i', '-',
|
||||
'-c:v', encoder,
|
||||
]
|
||||
writer_cmd.extend(encoder_options)
|
||||
writer_cmd.extend([
|
||||
'-pix_fmt', 'yuv420p',
|
||||
'-movflags', '+faststart',
|
||||
'-vf', 'colorspace=bt709:iall=bt601-6-625:fast=1',
|
||||
'-v', 'error',
|
||||
'-y', temp_output_path,
|
||||
])
|
||||
|
||||
reader = None
|
||||
writer = None
|
||||
try:
|
||||
reader = subprocess.Popen(
|
||||
reader_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
writer = subprocess.Popen(
|
||||
writer_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"[DLC.CORE] Failed to start FFmpeg pipes: {e}")
|
||||
for proc in (reader, writer):
|
||||
if proc:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
processed_count = 0
|
||||
bar_fmt = ('{l_bar}{bar}| {n_fmt}/{total_fmt} '
|
||||
'[{elapsed}<{remaining}, {rate_fmt}{postfix}]')
|
||||
|
||||
try:
|
||||
with tqdm(total=total_frames, desc='Processing', unit='frame',
|
||||
dynamic_ncols=True, bar_format=bar_fmt) as progress:
|
||||
progress.set_postfix({
|
||||
'execution_providers': modules.globals.execution_providers,
|
||||
'threads': modules.globals.execution_threads,
|
||||
'mode': 'in-memory',
|
||||
})
|
||||
|
||||
while True:
|
||||
raw = reader.stdout.read(frame_size)
|
||||
if len(raw) != frame_size:
|
||||
break
|
||||
|
||||
frame = np.frombuffer(raw, dtype=np.uint8).reshape(
|
||||
(height, width, 3)
|
||||
).copy()
|
||||
|
||||
# Run frame through every active processor
|
||||
for fp in frame_processors:
|
||||
frame = fp.process_frame(source_face, frame)
|
||||
|
||||
writer.stdin.write(frame.tobytes())
|
||||
processed_count += 1
|
||||
progress.update(1)
|
||||
|
||||
# Graceful shutdown
|
||||
writer.stdin.close()
|
||||
writer.wait()
|
||||
reader.wait()
|
||||
|
||||
if writer.returncode != 0:
|
||||
stderr_out = writer.stderr.read().decode(errors='ignore').strip()
|
||||
if stderr_out:
|
||||
print(f"[DLC.CORE] FFmpeg encoder error: {stderr_out}")
|
||||
return False
|
||||
|
||||
return processed_count > 0 and os.path.isfile(temp_output_path)
|
||||
|
||||
except BrokenPipeError:
|
||||
print("[DLC.CORE] FFmpeg pipe broken (encoder may not be available).")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[DLC.CORE] In-memory processing error: {e}")
|
||||
return False
|
||||
finally:
|
||||
for proc in (reader, writer):
|
||||
if proc:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user