CalvinBackup/facefusion
1
0
Fork 0
mirror of https://github.com/facefusion/facefusion.git synced 2026-07-26 03:51:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Push based receive with queue (#1146)

Browse Source 
* move to push based receive

* move to push based receive, fix mocks

* fix tests

* add todos

* remove asyncio

* remove asyncio

* resolve todos

* move to queue without events

* prevent debug spam

* concurrent stream inference

stream_video.py: pipeline face-swap inference across execution_thread_count workers (ThreadPoolExecutor + bounded in-flight deque, ordered encode) to keep the GPU busy during encode

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* add todos

* add todos

* add missing state

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Henry Ruhs
2026-06-05 00:18:57 +02:00
committed by GitHub
parent 1f494f54db
commit 775985645e
15 changed files with 193 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api_stream_audio.py
+32 -32
View File
@@ -1,13 +1,14 @@
import ctypes
import threading
from collections import deque
from functools import partial
from queue import Queue
from unittest.mock import MagicMock, patch
import numpy
import pytest
from facefusion import rtc, rtc_store, state_manager
from facefusion.apis.stream_audio import fill_audio_deque, receive_audio_frames, run_audio_encode_loop
from facefusion.apis.stream_audio import handle_audio_frame, receive_audio_frames, run_audio_encode_loop
from facefusion.download import conditional_download
from facefusion.ffmpeg import read_audio_buffer
from facefusion.hash_helper import create_hash
@@ -52,54 +53,36 @@ def test_run_audio_encode_loop() -> None:
'receiver_bitrate': ctypes.c_uint(0)
}
audio_deque : deque[AudioPack] = deque()
audio_event = threading.Event()
audio_queue : Queue[AudioPack] = Queue(maxsize = 300)
audio_deque.append((audio_frame, 0.100))
audio_event.set()
audio_queue.put((audio_frame, 0.100))
encoder_mock = MagicMock()
encoder_mock.encode.return_value = bytes([ 1 ] * 32)
with patch('facefusion.apis.stream_audio.opus_encoder.encode', encoder_mock.encode):
with patch('facefusion.apis.stream_audio.rtc.send_audio') as send_audio_mock:
audio_loop_thread = threading.Thread(target = run_audio_encode_loop, args = (rtc_peer, audio_deque, audio_event), daemon = True)
audio_loop_thread = threading.Thread(target = run_audio_encode_loop, args = (rtc_peer, audio_queue), daemon = True)
audio_loop_thread.start()
audio_deque.append((numpy.empty(0), 0.0))
audio_event.set()
audio_queue.put((numpy.empty(0), 0.0))
audio_loop_thread.join(timeout = 5.0)
assert encoder_mock.encode.called is True
assert send_audio_mock.called is True
def test_fill_audio_deque() -> None:
audio_buffer = read_audio_buffer(get_test_example_file('source.mp3'), 48000, 16, 2)
audio_frame = numpy.frombuffer(audio_buffer, dtype = numpy.int16).astype(numpy.float32) / 32768.0
audio_decoder_mock = MagicMock()
audio_deque : deque[AudioPack] = deque()
audio_event = threading.Event()
with patch('facefusion.apis.stream_audio.decode_audio_frame', return_value = audio_frame.tobytes()):
fill_audio_deque('opus', audio_decoder_mock, audio_frame.tobytes(), audio_deque, audio_event)
buffer_frame, _ = audio_deque.popleft()
assert audio_event.is_set()
assert create_hash(buffer_frame.tobytes()) == create_hash(audio_frame.tobytes())
@pytest.mark.parametrize('audio_codec', [ 'opus' ])
def test_receive_audio_frames(audio_codec : AudioCodec) -> None:
audio_buffer = read_audio_buffer(get_test_example_file('source.mp3'), 48000, 16, 2)
audio_frame = numpy.frombuffer(audio_buffer, dtype = numpy.int16).astype(numpy.float32) / 32768.0
audio_deque : deque[AudioPack] = deque()
audio_event = threading.Event()
audio_queue : Queue[AudioPack] = Queue(maxsize = 300)
datachannel_library_mock = MagicMock()
datachannel_library_mock.rtcReceiveMessage.side_effect = [ 0, -1 ]
datachannel_mock = MagicMock()
ready_event = threading.Event()
# todo: lambda not allowed
datachannel_mock.rtcSetClosedCallback.side_effect = partial(lambda event, *args: event.set(), ready_event)
with patch('facefusion.apis.stream_audio.datachannel_module.create_static_library', return_value = datachannel_library_mock):
with patch('facefusion.libraries.datachannel.create_static_library', return_value = datachannel_mock):
with patch('facefusion.apis.stream_audio.decode_audio_frame', return_value = audio_frame.tobytes()):
rtc_peer_audio : RtcPeerAudio =\
{
@@ -107,10 +90,27 @@ def test_receive_audio_frames(audio_codec : AudioCodec) -> None:
'receiver_track': 0,
'codec': audio_codec
}
audio_receiver_thread = threading.Thread(target = receive_audio_frames, args = (rtc_peer_audio, audio_deque, audio_event), daemon = True)
audio_receiver_thread = threading.Thread(target = receive_audio_frames, args = (rtc_peer_audio, audio_queue), daemon = True)
audio_receiver_thread.start()
ready_event.wait(timeout = 5.0)
datachannel_mock.rtcSetFrameCallback.call_args[0][1](0, bytes([ 0 ]), 1, None, None)
datachannel_mock.rtcSetClosedCallback.call_args[0][1](0, None)
audio_receiver_thread.join(timeout = 5.0)
buffer_frame, _ = audio_deque.popleft()
buffer_frame, _ = audio_queue.get_nowait()
assert create_hash(buffer_frame.tobytes()) == create_hash(audio_frame.tobytes())
def test_handle_audio_frame() -> None:
audio_buffer = read_audio_buffer(get_test_example_file('source.mp3'), 48000, 16, 2)
audio_frame = numpy.frombuffer(audio_buffer, dtype = numpy.int16).astype(numpy.float32) / 32768.0
audio_decoder_mock = MagicMock()
audio_queue : Queue[AudioPack] = Queue(maxsize = 300)
with patch('facefusion.apis.stream_audio.decode_audio_frame', return_value = audio_frame.tobytes()):
handle_audio_frame('opus', audio_decoder_mock, audio_queue, 0, ctypes.c_void_p(), 1, ctypes.c_void_p(), ctypes.c_void_p())
buffer_frame, _ = audio_queue.get_nowait()
assert create_hash(buffer_frame.tobytes()) == create_hash(audio_frame.tobytes())
tests/test_api_stream_manager.py
+1 -2
View File
@@ -1,4 +1,3 @@
import asyncio
import ctypes
import threading
from unittest.mock import AsyncMock, patch
@@ -138,7 +137,7 @@ def test_run_peer_loop(video_codec : VideoCodec, payload_type : int, session_id
with patch('facefusion.apis.stream_manager.receive_video_frames'):
with patch('facefusion.apis.stream_manager.run_video_encode_loop'):
thread = threading.Thread(target = asyncio.run, args = (run_peer_loop(session_id, rtc_peer),), daemon = True)
thread = threading.Thread(target = run_peer_loop, args = (session_id, rtc_peer), daemon = True)
thread.start()
thread.join(timeout = 5.0)
tests/test_api_stream_video.py
+44 -52
View File
@@ -1,7 +1,8 @@
import ctypes
import struct
import threading
from collections import deque
from functools import partial
from queue import Queue
from unittest.mock import MagicMock, patch
import cv2
@@ -9,7 +10,7 @@ import numpy
import pytest
from facefusion import rtc, rtc_store, state_manager
from facefusion.apis.stream_video import create_video_decoder, create_video_encoder, decode_video_frame, destroy_video_decoder, destroy_video_encoder, encode_video_frame, fill_video_deque, receive_video_frames, run_video_encode_loop, update_video_encoder_bitrate
from facefusion.apis.stream_video import create_video_decoder, create_video_encoder, decode_video_frame, destroy_video_decoder, destroy_video_encoder, encode_video_frame, handle_video_frame, receive_video_frames, run_video_encode_loop, update_video_encoder_bitrate
from facefusion.codecs import aom_encoder, vpx_encoder
from facefusion.common_helper import is_linux, is_macos, is_windows
from facefusion.download import conditional_download
@@ -23,6 +24,7 @@ from .assert_helper import get_test_example_file, get_test_examples_directory
@pytest.fixture(scope = 'module', autouse = True)
def before_all() -> None:
state_manager.init_item('download_providers', [ 'github', 'huggingface' ])
state_manager.init_item('execution_thread_count', 8)
state_manager.init_item('processors', [])
aom_module.pre_check()
@@ -59,18 +61,15 @@ def test_run_video_encode_loop(video_codec : VideoCodec, payload_type : int) ->
'receiver_bitrate': ctypes.c_uint(8000)
}
video_deque : deque[VideoPack] = deque()
video_event = threading.Event()
video_queue : Queue[VideoPack] = Queue(maxsize = 30)
video_deque.append((video_frame, 0.1))
video_event.set()
video_queue.put((video_frame, 0.1))
with patch('facefusion.apis.stream_video.rtc.send_video') as send_video_mock:
encode_loop_thread = threading.Thread(target = run_video_encode_loop, args = (rtc_peer, video_deque, video_event), daemon = True)
encode_loop_thread = threading.Thread(target = run_video_encode_loop, args = (rtc_peer, video_queue), daemon = True)
encode_loop_thread.start()
empty_vision_frame = numpy.empty(0)
video_deque.append((empty_vision_frame, 0.0))
video_event.set()
video_queue.put((empty_vision_frame, 0.0))
encode_loop_thread.join(timeout = 5.0)
assert send_video_mock.called
@@ -89,13 +88,14 @@ def test_run_video_encode_loop(video_codec : VideoCodec, payload_type : int) ->
@pytest.mark.parametrize('video_codec', [ 'av1', 'vp8' ])
def test_receive_video_frames(video_codec : VideoCodec) -> None:
video_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
video_deque : deque[VideoPack] = deque()
video_event = threading.Event()
video_queue : Queue[VideoPack] = Queue(maxsize = 30)
datachannel_library_mock = MagicMock()
datachannel_library_mock.rtcReceiveMessage.side_effect = [ 0, -1 ]
datachannel_mock = MagicMock()
ready_event = threading.Event()
#todo: lambda not allowed
datachannel_mock.rtcSetClosedCallback.side_effect = partial(lambda event, *args: event.set(), ready_event)
with patch('facefusion.apis.stream_video.datachannel_module.create_static_library', return_value = datachannel_library_mock):
with patch('facefusion.libraries.datachannel.create_static_library', return_value = datachannel_mock):
with patch('facefusion.apis.stream_video.decode_video_frame', return_value = video_frame):
rtc_peer_video : RtcPeerVideo =\
{
@@ -103,11 +103,14 @@ def test_receive_video_frames(video_codec : VideoCodec) -> None:
'receiver_track': 0,
'codec': video_codec
}
video_receiver_thread = threading.Thread(target = receive_video_frames, args = (rtc_peer_video, video_deque, video_event), daemon = True)
video_receiver_thread = threading.Thread(target = receive_video_frames, args = (rtc_peer_video, video_queue), daemon = True)
video_receiver_thread.start()
ready_event.wait(timeout = 5.0)
datachannel_mock.rtcSetFrameCallback.call_args[0][1](0, bytes([ 0 ]), 1, None, None)
datachannel_mock.rtcSetClosedCallback.call_args[0][1](0, None)
video_receiver_thread.join(timeout = 5.0)
vision_frame, _ = video_deque.popleft()
vision_frame, _ = video_queue.get_nowait()
if is_linux() or is_windows():
assert create_hash(vision_frame.tobytes()) == 'a17439db'
@@ -116,37 +119,6 @@ def test_receive_video_frames(video_codec : VideoCodec) -> None:
assert create_hash(vision_frame.tobytes()) == '38d00e2a'
@pytest.mark.parametrize('video_codec', [ 'av1', 'vp8' ])
def test_fill_video_deque(video_codec : VideoCodec) -> None:
video_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
input_buffer = cv2.cvtColor(video_frame, cv2.COLOR_BGR2YUV_I420).tobytes()
video_encoder = create_video_encoder(video_codec, (426, 226), 1000)
video_decoder = create_video_decoder(video_codec)
encode_buffer = encode_video_frame(video_codec, video_encoder, input_buffer, (426, 226), 0)
video_deque : deque[VideoPack] = deque()
video_event = threading.Event()
fill_video_deque(video_codec, video_decoder, encode_buffer, video_deque, video_event)
vision_frame, _ = video_deque.popleft()
assert video_event.is_set()
if is_linux() or is_windows():
if video_codec == 'av1':
assert create_hash(vision_frame.tobytes()) == 'b5b6486d'
if video_codec == 'vp8':
assert create_hash(vision_frame.tobytes()) == '99ef2c25'
if is_macos():
if video_codec == 'av1':
assert create_hash(vision_frame.tobytes()) == '74e9926f'
if video_codec == 'vp8':
assert create_hash(vision_frame.tobytes()) == 'ff3ecb43'
@pytest.mark.parametrize('video_codec', [ 'av1', 'vp8' ])
def test_encode_and_decode_video_frame(video_codec : VideoCodec) -> None:
video_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
@@ -165,7 +137,7 @@ def test_encode_and_decode_video_frame(video_codec : VideoCodec) -> None:
if is_macos():
if video_codec == 'av1':
assert create_hash(decode_buffer) == '74e9926f'
assert create_hash(decode_buffer) == 'eafd1fab'
if video_codec == 'vp8':
assert create_hash(decode_buffer) == 'ff3ecb43'
@@ -178,10 +150,11 @@ def test_create_and_destroy_video_decoder(video_codec : VideoCodec) -> None:
video_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
input_buffer = cv2.cvtColor(video_frame, cv2.COLOR_BGR2YUV_I420).tobytes()
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'av1':
video_encoder = aom_encoder.create((426, 226), 1000, 1, 0)
encode_buffer = aom_encoder.encode(video_encoder, input_buffer, (426, 226), 0)
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'vp8':
video_encoder = vpx_encoder.create((426, 226), 1000, 1, 0)
encode_buffer = vpx_encoder.encode(video_encoder, input_buffer, (426, 226), 0)
@@ -201,17 +174,18 @@ def test_create_and_destroy_video_encoder(video_codec : VideoCodec) -> None:
input_buffer = cv2.cvtColor(video_frame, cv2.COLOR_BGR2YUV_I420).tobytes()
video_encoder = create_video_encoder(video_codec, (426, 226), 4000)
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'av1':
assert aom_encoder.encode(video_encoder, input_buffer, (426, 226), 0)
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'vp8':
assert vpx_encoder.encode(video_encoder, input_buffer, (426, 226), 0)
destroy_video_encoder(video_codec, video_encoder)
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'av1':
assert aom_encoder.encode(video_encoder, input_buffer, (426, 226), 1) == bytes()
# todo: this head be hash based checks before, now the codnitions seem pointless
if video_codec == 'vp8':
assert vpx_encoder.encode(video_encoder, input_buffer, (426, 226), 1) == bytes()
@@ -235,3 +209,21 @@ def test_update_video_encoder_bitrate(video_codec : VideoCodec) -> None:
assert struct.unpack_from('I', video_encoder, 64 + 112)[0] == 6000
destroy_video_encoder(video_codec, video_encoder)
@pytest.mark.parametrize('video_codec', [ 'av1', 'vp8' ])
def test_handle_video_frame(video_codec : VideoCodec) -> None:
video_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
video_decoder = create_video_decoder(video_codec)
video_queue : Queue[VideoPack] = Queue(maxsize = 30)
with patch('facefusion.apis.stream_video.decode_video_frame', return_value = video_frame):
handle_video_frame(video_codec, video_decoder, video_queue, 0, ctypes.c_void_p(), 1, ctypes.c_void_p(), ctypes.c_void_p())
vision_frame, _ = video_queue.get_nowait()
if is_linux() or is_windows():
assert create_hash(vision_frame.tobytes()) == 'a17439db'
if is_macos():
assert create_hash(vision_frame.tobytes()) == '38d00e2a'
tests/test_codec_aom_decoder.py
+2 -2
View File
@@ -26,7 +26,7 @@ def before_all() -> None:
def test_create() -> None:
assert create(1)
with patch('facefusion.codecs.aom_decoder.aom_module.create_static_library', return_value = None):
with patch('facefusion.libraries.aom.create_static_library', return_value = None):
assert create(1) is None
@@ -42,7 +42,7 @@ def test_decode() -> None:
assert create_hash(decode(aom_decoder, encode_buffer).get('buffer')) == 'e3c0ebd8'
if is_macos():
assert create_hash(decode(aom_decoder, encode_buffer).get('buffer')) == '0a0ab3d0'
assert create_hash(decode(aom_decoder, encode_buffer).get('buffer')) == 'c8c6fdaa'
def test_destroy() -> None:
tests/test_codec_aom_encoder.py
+1 -1
View File
@@ -25,7 +25,7 @@ def before_all() -> None:
def test_create() -> None:
assert create((320, 240), 1000, 8, 16)
with patch('facefusion.codecs.aom_encoder.aom_module.create_static_library', return_value = None):
with patch('facefusion.libraries.aom.create_static_library', return_value = None):
assert create((320, 240), 1000, 8, 16) is None
tests/test_codec_opus_decoder.py
+1 -1
View File
@@ -26,7 +26,7 @@ def before_all() -> None:
def test_create() -> None:
assert create(48000, 2)
with patch('facefusion.codecs.opus_decoder.opus_module.create_static_library', return_value = None):
with patch('facefusion.libraries.opus.create_static_library', return_value = None):
assert create(48000, 2) is None
tests/test_codec_opus_encoder.py
+1 -1
View File
@@ -25,7 +25,7 @@ def before_all() -> None:
def test_create() -> None:
assert create(48000, 2)
with patch('facefusion.codecs.opus_encoder.opus_module.create_static_library', return_value = None):
with patch('facefusion.libraries.opus.create_static_library', return_value = None):
assert create(48000, 2) is None
tests/test_codec_vpx_decoder.py
+1 -1
View File
@@ -26,7 +26,7 @@ def before_all() -> None:
def test_create() -> None:
assert create(1)
with patch('facefusion.codecs.vpx_decoder.vpx_module.create_static_library', return_value = None):
with patch('facefusion.libraries.vpx.create_static_library', return_value = None):
assert create(1) is None
tests/test_codec_vpx_encoder.py
+1 -1
View File
@@ -25,7 +25,7 @@ def before_all() -> None:
def test_create() -> None:
assert create((320, 240), 1000, 8, 16)
with patch('facefusion.codecs.vpx_encoder.vpx_module.create_static_library', return_value = None):
with patch('facefusion.libraries.vpx.create_static_library', return_value = None):
assert create((320, 240), 1000, 8, 16) is None