CalvinBackup/facefusion
1
0
Fork 0
mirror of https://github.com/facefusion/facefusion.git synced 2026-08-17 06:20:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor/ffmpeg less stream (#1092)

Browse Source 
* remove ffmpeg from stream to use opus and vpx, add bunch of todos

* fix testing

* improve download checkout

* fix datachannel download, fix super dirty test clients - setup logic does not belong there

* fix testing
This commit is contained in:
Henry Ruhs
2026-05-11 16:36:23 +02:00
committed by henryruhs
parent 430b16ce56
commit 76c413a2c1
25 changed files with 371 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/stream_helper.py
+5 -3
View File
@@ -10,6 +10,7 @@ from facefusion.libraries import datachannel as datachannel_module
from facefusion.types import SdpOffer
# TODO: remove, use rtc.create_sdp with recvonly tracks instead
def create_sdp_offer() -> Optional[SdpOffer]:
datachannel_library = datachannel_module.create_static_library()
peer_connection = rtc.create_peer_connection(disable_auto_negotiation = True)
@@ -34,9 +35,10 @@ def create_sdp_offer() -> Optional[SdpOffer]:
return None
# TODO: remove, inline into test_api_stream.py
def open_websocket_stream(test_client : TestClient, subprotocols : list[str], source_content : bytes, ready_event : threading.Event, stop_event : threading.Event) -> None:
with test_client.websocket_connect('/stream', subprotocols = subprotocols) as websocket:
websocket.send_bytes(source_content)
with test_client.websocket_connect('/stream?mode=video', subprotocols = subprotocols) as websocket:
websocket.send_bytes(b'\x01' + source_content)
websocket.receive_text()
ready_event.set()
stop_event.wait()
stop_event.wait(timeout = 15)
tests/test_api_assets.py
+6 -6
View File
@@ -23,12 +23,6 @@ def before_all() -> None:
subprocess.run([ 'ffmpeg', '-i', get_test_example_file('target-240p.mp4'), '-vframes', '1', get_test_example_file('target-240p.jpg') ])
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
state_manager.init_item('temp_path', tempfile.gettempdir())
@@ -37,6 +31,12 @@ def before_each() -> None:
asset_store.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
def test_upload_asset(test_client : TestClient) -> None:
upload_response = test_client.post('/assets?type=source')
tests/test_api_capabilities.py
+8 -5
View File
@@ -8,8 +8,8 @@ from facefusion import capability_store, session_manager
from facefusion.apis.core import create_api
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
@pytest.fixture(scope = 'module', autouse = True)
def before_all() -> None:
program = ArgumentParser()
capability_store.register_capability_set(
[
@@ -31,15 +31,18 @@ def test_client() -> Iterator[TestClient]:
scopes = [ 'api' ]
)
with TestClient(create_api()) as test_client:
yield test_client
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
def test_get_capabilities(test_client : TestClient) -> None:
capabilities_response = test_client.get('/capabilities')
capabilities_body = capabilities_response.json()
tests/test_api_metrics.py
+5 -5
View File
@@ -8,17 +8,17 @@ from facefusion import metadata, session_manager
from facefusion.apis.core import create_api
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
@pytest.fixture(scope = 'function', autouse = True)
def mock_detect_execution_devices(mocker : MockerFixture) -> None:
mocker.patch('facefusion.system.state_manager.get_temp_path', return_value = '/tmp')
tests/test_api_ping.py
+5 -5
View File
@@ -7,17 +7,17 @@ from facefusion import metadata, session_manager
from facefusion.apis.core import create_api
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
def test_ping(test_client : TestClient) -> None:
create_session_response = test_client.post('/session', json =
{
tests/test_api_session.py
+5 -5
View File
@@ -10,17 +10,17 @@ from facefusion.apis.core import create_api
from facefusion.types import Session
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
session_manager.SESSIONS.clear()
def test_create_session(test_client : TestClient) -> None:
create_session_response = test_client.post('/session', json =
{
tests/test_api_state.py
+13 -12
View File
@@ -14,16 +14,6 @@ from .assert_helper import get_test_example_file, get_test_examples_directory
@pytest.fixture(scope = 'module', autouse = True)
def before_all() -> None:
conditional_download(get_test_examples_directory(),
[
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/source.jpg',
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/target-240p.mp4'
])
subprocess.run([ 'ffmpeg', '-i', get_test_example_file('target-240p.mp4'), '-vframes', '1', get_test_example_file('target-240p.jpg') ])
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
program = ArgumentParser()
capability_store.register_capability_set(
[
@@ -51,10 +41,15 @@ def test_client() -> Iterator[TestClient]:
],
scopes = [ 'api' ]
)
state_manager.init_item('execution_providers', [ 'cpu' ])
with TestClient(create_api()) as test_client:
yield test_client
conditional_download(get_test_examples_directory(),
[
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/source.jpg',
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/target-240p.mp4'
])
subprocess.run([ 'ffmpeg', '-i', get_test_example_file('target-240p.mp4'), '-vframes', '1', get_test_example_file('target-240p.jpg') ])
@pytest.fixture(scope = 'function', autouse = True)
@@ -63,6 +58,12 @@ def before_each() -> None:
asset_store.clear()
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
def test_get_state(test_client : TestClient) -> None:
get_state_response = test_client.get('/state')
tests/test_api_stream.py
+19 -23
View File
@@ -12,26 +12,18 @@ from facefusion.apis import asset_store
from facefusion.apis.core import create_api
from facefusion.core import common_pre_check, processors_pre_check
from facefusion.download import conditional_download
from facefusion.libraries import datachannel as datachannel_module
from .assert_helper import get_test_example_file, get_test_examples_directory
from .stream_helper import create_sdp_offer, open_websocket_stream
@pytest.fixture(scope = 'module', autouse = True)
def before_all() -> None:
conditional_download(get_test_examples_directory(),
[
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/source.jpg'
])
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
state_manager.init_item('execution_device_ids', [ 0 ])
state_manager.init_item('execution_providers', [ 'cpu' ])
state_manager.init_item('download_providers', [ 'github', 'huggingface' ])
state_manager.init_item('temp_path', tempfile.gettempdir())
state_manager.init_item('processors', [ 'face_swapper' ])
state_manager.init_item('face_selector_mode', 'many')
state_manager.init_item('face_detector_model', 'yolo_face')
state_manager.init_item('face_detector_size', '640x640')
state_manager.init_item('face_detector_score', 0.5)
@@ -44,24 +36,29 @@ def test_client() -> Iterator[TestClient]:
state_manager.init_item('face_mask_padding', [ 0, 0, 0, 0 ])
state_manager.init_item('face_swapper_model', 'hyperswap_1a_256')
state_manager.init_item('face_swapper_pixel_boost', '256x256')
state_manager.init_item('face_swapper_weight', 0.5)
common_pre_check()
processors_pre_check()
datachannel_module.pre_check()
with TestClient(create_api()) as test_client:
yield test_client
conditional_download(get_test_examples_directory(),
[
'https://github.com/facefusion/facefusion-assets/releases/download/examples-3.0.0/source.jpg'
])
@pytest.fixture(scope = 'function', autouse = True)
def before_each() -> None:
state_manager.init_item('source_paths', None)
session_manager.SESSIONS.clear()
asset_store.clear()
# TODO: enable again
@pytest.mark.skip
@pytest.fixture(scope = 'module')
def test_client() -> Iterator[TestClient]:
with TestClient(create_api()) as test_client:
yield test_client
def test_stream_image(test_client : TestClient) -> None:
create_session_response = test_client.post('/session', json =
{
@@ -92,10 +89,9 @@ def test_stream_image(test_client : TestClient) -> None:
assert select_response.status_code == 200
with test_client.websocket_connect('/stream', subprotocols =
with test_client.websocket_connect('/stream?mode=image', subprotocols =
[
'access_token.' + access_token,
'image'
'access_token.' + access_token
]) as websocket:
websocket.send_bytes(source_content)
output_bytes = websocket.receive_bytes()
@@ -104,8 +100,6 @@ def test_stream_image(test_client : TestClient) -> None:
assert output_vision_frame.shape == (1024, 1024, 3)
# TODO: enable again
@pytest.mark.skip
def test_stream_video(test_client : TestClient) -> None:
create_session_response = test_client.post('/session', json =
{
@@ -137,9 +131,11 @@ def test_stream_video(test_client : TestClient) -> None:
ready_event = threading.Event()
stop_event = threading.Event()
#TODO: use asyncio
stream_thread = threading.Thread(target = open_websocket_stream, args = (test_client, [ 'access_token.' + access_token, 'video' ], source_content, ready_event, stop_event))
stream_thread = threading.Thread(target = open_websocket_stream, args = (test_client, [ 'access_token.' + access_token ], source_content, ready_event, stop_event))
stream_thread.start()
ready_event.wait()
ready_event.wait(timeout = 10)
assert ready_event.is_set()
sdp_offer = create_sdp_offer()
stream_response = test_client.post('/stream', content = sdp_offer, headers =
@@ -152,4 +148,4 @@ def test_stream_video(test_client : TestClient) -> None:
assert stream_response.text
stop_event.set()
stream_thread.join()
stream_thread.join(timeout = 10)
tests/test_ffmpeg.py
+1 -23
View File
@@ -1,5 +1,4 @@
import os
import struct
import subprocess
import tempfile
@@ -8,7 +7,7 @@ import pytest
import facefusion.ffmpeg
from facefusion import process_manager, state_manager
from facefusion.download import conditional_download
from facefusion.ffmpeg import concat_video, extract_frames, merge_video, read_audio_buffer, replace_audio, restore_audio, sanitize_audio, sanitize_image, sanitize_video, spawn_frames, spawn_stream
from facefusion.ffmpeg import concat_video, extract_frames, merge_video, read_audio_buffer, replace_audio, restore_audio, sanitize_audio, sanitize_image, sanitize_video, spawn_frames
from facefusion.ffprobe import probe_entries
from facefusion.filesystem import copy_file, is_image
from facefusion.temp_helper import clear_temp_directory, create_temp_directory, get_temp_file_path, resolve_temp_frame_paths
@@ -256,24 +255,3 @@ def test_sanitize_video() -> None:
assert probe_entries(output_paths[1], [ 'codec_name' ]).get('codec_name') == 'hevc'
def test_spawn_stream() -> None: # TODO: Improve test
test_set =\
[
((426, 240), 25, 400, 800),
((640, 360), 30, 1000, 2000),
((1280, 720), 30, 2000, 4000)
]
for resolution, stream_fps, stream_bitrate, stream_bufsize in test_set:
encoder = spawn_stream(resolution, stream_fps, stream_bitrate, stream_bufsize)
frame_size = resolution[0] * resolution[1] * 3
stdout, _ = encoder.communicate(input = bytes(frame_size))
assert len(stdout) > 32
frame_header = stdout[:32]
assert frame_header[:4] == b'DKIF'
output_width = struct.unpack_from('<H', frame_header, 12)[0]
output_height = struct.unpack_from('<H', frame_header, 14)[0]
assert (output_width, output_height) == resolution
tests/test_ffmpeg_builder.py
+1 -29
View File
@@ -1,7 +1,7 @@
from shutil import which
from facefusion import ffmpeg_builder
from facefusion.ffmpeg_builder import capture_video, chain, concat, enforce_pixel_format, keep_video_alpha, run, select_frame_range, set_audio_quality, set_audio_sample_size, set_encoder_deadline, set_lag_in_frames, set_muxer, set_stream_keyframe, set_stream_mode, set_stream_quality, set_video_bufsize, set_video_encoder, set_video_fps, set_video_quality, use_wallclock
from facefusion.ffmpeg_builder import capture_video, chain, concat, enforce_pixel_format, keep_video_alpha, run, select_frame_range, set_audio_quality, set_audio_sample_size, set_stream_mode, set_stream_quality, set_video_encoder, set_video_fps, set_video_quality
def test_run() -> None:
@@ -110,10 +110,6 @@ def test_set_video_quality() -> None:
assert set_video_quality('hevc_videotoolbox', 100) == [ '-b:v', '50512k' ]
def test_use_wallclock_timestamps() -> None:
assert use_wallclock() == [ '-use_wallclock_as_timestamps', '1' ]
def test_capture_video() -> None:
assert capture_video() == [ '-f', 'rawvideo', '-pix_fmt', 'rgb24' ]
@@ -128,27 +124,3 @@ def test_set_stream_quality() -> None:
assert set_stream_quality(2000) == [ '-b:v', '2000k' ]
def test_set_keyframe_interval() -> None:
assert set_stream_keyframe(30) == [ '-g', '30', '-keyint_min', '30' ]
assert set_stream_keyframe(60) == [ '-g', '60', '-keyint_min', '60' ]
def test_set_output_format() -> None:
assert set_muxer('ivf') == [ '-f', 'ivf' ]
assert set_muxer('mpegts') == [ '-f', 'mpegts' ]
def test_set_video_bufsize() -> None:
assert set_video_bufsize(800) == [ '-bufsize', '800k' ]
assert set_video_bufsize(4000) == [ '-bufsize', '4000k' ]
def test_set_encoder_deadline() -> None:
assert set_encoder_deadline('best') == [ '-deadline', 'best' ]
assert set_encoder_deadline('good') == [ '-deadline', 'good' ]
assert set_encoder_deadline('realtime') == [ '-deadline', 'realtime' ]
def test_set_lag_in_frames() -> None:
assert set_lag_in_frames(0) == [ '-lag-in-frames', '0' ]
assert set_lag_in_frames(16) == [ '-lag-in-frames', '16' ]
tests/test_rtc.py
+10 -9
View File
@@ -9,12 +9,13 @@ from facefusion.types import RtcPeer
@pytest.fixture(scope = 'module')
def before_all() -> None:
rtc.pre_check()
datachannel_module.pre_check()
# TODO: add test_parse_sdp_payload_types
def test_build_media_description() -> None:
assert rtc.build_media_description('audio', 111, 'opus/48000/2', 'sendonly', 1) == b'm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2\r\na=sendonly\r\na=mid:1\r\na=rtcp-mux\r\n'
assert rtc.build_media_description('video', 96, 'VP8/90000', 'recvonly', 0) == b'm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=rtpmap:96 VP8/90000\r\na=recvonly\r\na=mid:0\r\na=rtcp-mux\r\n'
assert rtc.build_media_description('audio', 111, 'opus/48000/2', 'sendonly', 1) == b'm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2\r\na=rtcp-fb:111 nack\r\na=rtcp-fb:111 nack pli\r\na=sendonly\r\na=mid:1\r\na=rtcp-mux\r\n'
assert rtc.build_media_description('video', 96, 'VP8/90000', 'recvonly', 0) == b'm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=rtpmap:96 VP8/90000\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96 nack pli\r\na=recvonly\r\na=mid:0\r\na=rtcp-mux\r\n'
# TODO: enable again
@@ -32,7 +33,7 @@ def test_create_peer_connection() -> None:
def test_add_audio_track() -> None:
peer_connection = rtc.create_peer_connection()
assert rtc.add_audio_track(peer_connection, 'sendonly') > 0
assert rtc.add_audio_track(peer_connection, 'sendonly', 111) > 0
datachannel_module.create_static_library().rtcDeletePeerConnection(peer_connection)
@@ -42,7 +43,7 @@ def test_add_audio_track() -> None:
def test_add_video_track() -> None:
peer_connection = rtc.create_peer_connection()
assert rtc.add_video_track(peer_connection, 'sendonly') > 0
assert rtc.add_video_track(peer_connection, 'sendonly', 96) > 0
datachannel_module.create_static_library().rtcDeletePeerConnection(peer_connection)
@@ -53,13 +54,13 @@ def test_negotiate_sdp() -> None:
datachannel_library = datachannel_module.create_static_library()
sender_connection = rtc.create_peer_connection()
rtc.add_video_track(sender_connection, 'sendonly')
rtc.add_audio_track(sender_connection, 'sendonly')
rtc.add_video_track(sender_connection, 'sendonly', 96)
rtc.add_audio_track(sender_connection, 'sendonly', 111)
sdp_offer = rtc.create_sdp(sender_connection)
receiver_connection = rtc.create_peer_connection()
rtc.add_video_track(receiver_connection, 'recvonly')
rtc.add_audio_track(receiver_connection, 'recvonly')
rtc.add_video_track(receiver_connection, 'recvonly', 96)
rtc.add_audio_track(receiver_connection, 'recvonly', 111)
sdp_answer = rtc.negotiate_sdp(receiver_connection, sdp_offer)
assert sdp_answer
tests/test_rtc_store.py
+18
View File
@@ -0,0 +1,18 @@
import pytest
from facefusion.libraries import datachannel as datachannel_module
@pytest.fixture(scope = 'module')
def before_all() -> None:
datachannel_module.pre_check()
# TODO: test create_rtc_stream, get_rtc_stream, destroy_rtc_stream lifecycle
def test_rtc_stream_lifecycle() -> None:
pass
# TODO: test add_rtc_viewer with valid session and sdp offer
def test_add_rtc_viewer() -> None:
pass
tests/test_stream_helper.py
-31
View File
@@ -1,31 +0,0 @@
import os
from facefusion.apis.stream_helper import calculate_bitrate, calculate_buffer_size, read_pipe_buffer
def test_calculate_bitrate() -> None:
assert calculate_bitrate((320, 240)) == 674
assert calculate_bitrate((640, 480)) == 1347
assert calculate_bitrate((1280, 720)) == 2333
assert calculate_bitrate((1920, 1080)) == 3500
assert calculate_bitrate((3840, 2160)) == 7000
def test_calculate_buffer_size() -> None:
assert calculate_buffer_size((320, 240)) == 1348
assert calculate_buffer_size((640, 480)) == 2694
assert calculate_buffer_size((1280, 720)) == 4666
assert calculate_buffer_size((1920, 1080)) == 7000
assert calculate_buffer_size((3840, 2160)) == 14000
def test_read_pipe_buffer() -> None:
read_pipe, write_pipe = os.pipe()
os.write(write_pipe, b'123456')
os.close(write_pipe)
assert read_pipe_buffer(read_pipe, 3) == b'123'
assert read_pipe_buffer(read_pipe, 3) == b'456'
assert read_pipe_buffer(read_pipe, 1) is None
os.close(read_pipe)