Implement tier 1 REMB (#1129)

* implement tier 1

* fix lint

* cleanup

* cleanup

* fix lint

* use clear_remb method

* use single callback

* improve test

* improve test

* improve test

* improve test

* improve test

tests/test_api_stream.py

@@ -1,3 +1,4 @@
+import ctypes
 import tempfile
 from typing import Iterator
 from unittest.mock import patch
@@ -135,6 +136,14 @@ def test_stream_video(test_client : TestClient, video_codec : VideoCodec) -> Non
 		assert stream_response.status_code == 201
 		assert 'm=video' in stream_response.text
 
+		session_id = session_manager.find_session_id(access_token)
+
+		for peer in rtc_store.get_peers(session_id):
+			bitrate = peer.get('bitrate')
+			assert bitrate.value == 0
+			rtc.handle_remb(0, 6000000, ctypes.addressof(bitrate))
+			assert bitrate.value == 6000
+
 
 def test_delete_stream_video(test_client : TestClient) -> None:
 	create_session_response = test_client.post('/session', json =

tests/test_rtc.py

@@ -1,11 +1,12 @@
+import ctypes
 from typing import List
 
 import pytest
 
 from facefusion import state_manager
 from facefusion.libraries import datachannel as datachannel_module, opus as opus_module, vpx as vpx_module
-from facefusion.rtc import add_audio_track, add_video_track, create_peer_connection, create_sdp_answer, create_sdp_offer, delete_peers, get_payload_type, send_audio, send_video, set_remote_description
-from facefusion.types import RtcPeer
+from facefusion.rtc import add_audio_track, add_video_track, create_peer_connection, create_sdp_answer, create_sdp_offer, delete_peers, get_payload_type, handle_remb, send_audio, send_video, set_remote_description, wire_remb
+from facefusion.types import RtcPeer, VideoCodec
 
 
 @pytest.fixture(scope = 'module', autouse = True)
@@ -77,7 +78,8 @@ def test_send_video() -> None:
 			'sender_track': video_track,
 			'receiver_track': video_track,
 			'codec': 'vp8'
-		}
+		},
+		'bitrate': ctypes.c_uint(0)
 	}
 
 	send_video(rtc_peer, bytes(1024), 0)
@@ -103,7 +105,8 @@ def test_send_audio() -> None:
 			'sender_track': audio_track,
 			'receiver_track': audio_track,
 			'codec': 'opus'
-		}
+		},
+		'bitrate': ctypes.c_uint(0)
 	}
 
 	send_audio(rtc_peer, bytes(960), 0)
@@ -123,7 +126,8 @@ def test_delete_peers() -> None:
 				'sender_track': 0,
 				'receiver_track': 0,
 				'codec': 'vp8'
-			}
+			},
+			'bitrate': ctypes.c_uint(0)
 		}
 	]
 
@@ -132,6 +136,31 @@ def test_delete_peers() -> None:
 	assert datachannel_library.rtcDeletePeerConnection(peer_connection) == -1
 
 
+@pytest.mark.parametrize('video_codec, payload_type', [ ('av1', 35), ('vp8', 96) ])
+def test_wire_remb(video_codec : VideoCodec, payload_type : int) -> None:
+	datachannel_library = datachannel_module.create_static_library()
+	peer_connection = create_peer_connection()
+	video_sender_track = add_video_track(peer_connection, 'sendonly', video_codec, payload_type)
+	rtc_peer : RtcPeer =\
+	{
+		'peer_connection': peer_connection,
+		'video':
+		{
+			'sender_track': video_sender_track,
+			'receiver_track': video_sender_track,
+			'codec': video_codec
+		},
+		'bitrate': ctypes.c_uint(0)
+	}
+
+	wire_remb(video_sender_track, rtc_peer.get('bitrate'))
+	handle_remb(0, 6000000, ctypes.addressof(rtc_peer.get('bitrate')))
+
+	assert rtc_peer.get('bitrate').value == 6000
+
+	datachannel_library.rtcDeletePeerConnection(peer_connection)
+
+
 def test_get_payload_type() -> None:
 	peer_connection = create_peer_connection()
 	add_video_track(peer_connection, 'sendonly', 'vp8', 96)

tests/test_stream_helper.py

@@ -1,3 +1,4 @@
+import ctypes
 import queue
 import threading
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -10,7 +11,7 @@ from tests.assert_helper import get_test_example_file, get_test_examples_directo
 
 from facefusion import rtc, rtc_store, state_manager
 from facefusion.apis.endpoints.stream import websocket_stream
-from facefusion.apis.stream_helper import decode_video_frame, process_image, process_video, receive_audio_frames, receive_video_frames, receive_vision_frames, run_peer_loop
+from facefusion.apis.stream_helper import create_video_encoder, decode_video_frame, destroy_video_encoder, process_image, process_video, receive_audio_frames, receive_video_frames, receive_vision_frames, run_peer_loop
 from facefusion.codecs import aom_decoder, aom_encoder, vpx_decoder, vpx_encoder
 from facefusion.common_helper import is_linux, is_macos, is_windows
 from facefusion.download import conditional_download
@@ -94,6 +95,30 @@ def test_receive_video_frames() -> None:
 		assert create_hash(video_queue.get_nowait().tobytes()) == '38d00e2a'
 
 
+@pytest.mark.parametrize('video_codec', [ 'av1', 'vp8' ])
+def test_create_and_destroy_video_encoder(video_codec : VideoCodec) -> None:
+	vision_frame = read_video_frame(get_test_example_file('target-240p.mp4'))
+	resolution = (vision_frame.shape[1], vision_frame.shape[0])
+	frame_buffer = cv2.cvtColor(vision_frame, cv2.COLOR_BGR2YUV_I420).tobytes()
+
+	encoder = create_video_encoder(video_codec, resolution, 4000)
+	assert encoder is not None
+
+	if video_codec == 'av1':
+		assert aom_encoder.encode(encoder, frame_buffer, resolution, 0)
+
+	if video_codec == 'vp8':
+		assert vpx_encoder.encode(encoder, frame_buffer, resolution, 0)
+
+	destroy_video_encoder(video_codec, encoder)
+
+	if video_codec == 'av1':
+		assert not aom_encoder.encode(encoder, frame_buffer, resolution, 1)
+
+	if video_codec == 'vp8':
+		assert not vpx_encoder.encode(encoder, frame_buffer, resolution, 1)
+
+
 # TODO: refine test
 def test_receive_audio_frames() -> None:
 	audio_frame = numpy.zeros(960 * 2, dtype = numpy.float32)
@@ -128,7 +153,8 @@ def test_run_peer_loop() -> None:
 			'sender_track': video_sender_track,
 			'receiver_track': video_receiver_track,
 			'codec': 'vp8'
-		}
+		},
+		'bitrate': ctypes.c_uint(0)
 	}
 
 	session_id = 'test-run-peer-loop'