remove opus frame size (#1160)

* remove opus frame size

* improve naming, follow audio.py

facefusion/apis/stream_audio.py

@@ -16,8 +16,7 @@ def run_audio_encode_loop(rtc_peer : RtcPeer, audio_queue : Queue[Tuple[Time, Au
 	audio_encoder = opus_encoder.create(48000, 2)
 
 	while numpy.any(temp_audio_frame):
-		audio_frame_size = len(temp_audio_frame) // 2
-		audio_buffer = opus_encoder.encode(audio_encoder, temp_audio_frame.tobytes(), audio_frame_size)
+		audio_buffer = opus_encoder.encode(audio_encoder, temp_audio_frame.tobytes(), 2)
 
 		if audio_buffer:
 			audio_timestamp = rtc.convert_time_to_timestamp(audio_codec, temp_audio_time)
@@ -44,7 +43,7 @@ def receive_audio_frames(rtc_peer_audio : RtcPeerAudio, audio_queue : Queue[Tupl
 
 def decode_audio_frame(audio_codec : AudioCodec, audio_decoder : OpusDecoder, input_buffer : Buffer) -> Optional[Buffer]:
 	if audio_codec == 'opus':
-		return opus_decoder.decode(audio_decoder, input_buffer, 960, 2)
+		return opus_decoder.decode(audio_decoder, input_buffer, 2)
 	return None
 
 

facefusion/codecs/opus_decoder.py

@@ -14,17 +14,19 @@ def create(sample_rate : int, channel_total : int) -> Optional[OpusDecoder]:
 	return None
 
 
-def decode(opus_decoder : OpusDecoder, input_buffer : Buffer, frame_size : int, channel_total : int) -> Buffer:
+def decode(opus_decoder : OpusDecoder, input_buffer : Buffer, channel_total : int) -> Buffer:
 	opus_library = opus_module.create_static_library()
 	output_buffer = bytes()
 
 	if opus_library:
 		input_total = len(input_buffer)
-		decode_buffer = (ctypes.c_float * (frame_size * channel_total))()
-		decode_length = opus_library.opus_decode_float(opus_decoder, input_buffer, input_total, decode_buffer, frame_size, 0)
+		sample_size = ctypes.sizeof(ctypes.c_float)
+		sample_total = opus_library.opus_decoder_get_nb_samples(opus_decoder, input_buffer, input_total)
+		sample_buffer = (ctypes.c_float * (sample_total * channel_total))()
+		output_total = opus_library.opus_decode_float(opus_decoder, input_buffer, input_total, sample_buffer, sample_total, 0)
 
-		if decode_length:
-			output_buffer = ctypes.string_at(ctypes.addressof(decode_buffer), decode_length * channel_total * ctypes.sizeof(ctypes.c_float))
+		if output_total:
+			output_buffer = ctypes.string_at(ctypes.addressof(sample_buffer), output_total * channel_total * sample_size)
 
 	return output_buffer
 

facefusion/codecs/opus_encoder.py

@@ -14,17 +14,19 @@ def create(sample_rate : int, channel_total : int) -> Optional[OpusEncoder]:
 	return None
 
 
-def encode(opus_encoder : OpusEncoder, input_buffer : Buffer, frame_size : int) -> Buffer:
+def encode(opus_encoder : OpusEncoder, input_buffer : Buffer, channel_total : int) -> Buffer:
 	opus_library = opus_module.create_static_library()
 	output_buffer = bytes()
 
 	if opus_library:
+		sample_size = ctypes.sizeof(ctypes.c_float)
+		sample_total = len(input_buffer) // (sample_size * channel_total)
+		sample_buffer = (ctypes.c_float * (sample_total * channel_total)).from_buffer_copy(input_buffer)
 		temp_buffer = ctypes.create_string_buffer(2048)
-		encode_buffer = ctypes.cast(ctypes.create_string_buffer(input_buffer), ctypes.POINTER(ctypes.c_float))
-		encode_length = opus_library.opus_encode_float(opus_encoder, encode_buffer, frame_size, temp_buffer, 2048)
+		output_total = opus_library.opus_encode_float(opus_encoder, sample_buffer, sample_total, temp_buffer, 2048)
 
-		if encode_length:
-			output_buffer = temp_buffer.raw[:encode_length]
+		if output_total:
+			output_buffer = temp_buffer.raw[:output_total]
 
 	return output_buffer
 

facefusion/libraries/opus.py

@@ -115,6 +115,9 @@ def init_ctypes(library : ctypes.CDLL) -> ctypes.CDLL:
 	library.opus_decode_float.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_int, ctypes.c_int ]
 	library.opus_decode_float.restype = ctypes.c_int
 
+	library.opus_decoder_get_nb_samples.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int ]
+	library.opus_decoder_get_nb_samples.restype = ctypes.c_int
+
 	library.opus_decoder_destroy.argtypes = [ ctypes.c_void_p ]
 	library.opus_decoder_destroy.restype = None
 

facefusion/workflows/core.py

@@ -54,7 +54,7 @@ def conditional_get_source_audio_frame(frame_number : int) -> AudioFrame:
 	return create_empty_audio_frame()
 
 
-def conditional_get_source_voice_frame(frame_number: int) -> AudioFrame:
+def conditional_get_source_voice_frame(frame_number : int) -> AudioFrame:
 	if state_manager.get_item('workflow') in [ 'audio-to-image:frames', 'audio-to-image:video', 'image-to-video' ]:
 		source_audio_path = get_first(filter_audio_paths(state_manager.get_item('source_paths')))
 		temp_fps = state_manager.get_item('output_audio_fps')

tests/test_codec_opus_decoder.py

@@ -33,15 +33,16 @@ def test_create() -> None:
 def test_decode() -> None:
 	audio_buffer = read_audio_buffer(get_test_example_file('source.mp3'), 48000, 16, 2)
 	audio_sample = numpy.frombuffer(audio_buffer, dtype = numpy.int16).astype(numpy.float32) / 32768.0
+	audio_frame = audio_sample.reshape(-1, 2)[:960]
 	opus_encoder = create_encoder(48000, 2)
-	encode_buffer = encode(opus_encoder, audio_sample.tobytes(), 960)
+	encode_buffer = encode(opus_encoder, audio_frame.tobytes(), 2)
 	opus_decoder = create(48000, 2)
 
 	if is_linux() or is_windows():
-		assert create_hash(decode(opus_decoder, encode_buffer, 960, 2)) == 'cadd63d1'
+		assert create_hash(decode(opus_decoder, encode_buffer, 2)) == 'cadd63d1'
 
 	if is_macos():
-		assert create_hash(decode(opus_decoder, encode_buffer, 960, 2)) == '92f7997d'
+		assert create_hash(decode(opus_decoder, encode_buffer, 2)) == '92f7997d'
 
 
 def test_destroy() -> None:

tests/test_codec_opus_encoder.py

@@ -32,10 +32,11 @@ def test_create() -> None:
 def test_encode() -> None:
 	audio_buffer = read_audio_buffer(get_test_example_file('source.mp3'), 48000, 16, 2)
 	audio_sample = numpy.frombuffer(audio_buffer, dtype = numpy.int16).astype(numpy.float32) / 32768.0
+	audio_frame = audio_sample.reshape(-1, 2)[:960]
 	opus_encoder = create(48000, 2)
 
 	if is_linux() or is_windows():
-		assert create_hash(encode(opus_encoder, audio_sample.tobytes(), 960)) == '8abe71cf'
+		assert create_hash(encode(opus_encoder, audio_frame.tobytes(), 2)) == '8abe71cf'
 
 	if is_macos():
 		pytest.skip()