Voice processor: add support for webrtc-audio-processing 1.x and 2.x

4 files changed, 202 insertions, 34 deletions

diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt
index 83df5df8..166229b3 100644
--- a/plugins/rtp/CMakeLists.txt
+++ b/plugins/rtp/CMakeLists.txt
@@ -52,16 +52,25 @@ if(RTP_ENABLE_MSDK)
     set(RTP_DEFINITIONS ${RTP_DEFINITIONS} ENABLE_MSDK)
 endif()
 
-if(WebRTCAudioProcessing_VERSION GREATER "0.4")
-    message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far")
+if(WebRTCAudioProcessing_VERSION GREATER_EQUAL "3.0")
+    message(STATUS "Ignoring WebRTCAudioProcessing, only versions 0.2+, 1.* and 2.* supported so far")
     unset(WebRTCAudioProcessing_FOUND)
 endif()
 
 if(WebRTCAudioProcessing_FOUND)
     set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR)
+    if(WebRTCAudioProcessing_VERSION GREATER_EQUAL "2.0")
+        set(RTP_VOICE_PROCESSOR_CFLAGS "-DWEBRTC2")
+        set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing-2)
+    elseif(WebRTCAudioProcessing_VERSION GREATER_EQUAL "1.0")
+        set(RTP_VOICE_PROCESSOR_CFLAGS "-DWEBRTC1")
+        set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing-1)
+    else()
+        set(RTP_VOICE_PROCESSOR_CFLAGS "-DWEBRTC0")
+        set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing)
+    endif()
     set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala)
     set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp)
-    set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing)
 else()
     message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!")
 endif()
@@ -89,7 +98,7 @@ OPTIONS
     ${RTP_EXTRA_OPTIONS}
 )
 
-add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src)
+add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src ${RTP_VOICE_PROCESSOR_CFLAGS})
 add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX} src/gst_fixes.c)
 target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB})
 set_target_properties(rtp PROPERTIES PREFIX "")
diff --git a/plugins/rtp/meson.build b/plugins/rtp/meson.build
index a2bae51b..c70a414b 100644
--- a/plugins/rtp/meson.build
+++ b/plugins/rtp/meson.build
@@ -28,7 +28,7 @@ c_args = [
     '-DG_LOG_DOMAIN="rtp"',
 ]
 vala_args = []
-if dep_webrtc_audio_processing.found()
+if dep_webrtc_audio_processing.found() and get_option('plugin-rtp-webrtc-audio-processing').allowed()
     voice_dependencies = [
         dep_gstreamer_audio,
         dep_webrtc_audio_processing,
@@ -36,13 +36,23 @@ if dep_webrtc_audio_processing.found()
     voice_sources = files(
         'src/voice_processor_native.cpp',
     )
-    lib_rtp_voice_processor = static_library('rtp-voice-processor', voice_sources, c_args: c_args, dependencies: voice_dependencies, install: false)
+    voice_c_args = c_args
+    if dep_webrtc_audio_processing.version().version_compare('>=2.0')
+        voice_c_args += ['-DWEBRTC2']
+    elif dep_webrtc_audio_processing.version().version_compare('>=1.0')
+        voice_c_args += ['-DWEBRTC1']
+    else
+        voice_c_args += ['-DWEBRTC0']
+    endif
+    lib_rtp_voice_processor = static_library('rtp-voice-processor', voice_sources, c_args: voice_c_args, cpp_args: voice_c_args, dependencies: voice_dependencies, install: false)
     dep_rtp_voice_processor = declare_dependency(link_with: lib_rtp_voice_processor)
     vala_args += ['-D', 'WITH_VOICE_PROCESSOR']
     dependencies += [dep_rtp_voice_processor]
     sources += files(
         'src/voice_processor.vala',
     )
+elif get_option('plugin-rtp-webrtc-audio-processing').enabled()
+    error('No compatible webrtc-audio-processing found, but plugin-rtp-webrtc-audio-processing option set.')
 endif
 if dep_gstreamer_rtp.version() == 'unknown' or dep_gstreamer_rtp.version().version_compare('>=1.16')
     vala_args += ['-D', 'GST_1_16']
diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala
index 66e95d72..a522a6ce 100644
--- a/plugins/rtp/src/voice_processor.vala
+++ b/plugins/rtp/src/voice_processor.vala
@@ -8,9 +8,11 @@ public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter {
     private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
     private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
     public Audio.Info audio_info { get; private set; }
-    public signal void on_new_buffer(Buffer buffer);
+    public signal void on_new_buffer(owned Buffer buffer);
+    public signal void on_new_delay(int delay);
     private uint period_samples;
     private uint period_size;
+    public int delay { get; private set; }
     private Base.Adapter adapter = new Base.Adapter();
 
     static construct {
@@ -30,6 +32,25 @@ public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter {
         return true;
     }
 
+    public override bool src_event(owned Event event) {
+        Query query = new Query.latency();
+        if (event.type == EventType.LATENCY && srcpad != null && srcpad.query(query)) {
+            ClockTime upstream_latency;
+            query.parse_latency(null, out upstream_latency, null);
+            int delay = this.delay;
+            if (upstream_latency != CLOCK_TIME_NONE) {
+                delay = (int) (upstream_latency / MSECOND);
+            } else {
+                delay = 0;
+            }
+            if (delay != this.delay) {
+                debug("Delay adjusted from %ms to %dms", this.delay, delay);
+                this.delay = delay;
+                on_new_delay(delay);
+            }
+        }
+        return base.src_event((owned) event);
+    }
 
     public override FlowReturn transform_ip(Buffer buf) {
         lock (adapter) {
@@ -52,6 +73,7 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
     private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
     public Audio.Info audio_info { get; private set; }
     private ulong process_outgoing_buffer_handler_id;
+    private ulong process_stream_delay_handler_id;
     private uint adjust_delay_timeout_id;
     private uint period_samples;
     private uint period_size;
@@ -82,6 +104,7 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
     private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer);
     private static extern void process_stream(void* native, Audio.Info info, Buffer buffer);
     private static extern void adjust_stream_delay(void* native);
+    private static extern void set_stream_delay(void* native, int delay);
     private static extern void notify_gain_level(void* native, int gain_level);
     private static extern int get_suggested_gain_level(void* native);
     private static extern bool get_stream_has_voice(void* native);
@@ -97,10 +120,13 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
     }
 
     public override bool start() {
-        native = init_native(150);
+        native = init_native(echo_probe.delay);
         if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) {
             process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer);
         }
+        if (process_stream_delay_handler_id == 0 && echo_probe != null) {
+            process_stream_delay_handler_id = echo_probe.on_new_delay.connect(process_stream_delay);
+        }
         if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) {
             stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume;
         }
@@ -117,16 +143,25 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
         }
     }
 
-    private void process_outgoing_buffer(Buffer buffer) {
+    private void process_outgoing_buffer(owned Buffer buffer) {
         if (buffer.pts != uint64.MAX) {
             last_reverse = buffer.pts;
         }
-        analyze_reverse_stream(native, echo_probe.audio_info, buffer);
+        if (native != null) {
+            buffer = (Buffer) buffer.make_writable();
+            analyze_reverse_stream(native, echo_probe.audio_info, buffer);
+        }
         if (adjust_delay_timeout_id == 0 && echo_probe != null) {
             adjust_delay_timeout_id = Timeout.add(1000, adjust_delay);
         }
     }
 
+    private void process_stream_delay(int stream_delay) {
+        if (native != null) {
+            set_stream_delay(native, stream_delay);
+        }
+    }
+
     public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) {
         lock (adapter) {
             if (is_discont) {
diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp
index 8a052cf8..e41feb49 100644
--- a/plugins/rtp/src/voice_processor_native.cpp
+++ b/plugins/rtp/src/voice_processor_native.cpp
@@ -1,18 +1,28 @@
 #include <algorithm>
 #include <gst/gst.h>
 #include <gst/audio/audio.h>
+#if defined(WEBRTC0)
 #include <webrtc/modules/audio_processing/include/audio_processing.h>
 #include <webrtc/modules/interface/module_common_types.h>
 #include <webrtc/system_wrappers/include/trace.h>
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+#include <modules/audio_processing/include/audio_processing.h>
+#else
+#error "Need to define WEBRTC0, WEBRTC1 or WEBRTC2"
+#endif
 
 #define SAMPLE_RATE 48000
 #define SAMPLE_CHANNELS 1
 
 struct _DinoPluginsRtpVoiceProcessorNative {
+#if defined(WEBRTC0)
     webrtc::AudioProcessing *apm;
-    gint stream_delay;
-    gint last_median;
-    gint last_poor_delays;
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm;
+#endif
+    gint stream_delay = 0;
+    gint last_median = 0;
+    gint last_poor_delays = 0;
 };
 
 extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) {
@@ -22,19 +32,20 @@ extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *trans
 }
 
 extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) {
-    _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative();
+    auto *native = new _DinoPluginsRtpVoiceProcessorNative();
     webrtc::Config config;
+#ifdef WEBRTC0
     config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true));
     config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85));
     native->apm = webrtc::AudioProcessing::Create(config);
+#endif
     native->stream_delay = stream_delay;
-    native->last_median = 0;
-    native->last_poor_delays = 0;
     return native;
 }
 
 extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+#if defined(WEBRTC0)
     webrtc::AudioProcessing *apm = native->apm;
     webrtc::ProcessingConfig pconfig;
     pconfig.streams[webrtc::ProcessingConfig::kInputStream] =
@@ -61,14 +72,36 @@ extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr)
     apm->gain_control()->Enable(true);
     apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood);
     apm->voice_detection()->Enable(true);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    webrtc::AudioProcessing::Config config;
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create();
+    native->apm = apm;
+    config.high_pass_filter.enabled = true;
+    config.echo_canceller.enabled = true;
+    config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::Level::kModerate;
+    config.noise_suppression.enabled = true;
+    config.gain_controller1.analog_level_minimum = 0;
+    config.gain_controller1.analog_level_maximum = 255;
+    config.gain_controller1.target_level_dbfs = 3;
+    config.gain_controller1.compression_gain_db = 9;
+    config.gain_controller1.enable_limiter = true;
+    config.gain_controller1.enabled = true;
+#ifdef WEBRTC1
+    config.level_estimation.enabled = true;
+    config.voice_detection.enabled = true;
+#endif
+    apm->ApplyConfig(config);
+#endif
 }
 
 extern "C" void
 dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    int err;
+
+#if defined(WEBRTC0)
     webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
     webrtc::AudioProcessing *apm = native->apm;
-
     GstMapInfo map;
     gst_buffer_map(buffer, &map, GST_MAP_READ);
 
@@ -78,53 +111,111 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud
     frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf;
     memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
 
-    int err = apm->AnalyzeReverseStream(&frame);
-    if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err);
+    apm->set_stream_delay_ms (native->stream_delay);
+    err = apm->AnalyzeReverseStream(&frame);
 
     gst_buffer_unmap(buffer, &map);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+#ifdef WEBRTC1
+    webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+#else
+    webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS);
+#endif
+    GstAudioBuffer abuf;
+    if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) {
+        g_warning("voice_processor_native.cpp: analyze_reverse_stream: gst_audio_buffer_map failed");
+        return;
+    }
+
+    apm->set_stream_delay_ms (native->stream_delay);
+    auto * const data = (int16_t * const) abuf.planes[0];
+    err = apm->ProcessReverseStream (data, config, config, data);
+
+    gst_audio_buffer_unmap (&abuf);
+#endif
+    if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err);
 }
 
 extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+#if defined(WEBRTC0)
     webrtc::AudioProcessing *apm = native->apm;
     apm->gain_control()->set_stream_analog_level(gain_level);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+    apm->set_stream_analog_level(gain_level);
+#endif
 }
 
 extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    int level = 0;
+#if defined(WEBRTC0)
     webrtc::AudioProcessing *apm = native->apm;
-    return apm->gain_control()->stream_analog_level();
+    level = apm->gain_control()->stream_analog_level();
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+    level = apm->recommended_stream_analog_level();
+#endif
+    return level;
 }
 
 extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    bool has_voice = false;
+#if defined(WEBRTC0)
     webrtc::AudioProcessing *apm = native->apm;
-    return apm->voice_detection()->stream_has_voice();
+    has_voice = apm->voice_detection()->stream_has_voice();
+#elif defined(WEBRTC1)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+    webrtc::AudioProcessingStats stats = apm->GetStatistics ();
+    has_voice = stats.voice_detected.value_or(false);
+#endif
+    return has_voice;
+}
+
+extern "C" void dino_plugins_rtp_voice_processor_set_stream_delay(void *native_ptr, gint stream_delay) {
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    native->stream_delay = stream_delay;
 }
 
 extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
-    webrtc::AudioProcessing *apm = native->apm;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
     int median, std, poor_delays;
     float fraction_poor_delays;
+#if defined(WEBRTC0)
+    webrtc::AudioProcessing *apm = native->apm;
     apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays);
     poor_delays = (int)(fraction_poor_delays * 100.0);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+    webrtc::AudioProcessingStats stats = apm->GetStatistics();
+    median = stats.delay_median_ms.value_or(-1);
+    std = stats.delay_standard_deviation_ms.value_or(-1);
+    fraction_poor_delays = (float) stats.divergent_filter_fraction.value_or(-1.0);
+    poor_delays = (int) (fraction_poor_delays * 100.0);
+#endif
     if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return;
     g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays);
     native->last_median = median;
     native->last_poor_delays = poor_delays;
+#if defined(WEBRTC0)
     if (poor_delays > 90) {
         native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384);
         g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay);
     }
+#endif
 }
 
 extern "C" void
 dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    int err;
+
+#if defined(WEBRTC0)
     webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
     webrtc::AudioProcessing *apm = native->apm;
-
     GstMapInfo map;
     gst_buffer_map(buffer, &map, GST_MAP_READWRITE);
 
@@ -134,15 +225,38 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *
     frame.samples_per_channel_ = info->rate / 100;
     memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
 
-    apm->set_stream_delay_ms(native->stream_delay);
-    int err = apm->ProcessStream(&frame);
+    err = apm->ProcessStream(&frame);
     if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf);
-    if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err);
 
     gst_buffer_unmap(buffer, &map);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+#ifdef WEBRTC1
+    webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+#else
+    webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS);
+#endif
+    GstAudioBuffer abuf;
+    if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) {
+        g_warning("voice_processor_native.cpp: process_stream: gst_audio_buffer_map failed");
+        return;
+    }
+
+    auto * const data = (int16_t * const) abuf.planes[0];
+    err = apm->ProcessStream (data, config, config, data);
+
+    gst_audio_buffer_unmap (&abuf);
+#endif
+    if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err);
 }
 
 extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) {
-    _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+    auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+#if defined(WEBRTC0)
+    delete native->apm;
+    native->apm = NULL;
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+    native->apm = nullptr;
+#endif
     delete native;
 }
\ No newline at end of file