From dba577f39c489b9db29960f2bf5cc78ac4f29d7b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 4 Oct 2026 14:42:16 +0200 Subject: Move OMEMO, OpenPGP and RTP plugins into main/libdino --- plugins/rtp/src/codec_util.vala | 418 -------------- plugins/rtp/src/device.vala | 592 -------------------- plugins/rtp/src/gst_fixes.c | 10 - plugins/rtp/src/module.vala | 235 -------- plugins/rtp/src/participant.vala | 39 -- plugins/rtp/src/plugin.vala | 504 ----------------- plugins/rtp/src/register_plugin.vala | 3 - plugins/rtp/src/stream.vala | 851 ----------------------------- plugins/rtp/src/video_widget.vala | 293 ---------- plugins/rtp/src/voice_processor.vala | 209 ------- plugins/rtp/src/voice_processor_native.cpp | 256 --------- 11 files changed, 3410 deletions(-) delete mode 100644 plugins/rtp/src/codec_util.vala delete mode 100644 plugins/rtp/src/device.vala delete mode 100644 plugins/rtp/src/gst_fixes.c delete mode 100644 plugins/rtp/src/module.vala delete mode 100644 plugins/rtp/src/participant.vala delete mode 100644 plugins/rtp/src/plugin.vala delete mode 100644 plugins/rtp/src/register_plugin.vala delete mode 100644 plugins/rtp/src/stream.vala delete mode 100644 plugins/rtp/src/video_widget.vala delete mode 100644 plugins/rtp/src/voice_processor.vala delete mode 100644 plugins/rtp/src/voice_processor_native.cpp (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala deleted file mode 100644 index 9f21f5a6..00000000 --- a/plugins/rtp/src/codec_util.vala +++ /dev/null @@ -1,418 +0,0 @@ -using Gee; -using Xmpp; -using Xmpp.Xep; - -public class Dino.Plugins.Rtp.CodecUtil { - private Set supported_elements = new HashSet(); - private Set unsupported_elements = new HashSet(); - - public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) { - Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", - "media", typeof(string), media, - "payload", typeof(int), payload_type.id); - //"channels", typeof(int), payloadType.channels, - //"max-ptime", typeof(int), payloadType.maxptime); - unowned Gst.Structure s = caps.get_structure(0); - if (payload_type.clockrate != 0) { - s.set("clock-rate", typeof(int), payload_type.clockrate); - } - if (payload_type.name != null) { - s.set("encoding-name", typeof(string), payload_type.name.up()); - } - if (incoming) { - foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) { - if (rtcp_fb.subtype == null) { - s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true); - } else { - s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true); - } - } - } - return caps; - } - - public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) { - if (payload_type.name != null) return payload_type.name.down(); - if (media == "audio") { - switch (payload_type.id) { - case 0: - return "pcmu"; - case 8: - return "pcma"; - } - } - return null; - } - - public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) { - return get_media_type(media, get_codec_from_payload(media, payload_type)); - } - - public static string? get_media_type(string media, string? codec) { - if (codec == null) return null; - if (media == "audio") { - switch (codec) { - case "pcma": - return "audio/x-alaw"; - case "pcmu": - return "audio/x-mulaw"; - } - } - return @"$media/x-$codec"; - } - - public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { - return get_pay_candidate(media, get_codec_from_payload(media, payload_type)); - } - - public static string? get_pay_candidate(string media, string? codec) { - if (codec == null) return null; - return @"rtp$(codec)pay"; - } - - public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { - return get_depay_candidate(media, get_codec_from_payload(media, payload_type)); - } - - public static string? get_depay_candidate(string media, string? codec) { - if (codec == null) return null; - return @"rtp$(codec)depay"; - } - - public static string[] get_encode_candidates(string media, string? codec) { - if (codec == null) return new string[0]; - if (media == "audio") { - switch (codec) { - case "opus": - return new string[] {"opusenc"}; - case "speex": - return new string[] {"speexenc"}; - case "pcma": - return new string[] {"alawenc"}; - case "pcmu": - return new string[] {"mulawenc"}; - case "g722": - return new string[] {"avenc_g722"}; - } - } else if (media == "video") { - switch (codec) { - case "h264": - return new string[] { -#if ENABLE_MSDK - "msdkh264enc", -#endif -#if ENABLE_VAAPI - "vaapih264enc", -#endif - "x264enc" - }; - case "vp9": - return new string[] { -#if ENABLE_MSDK - "msdkvp9enc", -#endif -#if ENABLE_VAAPI - "vaapivp9enc", -#endif - "vp9enc" - }; - case "vp8": - return new string[] { -#if ENABLE_MSDK - "msdkvp8enc", -#endif -#if ENABLE_VAAPI - "vaapivp8enc", -#endif - "vp8enc" - }; - } - } - return new string[0]; - } - - public static string[] get_decode_candidates(string media, string? codec) { - if (codec == null) return new string[0]; - if (media == "audio") { - switch (codec) { - case "opus": - return new string[] {"opusdec"}; - case "speex": - return new string[] {"speexdec"}; - case "pcma": - return new string[] {"alawdec"}; - case "pcmu": - return new string[] {"mulawdec"}; - case "g722": - return new string[] {"avdec_g722"}; - } - } else if (media == "video") { - switch (codec) { - case "h264": - return new string[] { -#if ENABLE_MSDK - "msdkh264dec", -#endif -#if ENABLE_VAAPI - "vaapih264dec", -#endif - null - }; - case "vp9": - return new string[] { -#if ENABLE_MSDK - "msdkvp9dec", -#endif -#if ENABLE_VAAPI - "vaapivp9dec", -#endif - "vp9dec" - }; - case "vp8": - return new string[] { -#if ENABLE_MSDK - "msdkvp8dec", -#endif -#if ENABLE_VAAPI - "vaapivp8dec", -#endif - "vp8dec" - }; - } - } - return new string[0]; - } - - public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { - if (encode == "msdkh264enc") return "capsfilter caps=video/x-raw,format=NV12 ! "; - if (encode == "vaapih264enc") return "capsfilter caps=video/x-raw,format=NV12 ! "; - return null; - } - - public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { - // H264 - if (encode == "msdkh264enc") return @" rate-control=vbr"; - if (encode == "vaapih264enc") return @" rate-control=vbr"; - if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency bframes=0 cabac=false dct8x8=false"; - - // VP8 - if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; - if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; - - // VP9 - if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; - if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; - - // OPUS - if (encode == "opusenc") { - if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true"; - return " audio-type=voice"; - } - - return null; - } - - public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { - // H264 - if (media == "video" && codec == "h264") return " ! capsfilter caps=video/x-h264,profile=constrained-baseline ! h264parse"; - if (media == "video" && codec == "vp8" && encode == "vp8enc") return " ! capsfilter caps=video/x-vp8,profile=(string)1"; - return null; - } - - public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) { - Gst.Bin? encode_bin = encode_element as Gst.Bin; - if (encode_bin == null) return 0; - string? codec = get_codec_from_payload(media, payload_type); - string? encode_name = get_encode_element_name(media, codec); - if (encode_name == null) return 0; - Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); - - switch (encode_name) { - case "msdkh264enc": - case "vaapih264enc": - case "x264enc": - case "msdkvp9enc": - case "vaapivp9enc": - case "msdkvp8enc": - case "vaapivp8enc": - bitrate = uint.min(2048000, bitrate); - encode.set("bitrate", bitrate); - return bitrate; - case "vp9enc": - case "vp8enc": - bitrate = uint.min(2147483, bitrate); - encode.set("target-bitrate", bitrate * 1024); - return bitrate; - } - - return 0; - } - - public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) { - Gst.Bin? encode_bin = encode_element as Gst.Bin; - if (encode_bin == null) return; - Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); - rescale_caps.set("caps", caps); - } - - public Gst.Caps? get_rescale_caps(Gst.Element encode_element) { - Gst.Bin? encode_bin = encode_element as Gst.Bin; - if (encode_bin == null) return null; - Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); - Gst.Caps caps; - rescale_caps.get("caps", out caps); - return caps; - } - - public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { - return null; - } - - public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { - if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; - if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; - if (decode == "vp8dec" || decode == "vp9dec") return " threads=8"; - return null; - } - - public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { - return null; - } - - public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { - if (codec == "vp8") return " wait-for-keyframe=true"; - return null; - } - - public bool is_element_supported(string? element_name) { - if (element_name == null) return false; - if (unsupported_elements.contains(element_name)) return false; - if (supported_elements.contains(element_name)) return true; - var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name"); - if (test_element != null) { - supported_elements.add(element_name); - return true; - } else { - warning("%s is not installed or supported on this system", element_name); - unsupported_elements.add(element_name); - return false; - } - } - - public string? get_encode_element_name(string media, string? codec) { - if (get_pay_element_name(media, codec) == null) return null; - foreach (string candidate in get_encode_candidates(media, codec)) { - if (is_element_supported(candidate)) return candidate; - } - return null; - } - - public string? get_pay_element_name(string media, string? codec) { - string? candidate = get_pay_candidate(media, codec); - if (candidate != null && is_element_supported(candidate)) return candidate; - return null; - } - - public string? get_decode_element_name(string media, string? codec) { - if (get_depay_element_name(media, codec) == null) return null; - foreach (string candidate in get_decode_candidates(media, codec)) { - if (is_element_supported(candidate)) return candidate; - } - return null; - } - - public string? get_depay_element_name(string media, string? codec) { - string? candidate = get_depay_candidate(media, codec); - if (candidate != null && is_element_supported(candidate)) return candidate; - return null; - } - - public void mark_element_unsupported(string element_name) { - unsupported_elements.add(element_name); - } - - public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { - if (codec == null) return null; - string base_name = name ?? @"encode-$codec-$(Random.next_int())"; - string? depay = get_depay_element_name(media, codec); - string? decode = element_name ?? get_decode_element_name(media, codec); - if (depay == null || decode == null) return null; - string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? ""; - string decode_args = get_decode_args(media, codec, decode, payload_type) ?? ""; - string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; - string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"queue ! $depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; - } - - public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { - string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"decode_$(codec)_$(Random.next_int())"; - string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name); - if (desc == null) return null; - debug("Pipeline to decode %s %s: %s", media, codec, desc); - Gst.Element bin = Gst.parse_bin_from_description(desc, true); - bin.name = name; - return bin; - } - - public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { - string? desc1 = get_encode_bin_without_payloader_description(media, codec, payload_type, element_name, name); - string? desc2 = get_payloader_bin_description(media, codec, payload_type, name); - return @"$desc1 ! $desc2"; - } - - public string? get_payloader_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? name = null) { - if (codec == null) return null; - string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; - string? pay = get_pay_element_name(media, codec); - if (pay == null) return null; - return @"$pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; - } - - public string? get_encode_bin_without_payloader_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { - if (codec == null) return null; - string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; - string? encode = element_name ?? get_encode_element_name(media, codec); - if (encode == null) return null; - string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; - string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; - string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; - string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps"; - return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; - } - - public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { - string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; - string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name); - if (desc == null) return null; - debug("Pipeline to encode %s %s: %s", media, codec, desc); - Gst.Element bin = Gst.parse_bin_from_description(desc, true); - bin.name = name; - return bin; - } - - public Gst.Element? get_encode_bin_without_payloader(string media, JingleRtp.PayloadType payload_type, string? name = null) { - string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; - string? desc = get_encode_bin_without_payloader_description(media, codec, payload_type, null, base_name); - if (desc == null) return null; - debug("Pipeline to encode %s %s without payloader: %s", media, codec, desc); - Gst.Element bin = Gst.parse_bin_from_description(desc, true); - bin.name = name; - return bin; - } - - public Gst.Element? get_payloader_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { - string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; - string? desc = get_payloader_bin_description(media, codec, payload_type, base_name); - if (desc == null) return null; - debug("Pipeline to payload %s %s: %s", media, codec, desc); - Gst.Element bin = Gst.parse_bin_from_description(desc, true); - bin.name = name; - return bin; - } - -} diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala deleted file mode 100644 index bdb597b6..00000000 --- a/plugins/rtp/src/device.vala +++ /dev/null @@ -1,592 +0,0 @@ -using Xmpp.Xep.JingleRtp; -using Gee; - -public enum Dino.Plugins.Rtp.DeviceProtocol { - OTHER, - PIPEWIRE, - V4L2, - PULSEAUDIO, - ALSA -} - -public class Dino.Plugins.Rtp.Device : MediaDevice, Object { - private const int[] common_widths = {320, 352, 416, 480, 640, 960, 1280, 1920, 2560, 3840}; - - public Plugin plugin { get; private set; } - public CodecUtil codec_util { get { return plugin.codec_util; } } - public Gst.Device device { get; private set; } - - public string id { owned get { return device_name; }} - public string display_name { owned get { return device_display_name; }} - public string? detail_name { owned get { - if (device.properties == null) return null; - if (device.properties.has_field("alsa.card_name")) return device.properties.get_string("alsa.card_name"); - if (device.properties.has_field("alsa.name")) return device.properties.get_string("alsa.name"); - if (device.properties.has_field("alsa.id")) return device.properties.get_string("alsa.id"); - if (device.properties.has_field("api.v4l2.cap.card")) return device.properties.get_string("api.v4l2.cap.card"); - return null; - }} - public string? media { owned get { - if (device.has_classes("Audio")) { - return "audio"; - } else if (device.has_classes("Video")) { - return "video"; - } else { - return null; - } - }} - public bool incoming { get { return is_sink; } } - - public Gst.Pipeline pipe { get { return plugin.pipe; }} - public bool is_source { get { return device.has_classes("Source"); }} - public bool is_sink { get { return device.has_classes("Sink"); }} - public bool is_monitor { get { return (device.properties != null && device.properties.get_string("device.class") == "monitor") || (protocol == DeviceProtocol.PIPEWIRE && device.has_classes("Stream")); } } - public bool is_default { get { - if (device.properties == null) return false; - bool ret; - device.properties.get_boolean("is-default", out ret); - return ret; - }} - public DeviceProtocol protocol { get { - if (device.properties == null) return DeviceProtocol.OTHER; - if (device.properties.has_name("pulse-proplist")) return DeviceProtocol.PULSEAUDIO; - if (device.properties.has_name("pipewire-proplist")) return DeviceProtocol.PIPEWIRE; - if (device.properties.has_name("v4l2deviceprovider")) return DeviceProtocol.V4L2; - return DeviceProtocol.OTHER; - }} - - private string device_name; - private string device_display_name; - - private Gst.Caps device_caps; - private Gst.Element element; - private Gst.Element tee; - private Gst.Element dsp; - private Gst.Base.Aggregator mixer; - private Gst.Element filter; - private int links; - - // Codecs - private Gee.Map codecs = new HashMap(PayloadType.hash_func, PayloadType.equals_func); - private Gee.Map codec_tees = new HashMap(PayloadType.hash_func, PayloadType.equals_func); - - // Payloaders - private Gee.Map> payloaders = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); - private Gee.Map> payloader_tees = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); - private Gee.Map> payloader_links = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); - - // Bitrate - private Gee.Map> codec_bitrates = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); - - private class CodecBitrate { - public uint bitrate; - public int64 timestamp; - - public CodecBitrate(uint bitrate) { - this.bitrate = bitrate; - this.timestamp = get_monotonic_time(); - } - } - - public Device(Plugin plugin, Gst.Device device) { - this.plugin = plugin; - update(device); - } - - public bool matches(Gst.Device device) { - if (this.device.name == device.name) return true; - return false; - } - - public void update(Gst.Device device) { - this.device = device; - this.device_name = device.name; - this.device_display_name = device.display_name; - } - - public Gst.Element? link_sink() { - if (!is_sink) return null; - if (element == null) create(); - links++; - if (mixer != null) { - Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())"); - pipe.add(rate); - rate.link(mixer); - return rate; - } - if (media == "audio") return filter; - return element; - } - - public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) { - if (!is_source) return null; - if (element == null) create(); - links++; - if (payload_type != null && ssrc != 0 && tee != null) { - bool new_codec = false; - string? codec = CodecUtil.get_codec_from_payload(media, payload_type); - if (!codecs.has_key(payload_type)) { - codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder"); - pipe.add(codecs[payload_type]); - new_codec = true; - } - if (!codec_tees.has_key(payload_type)) { - codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee"); - codec_tees[payload_type].@set("allow-not-linked", true); - pipe.add(codec_tees[payload_type]); - codecs[payload_type].link(codec_tees[payload_type]); - } - if (!payloaders.has_key(payload_type)) { - payloaders[payload_type] = new HashMap(); - } - if (!payloaders[payload_type].has_key(ssrc)) { - payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)"); - var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); - payload.ssrc = ssrc; - payload.seqnum_offset = seqnum_offset; - if (timestamp_offset != 0) { - payload.timestamp_offset = timestamp_offset; - } - pipe.add(payloaders[payload_type][ssrc]); - codec_tees[payload_type].link(payloaders[payload_type][ssrc]); - debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset); - } - if (!payloader_tees.has_key(payload_type)) { - payloader_tees[payload_type] = new HashMap(); - } - if (!payloader_tees[payload_type].has_key(ssrc)) { - payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee"); - payloader_tees[payload_type][ssrc].@set("allow-not-linked", true); - pipe.add(payloader_tees[payload_type][ssrc]); - payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]); - } - if (!payloader_links.has_key(payload_type)) { - payloader_links[payload_type] = new HashMap(); - } - if (!payloader_links[payload_type].has_key(ssrc)) { - payloader_links[payload_type][ssrc] = 1; - } else { - payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1; - } - if (new_codec) { - tee.link(codecs[payload_type]); - } - return payloader_tees[payload_type][ssrc]; - } - if (tee != null) return tee; - return element; - } - - private static double get_target_bitrate(Gst.Caps caps) { - if (caps == null || caps.get_size() == 0) return uint.MAX; - unowned Gst.Structure? that = caps.get_structure(0); - int num = 0, den = 0, width = 0, height = 0; - if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX; - if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX; - if (!that.has_field("framerate")) return uint.MAX; - Value framerate = that.get_value("framerate"); - if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX; - num = Gst.Value.get_fraction_numerator(framerate); - den = Gst.Value.get_fraction_denominator(framerate); - double pxs = ((double)num/(double)den) * (double)width * (double)height; - double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0; - if (br < 128.0) return 128.0; - return br; - } - - private Gst.Caps get_active_caps(PayloadType payload_type) { - return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; - } - - private void apply_caps(PayloadType payload_type, Gst.Caps caps) { - plugin.pause(); - debug("Set scaled caps to %s", caps.to_string()); - codec_util.update_rescale_caps(codecs[payload_type], caps); - plugin.unpause(); - } - - private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { - int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; - device_caps.get_structure(0).get_int("width", out device_caps_width); - device_caps.get_structure(0).get_int("height", out device_caps_height); - device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den); - Gst.Caps active_caps = get_active_caps(payload_type); - if (active_caps != null && active_caps.get_size() > 0) { - active_caps.get_structure(0).get_int("width", out active_caps_width); - } else { - active_caps_width = device_caps_width; - } - if (new_width == active_caps_width) return; - int new_height = device_caps_height * new_width / device_caps_width; - if ((new_height % 2) != 0) { - new_height += 1; - } - Gst.Caps new_caps; - if (device_caps_framerate_den != 0) { - new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); - } else { - new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null); - } - double required_bitrate = get_target_bitrate(new_caps); - debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); - if (bitrate < required_bitrate && new_width > active_caps_width) return; - apply_caps(payload_type, new_caps); - } - - public void update_bitrate(PayloadType payload_type, uint bitrate) { - if (codecs.has_key(payload_type)) { - lock(codec_bitrates); - if (!codec_bitrates.has_key(payload_type)) { - codec_bitrates[payload_type] = new ArrayList(); - } - codec_bitrates[payload_type].add(new CodecBitrate(bitrate)); - var remove = new ArrayList(); - foreach (CodecBitrate rate in codec_bitrates[payload_type]) { - if (rate.timestamp < get_monotonic_time() - 5000000L) { - remove.add(rate); - continue; - } - if (rate.bitrate < bitrate) { - bitrate = rate.bitrate; - } - } - codec_bitrates[payload_type].remove_all(remove); - if (media == "video") { - if (bitrate < 128) bitrate = 128; - Gst.Caps active_caps = get_active_caps(payload_type); - double max_bitrate = get_target_bitrate(device_caps) * 2; - double current_target_bitrate = get_target_bitrate(active_caps); - int device_caps_width, active_caps_width; - device_caps.get_structure(0).get_int("width", out device_caps_width); - if (active_caps != null && active_caps.get_size() > 0) { - active_caps.get_structure(0).get_int("width", out active_caps_width); - } else { - active_caps_width = device_caps_width; - } - if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { - // Lower video resolution - int i = 1; - for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) { - debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate); - } - apply_width(payload_type, common_widths[i-1], bitrate); - } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { - // Higher video resolution - int i = 0; - for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); - if (common_widths[i] != active_caps_width) { - debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate); - } - if (common_widths[i] > device_caps_width) { - // We never scale up, so just stick with what the device gives - apply_width(payload_type, device_caps_width, bitrate); - } else if (common_widths[i] != active_caps_width) { - apply_width(payload_type, common_widths[i], bitrate); - } - } - if (bitrate > max_bitrate) bitrate = (uint) max_bitrate; - } - codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); - unlock(codec_bitrates); - } - } - - public void unlink(Gst.Element? link = null) { - if (links <= 0) { - critical("Link count below zero."); - return; - } - if (link != null && is_source && tee != null) { - PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key; - uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key; - payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1; - if (payloader_links[payload_type][ssrc] == 0) { - plugin.pause(); - - codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]); - payloaders[payload_type][ssrc].set_locked_state(true); - payloaders[payload_type][ssrc].set_state(Gst.State.NULL); - payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]); - pipe.remove(payloaders[payload_type][ssrc]); - payloaders[payload_type].unset(ssrc); - payloader_tees[payload_type][ssrc].set_locked_state(true); - payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL); - pipe.remove(payloader_tees[payload_type][ssrc]); - payloader_tees[payload_type].unset(ssrc); - - payloader_links[payload_type].unset(ssrc); - plugin.unpause(); - } - if (payloader_links[payload_type].size == 0) { - plugin.pause(); - - tee.unlink(codecs[payload_type]); - codecs[payload_type].set_locked_state(true); - codecs[payload_type].set_state(Gst.State.NULL); - codecs[payload_type].unlink(codec_tees[payload_type]); - pipe.remove(codecs[payload_type]); - codecs.unset(payload_type); - codec_tees[payload_type].set_locked_state(true); - codec_tees[payload_type].set_state(Gst.State.NULL); - pipe.remove(codec_tees[payload_type]); - codec_tees.unset(payload_type); - - payloaders.unset(payload_type); - payloader_tees.unset(payload_type); - payloader_links.unset(payload_type); - plugin.unpause(); - } - } - if (link != null && is_sink && mixer != null) { - plugin.pause(); - link.set_locked_state(true); - Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer(); - link.get_static_pad("src").unlink(mixer_sink_pad); - mixer_sink_pad.set_active(false); - link.set_state(Gst.State.NULL); - pipe.remove(link); - mixer.release_request_pad(mixer_sink_pad); - plugin.unpause(); - } - links--; - if (links == 0) { - destroy(); - } - } - - private Gst.Caps get_best_caps() { - if (media == "audio") { - return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); - } else if (media == "video" && device.caps.get_size() > 0) { - int best_index = -1; - Value? best_fraction = null; - int best_fps = 0; - int best_width = 0; - int best_height = 0; - for (int i = 0; i < device.caps.get_size(); i++) { - unowned Gst.Structure? that = device.caps.get_structure(i); - unowned Gst.CapsFeatures? features = device.caps.get_features(i); - Value? best_fraction_now = null; - if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue; - int num = 0, den = 0, width = 0, height = 0; - if (!that.has_field("framerate")) continue; - Value framerate = that.get_value("framerate"); - if (framerate.type() == typeof(Gst.Fraction)) { - num = Gst.Value.get_fraction_numerator(framerate); - den = Gst.Value.get_fraction_denominator(framerate); - } else if (framerate.type() == typeof(Gst.ValueList)) { - for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { - Value fraction = Gst.ValueList.get_value(framerate, j); - int in_num = Gst.Value.get_fraction_numerator(fraction); - int in_den = Gst.Value.get_fraction_denominator(fraction); - int fps = den > 0 ? (num/den) : 0; - int in_fps = in_den > 0 ? (in_num/in_den) : 0; - if (in_fps > fps) { - best_fraction_now = fraction; - num = in_num; - den = in_den; - } - } - } else { - debug("Unknown type for framerate: %s", framerate.type_name()); - } - if (den == 0) continue; - if (!that.has_field("width") || !that.get_int("width", out width)) continue; - if (!that.has_field("height") || !that.get_int("height", out height)) continue; - int fps = num/den; - if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) { - best_fps = fps; - best_width = width; - best_height = height; - best_index = i; - best_fraction = best_fraction_now; - } - } - if (best_index == -1) { - // No caps in first round, try without framerate - int target_pixel_count = 1280 * 720; - double target_pixel_ratio = 16.0 / 9.0; - double best_diversion_factor = 0.0; - - for (int i = 0; i < device.caps.get_size(); i++) { - unowned Gst.Structure? that = device.caps.get_structure(i); - unowned Gst.CapsFeatures? features = device.caps.get_features(i); - if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue; - int width = 0, height = 0; - if (!that.has_field("width") || !that.get_int("width", out width) || width == 0) continue; - if (!that.has_field("height") || !that.get_int("height", out height) || height == 0) continue; - - int pixel_count = width * height; - double pixel_count_diversion_factor; - if (pixel_count > target_pixel_count) - pixel_count_diversion_factor = (double)pixel_count / target_pixel_count; - else - pixel_count_diversion_factor = (double)target_pixel_count / pixel_count; - - double pixel_ratio; - if (width > height) - pixel_ratio = (double)width / height; - else - pixel_ratio = (double)height / width; - - double ratio_diversion_factor; - if (pixel_ratio > target_pixel_ratio) - ratio_diversion_factor = pixel_ratio / target_pixel_ratio; - else - ratio_diversion_factor = target_pixel_ratio / pixel_ratio; - - // 1.0 is the best possible value. Any diversion from - // target_pixel_count or target_pixel_ratio increases it. - double diversion_factor = pixel_count_diversion_factor * ratio_diversion_factor; - - if (best_index == -1 || diversion_factor < best_diversion_factor) { - best_index = i; - best_width = width; - best_height = height; - best_diversion_factor = diversion_factor; - } - } - } - Gst.Caps res = caps_copy_nth(device.caps, best_index); - unowned Gst.Structure? that = res.get_structure(0); - Value? framerate = that.get_value("framerate"); - if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) { - that.set_value("framerate", best_fraction); - } else if (framerate == null) { - // The source does not support framerate negotiation. Just set a - // reasonable value. The source might be able to honor it and - // should ignore it otherwise. - that.@set("framerate", typeof(Gst.Fraction), 30, 1, null); - } - debug("Selected caps %s", res.to_string()); - return res; - } else if (device.caps.get_size() > 0) { - return caps_copy_nth(device.caps, 0); - } else { - return new Gst.Caps.any(); - } - } - - // Backport from gst_caps_copy_nth added in GStreamer 1.16 - private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) { - Gst.Caps target = new Gst.Caps.empty(); - target.flags = source.flags; - target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy()); - return target; - } - - private void create() { - debug("Creating device %s", id); - plugin.pause(); - element = device.create_element(id); - if (is_sink) { - element.@set("async", false); - element.@set("sync", false); - } - pipe.add(element); - device_caps = get_best_caps(); - if (is_source) { - element.@set("do-timestamp", true); - filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", device_caps); - pipe.add(filter); - element.link(filter); -#if WITH_VOICE_PROCESSOR - if (media == "audio" && plugin.echoprobe != null) { - dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); - dsp.name = @"dsp_$id"; - pipe.add(dsp); - filter.link(dsp); - } -#endif - tee = Gst.ElementFactory.make("tee", @"tee_$id"); - tee.@set("allow-not-linked", true); - pipe.add(tee); - (dsp ?? filter).link(tee); - } - if (is_sink && media == "audio") { - mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id"); - pipe.add(mixer); - if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) { - mixer.link(plugin.echoprobe); - plugin.echoprobe.link(element); - } else { - filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", device_caps); - pipe.add(filter); - mixer.link(filter); - filter.link(element); - } - } - plugin.unpause(); - } - - private void destroy() { - if (is_sink) { - if (mixer != null) { - int linked_sink_pads = 0; - mixer.foreach_sink_pad((_, pad) => { - if (pad.is_linked()) linked_sink_pads++; - return true; - }); - if (linked_sink_pads > 0) { - warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); - } - mixer.unlink(plugin.echoprobe ?? element); - } - if (filter != null) { - filter.set_locked_state(true); - filter.set_state(Gst.State.NULL); - filter.unlink(element); - pipe.remove(filter); - filter = null; - } - if (plugin.echoprobe != null) { - plugin.echoprobe.unlink(element); - } - } - element.set_locked_state(true); - element.set_state(Gst.State.NULL); - if (filter != null) element.unlink(filter); - else if (is_source) element.unlink(tee); - pipe.remove(element); - element = null; - if (mixer != null) { - mixer.set_locked_state(true); - mixer.set_state(Gst.State.NULL); - pipe.remove(mixer); - mixer = null; - } - if (is_source) { - if (filter != null) { - filter.set_locked_state(true); - filter.set_state(Gst.State.NULL); - filter.unlink(dsp ?? tee); - pipe.remove(filter); - filter = null; - } - if (dsp != null) { - dsp.set_locked_state(true); - dsp.set_state(Gst.State.NULL); - dsp.unlink(tee); - pipe.remove(dsp); - dsp = null; - } - if (tee != null) { - int linked_src_pads = 0; - tee.foreach_src_pad((_, pad) => { - if (pad.is_linked()) linked_src_pads++; - return true; - }); - if (linked_src_pads != 0) { - warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); - } - tee.set_locked_state(true); - tee.set_state(Gst.State.NULL); - pipe.remove(tee); - tee = null; - } - } - debug("Destroyed device %s", id); - } -} diff --git a/plugins/rtp/src/gst_fixes.c b/plugins/rtp/src/gst_fixes.c deleted file mode 100644 index 86b2b598..00000000 --- a/plugins/rtp/src/gst_fixes.c +++ /dev/null @@ -1,10 +0,0 @@ -#include - -GstVideoInfo *gst_video_frame_get_video_info(GstVideoFrame *frame) { - return &frame->info; -} - -void *gst_video_frame_get_data(GstVideoFrame *frame, size_t* length) { - *length = frame->info.height * frame->info.stride[0]; - return frame->data[0]; -} \ No newline at end of file diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala deleted file mode 100644 index be153b10..00000000 --- a/plugins/rtp/src/module.vala +++ /dev/null @@ -1,235 +0,0 @@ -using Gee; -using Xmpp; -using Xmpp.Xep; - -public class Dino.Plugins.Rtp.Module : JingleRtp.Module { - private Set supported_codecs = new HashSet(); - private Set unsupported_codecs = new HashSet(); - public Plugin plugin { get; private set; } - public CodecUtil codec_util { get { - return plugin.codec_util; - }} - - public Module(Plugin plugin) { - base(); - this.plugin = plugin; - } - - private async bool pipeline_works(string media, string element_desc) { - var supported = false; - string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output"; - try { - var pipeline = Gst.parse_launch(pipeline_desc); - var output = ((Gst.Bin) pipeline).get_by_name("output") as Gst.App.Sink; - SourceFunc callback = pipeline_works.callback; - var finished = false; - output.emit_signals = true; - output.new_sample.connect(() => { - if (!finished) { - finished = true; - supported = true; - Idle.add(() => { - callback(); - return Source.REMOVE; - }); - } - return Gst.FlowReturn.EOS; - }); - pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => { - if (message.type == Gst.MessageType.ERROR && !finished) { - Error e; - string d; - message.parse_error(out e, out d); - debug("pipeline [%s] failed: %s", pipeline_desc, e.message); - debug(d); - finished = true; - callback(); - } - return true; - }); - Timeout.add(2000, () => { - if (!finished) { - finished = true; - callback(); - } - return Source.REMOVE; - }); - pipeline.set_state(Gst.State.PLAYING); - yield; - pipeline.set_state(Gst.State.NULL); - } catch (Error e) { - debug("pipeline [%s] failed: %s", pipeline_desc, e.message); - } - return supported; - } - - public override async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { - string? codec = CodecUtil.get_codec_from_payload(media, payload_type); - if (codec == null) return false; - if (unsupported_codecs.contains(codec)) return false; - if (supported_codecs.contains(codec)) return true; - - string? encode_element = codec_util.get_encode_element_name(media, codec); - string? decode_element = codec_util.get_decode_element_name(media, codec); - if (encode_element == null || decode_element == null) { - warning("No suitable encoder or decoder found for %s", codec); - unsupported_codecs.add(codec); - return false; - } - - string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); - while (!(yield pipeline_works(media, encode_bin))) { - debug("%s not suited for encoding %s", encode_element, codec); - codec_util.mark_element_unsupported(encode_element); - encode_element = codec_util.get_encode_element_name(media, codec); - if (encode_element == null) { - warning("No suitable encoder found for %s", codec); - unsupported_codecs.add(codec); - return false; - } - encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); - } - debug("using %s to encode %s", encode_element, codec); - - string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); - while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { - debug("%s not suited for decoding %s", decode_element, codec); - codec_util.mark_element_unsupported(decode_element); - decode_element = codec_util.get_decode_element_name(media, codec); - if (decode_element == null) { - warning("No suitable decoder found for %s", codec); - unsupported_codecs.add(codec); - return false; - } - decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); - } - debug("using %s to decode %s", decode_element, codec); - - supported_codecs.add(codec); - return true; - } - - public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) { - if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true; - return false; - } - - public override Gee.List get_suggested_header_extensions(string media) { - Gee.List exts = new ArrayList(); - if (media == "video") { - exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation")); - } - return exts; - } - - public async void add_if_supported(Gee.List list, string media, JingleRtp.PayloadType payload_type) { - if (yield is_payload_supported(media, payload_type)) { - list.add(payload_type); - } - } - - public override async Gee.List get_supported_payloads(string media) { - Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); - if (media == "audio") { - var opus = new JingleRtp.PayloadType() { channels = 1, clockrate = 48000, name = "opus", id = 111, channels = 2 }; - opus.parameters["useinbandfec"] = "1"; - var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 112 }; - var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 113 }; - var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 114 }; - var g722 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "G722", id = 9 }; - var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 }; - var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 }; - yield add_if_supported(list, media, opus); - yield add_if_supported(list, media, speex32); - yield add_if_supported(list, media, speex16); - yield add_if_supported(list, media, speex8); - yield add_if_supported(list, media, g722); - yield add_if_supported(list, media, pcmu); - yield add_if_supported(list, media, pcma); - } else if (media == "video") { - var rtcp_fbs = new ArrayList(); - rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb")); - rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir")); - rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack")); - rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli")); -#if ENABLE_H264 - var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 }; - yield add_if_supported(list, media, h264); - h264.rtcp_fbs.add_all(rtcp_fbs); -#endif -#if ENABLE_VP9 - var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 }; - vp9.rtcp_fbs.add_all(rtcp_fbs); - yield add_if_supported(list, media, vp9); -#endif - var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 }; - vp8.rtcp_fbs.add_all(rtcp_fbs); - yield add_if_supported(list, media, vp8); - } else { - warning("Unsupported media type: %s", media); - } - return list; - } - - public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List payloads) { - if (media == "audio" || media == "video") { - foreach (JingleRtp.PayloadType type in payloads) { - if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); - } - } else { - warning("Unsupported media type: %s", media); - } - return null; - } - - public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) { - var iter = type.rtcp_fbs.iterator(); - while (iter.next()) { - var fb = iter.@get(); - switch (fb.type_) { - case "goog-remb": - if (fb.subtype != null) iter.remove(); - break; - case "ccm": - if (fb.subtype != "fir") iter.remove(); - break; - case "nack": - if (fb.subtype != null && fb.subtype != "pli") iter.remove(); - break; - default: - iter.remove(); - break; - } - } - return type; - } - - public override JingleRtp.Stream create_stream(Jingle.Content content) { - return plugin.open_stream(content); - } - - public override void close_stream(JingleRtp.Stream stream) { - var rtp_stream = stream as Rtp.Stream; - plugin.close_stream(rtp_stream); - } - - public override JingleRtp.Crypto? generate_local_crypto() { - uint8[] key_and_salt = new uint8[30]; - Crypto.randomize(key_and_salt); - return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt); - } - - public override JingleRtp.Crypto? pick_remote_crypto(Gee.List cryptos) { - foreach (JingleRtp.Crypto crypto in cryptos) { - if (crypto.is_valid) return crypto; - } - return null; - } - - public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { - if (remote == null || !remote.is_valid) return null; - uint8[] key_and_salt = new uint8[30]; - Crypto.randomize(key_and_salt); - return remote.rekey(key_and_salt); - } -} diff --git a/plugins/rtp/src/participant.vala b/plugins/rtp/src/participant.vala deleted file mode 100644 index 1ca13191..00000000 --- a/plugins/rtp/src/participant.vala +++ /dev/null @@ -1,39 +0,0 @@ -using Gee; -using Xmpp; - -public class Dino.Plugins.Rtp.Participant { - public Jid full_jid { get; private set; } - - protected Gst.Pipeline pipe; - private Map ssrcs = new HashMap(); - - public Participant(Gst.Pipeline pipe, Jid full_jid) { - this.pipe = pipe; - this.full_jid = full_jid; - } - - public uint32 get_ssrc(Stream stream) { - if (ssrcs.has_key(stream)) { - return ssrcs[stream]; - } - return 0; - } - - public void set_ssrc(Stream stream, uint32 ssrc) { - if (ssrcs.has_key(stream)) { - warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]); - } else { - stream.on_destroy.connect(unset_ssrc); - } - ssrcs[stream] = ssrc; - } - - public void unset_ssrc(Stream stream) { - ssrcs.unset(stream); - stream.on_destroy.disconnect(unset_ssrc); - } - - public string to_string() { - return @"participant $full_jid"; - } -} \ No newline at end of file diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala deleted file mode 100644 index d09b7edf..00000000 --- a/plugins/rtp/src/plugin.vala +++ /dev/null @@ -1,504 +0,0 @@ -using Gee; -using Xmpp; -using Xmpp.Xep; - -public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { - public Dino.Application app { get; private set; } - public CodecUtil codec_util { get; private set; } - public Gst.DeviceMonitor? device_monitor { get; private set; } - public Gst.Pipeline? pipe { get; private set; } - public Gst.Bin? rtpbin { get; private set; } - public Gst.Element? echoprobe { get; private set; } - - private Gee.List streams = new ArrayList(); - private Gee.List devices = new ArrayList(); - private int64 last_devices_refresh = 0; - // private Gee.List participants = new ArrayList(); - - public void registered(Dino.Application app) { - this.app = app; - this.codec_util = new CodecUtil(); - app.add_option_group(Gst.init_get_option_group()); - app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { - list.add(new Module(this)); - }); - app.plugin_registry.video_call_plugin = this; - } - - private int pause_count = 0; - public void pause() { -// if (pause_count == 0) { -// debug("Pausing pipe for modifications"); -// pipe.set_state(Gst.State.PAUSED); -// } - pause_count++; - } - public void unpause() { - pause_count--; - if (pause_count == 0) { - debug("Continue pipe after modifications"); - pipe.set_state(Gst.State.PLAYING); - } - if (pause_count < 0) warning("Pause count below zero!"); - } - - private void handle_existing_devices(Gst.DeviceMonitor device_monitor) { - var new_devices = new ArrayList(); - foreach (Gst.Device device in device_monitor.get_devices()) { - if (device.properties == null) continue; - if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue; - if (device.properties.get_string("device.class") == "monitor") continue; - var pre_device = devices.first_match((it) => it.matches(device)); - if (pre_device != null) { - if (!new_devices.contains(pre_device)) new_devices.add(pre_device); - continue; - } - var new_device = new Device(this, device); - devices.add(new_device); - new_devices.add(new_device); - } - devices.retain_all(new_devices); - } - - private void refresh_devices() { - if (device_monitor != null) return; // No manual refresh needed, we're monitoring actively. - if (last_devices_refresh > 0 && last_devices_refresh + 5000000 > get_monotonic_time()) return; // Already current - var device_monitor = new Gst.DeviceMonitor(); - device_monitor.show_all = true; - device_monitor.start(); - handle_existing_devices(device_monitor); - device_monitor.stop(); - last_devices_refresh = get_monotonic_time(); - } - - private void start_device_monitor() { - if (device_monitor != null) return; - device_monitor = new Gst.DeviceMonitor(); - device_monitor.show_all = true; - device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); - device_monitor.start(); - handle_existing_devices(device_monitor); - } - - private void stop_device_monitor() { - if (device_monitor == null) return; - device_monitor.stop(); - device_monitor = null; - } - - private void init_call_pipe() { - if (pipe != null) return; - debug("Creating call pipe."); - start_device_monitor(); - pipe = new Gst.Pipeline(null); - - // RTP - rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin; - if (rtpbin == null) { - warning("RTP not supported"); - pipe = null; - return; - } - rtpbin.pad_added.connect(on_rtp_pad_added); - rtpbin.@set("latency", 100); - rtpbin.@set("do-lost", true); -// rtpbin.@set("do-sync-event", true); - rtpbin.@set("drop-on-latency", true); - rtpbin.connect("signal::request-pt-map", request_pt_map, this); - pipe.add(rtpbin); - -#if WITH_VOICE_PROCESSOR - // Audio echo probe - echoprobe = new EchoProbe(); - if (echoprobe != null) pipe.add(echoprobe); -#endif - - // Pipeline - pipe.auto_flush_bus = true; - pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => { - on_pipe_bus_message(message); - return true; - }); - pipe.set_state(Gst.State.PLAYING); - } - - public void destroy_call_pipe_if_unused() { - if (streams.is_empty && !VideoWidget.has_instances()) { - destroy_call_pipe(); - } - } - - private void destroy_call_pipe() { - if (pipe == null) return; - pipe.set_state(Gst.State.NULL); - rtpbin = null; -#if WITH_VOICE_PROCESSOR - echoprobe = null; -#endif - pipe = null; - stop_device_monitor(); - debug("Call pipe destroyed"); - } - - private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { - debug("request-pt-map"); - return null; - } - - private void on_rtp_pad_added(Gst.Pad pad) { - debug("pad added: %s", pad.name); - if (pad.name.has_prefix("recv_rtp_src_")) { - string[] split = pad.name.split("_"); - uint8 rtpid = (uint8)int.parse(split[3]); - foreach (Stream stream in streams) { - if (stream.rtpid == rtpid) { - stream.on_ssrc_pad_added((uint32) split[4].to_uint64(), pad); - } - } - } - if (pad.name.has_prefix("send_rtp_src_")) { - string[] split = pad.name.split("_"); - uint8 rtpid = (uint8)int.parse(split[3]); - debug("pad %s for stream %hhu", pad.name, rtpid); - foreach (Stream stream in streams) { - if (stream.rtpid == rtpid) { - stream.on_send_rtp_src_added(pad); - } - } - } - } - - private void on_pipe_bus_message(Gst.Message message) { - switch (message.type) { - case Gst.MessageType.ERROR: - Error error; - string str; - message.parse_error(out error, out str); - warning("Error in pipeline: %s", error.message); - debug(str); - break; - case Gst.MessageType.WARNING: - Error error; - string str; - message.parse_warning(out error, out str); - warning("Warning in pipeline: %s", error.message); - debug(str); - break; - case Gst.MessageType.CLOCK_LOST: - debug("Clock lost. Restarting"); - pipe.set_state(Gst.State.READY); - pipe.set_state(Gst.State.PLAYING); - break; - case Gst.MessageType.STATE_CHANGED: - // Ignore - break; - case Gst.MessageType.STREAM_STATUS: - Gst.StreamStatusType status; - Gst.Element owner; - message.parse_stream_status(out status, out owner); - if (owner != null) { - debug("%s stream changed status to %s", owner.name, status.to_string()); - } - break; - case Gst.MessageType.ELEMENT: - unowned Gst.Structure struc = message.get_structure(); - if (struc != null && message.src is Gst.Element) { - debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string()); - } - break; - case Gst.MessageType.NEW_CLOCK: - debug("New clock."); - break; - case Gst.MessageType.TAG: - // Ignore - break; - case Gst.MessageType.QOS: - // Ignore - break; - case Gst.MessageType.LATENCY: - if (message.src != null && message.src.name != null && message.src is Gst.Element) { - Gst.Query latency_query = new Gst.Query.latency(); - if (((Gst.Element)message.src).query(latency_query)) { - bool live; - Gst.ClockTime min_latency, max_latency; - latency_query.parse_latency(out live, out min_latency, out max_latency); - debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string()); - } - } - break; - default: - debug("Pipe bus message: %s", message.type.to_string()); - break; - } - } - - private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { - Gst.Device? old_gst_device = null; - Gst.Device? gst_device = null; - Device? device = null; - switch (message.type) { - case Gst.MessageType.DEVICE_ADDED: - message.parse_device_added(out gst_device); - if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE; - device = new Device(this, gst_device); - devices.add(device); - break; - case Gst.MessageType.DEVICE_CHANGED: - message.parse_device_changed(out gst_device, out old_gst_device); - device = devices.first_match((it) => it.matches(old_gst_device)); - if (device != null) device.update(gst_device); - break; - case Gst.MessageType.DEVICE_REMOVED: - message.parse_device_removed(out gst_device); - device = devices.first_match((it) => it.matches(gst_device)); - if (device != null) devices.remove(device); - break; - default: - break; - } - if (device != null) { - devices_changed(device.media, device.is_sink); - } - return Source.CONTINUE; - } - - public uint8 next_free_id() { - uint8 rtpid = 0; - while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) { - rtpid++; - } - return rtpid; - } - - // public Participant get_participant(Jid full_jid, bool self) { -// foreach (Participant participant in participants) { -// if (participant.full_jid.equals(full_jid)) { -// return participant; -// } -// } -// Participant participant; -// if (self) { -// participant = new SelfParticipant(pipe, full_jid); -// } else { -// participant = new Participant(pipe, full_jid); -// } -// participants.add(participant); -// return participant; -// } - - public Stream open_stream(Xmpp.Xep.Jingle.Content content) { - init_call_pipe(); - var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; - if (content_params == null) return null; - Stream stream; - if (content_params.media == "video") { - stream = new VideoStream(this, content); - } else { - stream = new Stream(this, content); - } - streams.add(stream); - return stream; - } - - public void close_stream(Stream stream) { - streams.remove(stream); - stream.destroy(); - destroy_call_pipe_if_unused(); - } - - public void shutdown() { - stop_device_monitor(); - destroy_call_pipe(); - Gst.deinit(); - } - - public bool supported() { - return codec_util.is_element_supported("rtpbin"); - } - - public VideoCallWidget? create_widget(WidgetType type) { - init_call_pipe(); - if (type == WidgetType.GTK4) { - return new VideoWidget(this); - } - return null; - } - - public Gee.List get_devices(string media, bool incoming) { - refresh_devices(); - Gee.List devices; - if (media == "video" && !incoming) { - devices = get_video_sources(); - } else if (media == "audio") { - devices = get_audio_devices(incoming); - } else { - devices = new ArrayList(); - devices.add_all_iterator(this.devices.filter(it => it.media == media && (incoming && it.is_sink || !incoming && it.is_source) && !it.is_monitor)); - } - devices.sort((media_left, media_right) => { - return strcmp(media_left.id, media_right.id); - }); - - return devices; - } - - public Gee.List get_audio_devices(bool incoming) { - ArrayList pulse_devices = new ArrayList(); - ArrayList other_devices = new ArrayList(); - - foreach (Device device in devices) { - if (device.media != "audio") continue; - if (incoming && !device.is_sink || !incoming && !device.is_source) continue; - - // Skip monitors - if (device.is_monitor) continue; - - if (device.protocol == DeviceProtocol.PULSEAUDIO) { - pulse_devices.add(device); - } else { - other_devices.add(device); - } - } - - // If we have any pulseaudio devices, present only those. Don't want duplicated devices from pipewire and pulseaudio. - return pulse_devices.size > 0 ? pulse_devices : other_devices; - } - - public Gee.List get_video_sources() { - ArrayList pipewire_devices = new ArrayList(); - ArrayList other_devices = new ArrayList(); - - foreach (Device device in devices) { - if (device.media != "video") continue; - if (device.is_sink) continue; - - // Skip monitors - if (device.is_monitor) continue; - - bool is_color = false; - for (int i = 0; i < device.device.caps.get_size(); i++) { - unowned Gst.Structure structure = device.device.caps.get_structure(i); - if (!structure.has_field("format")) continue; - // "format" might be an array and get_string() will then return null. We just assume arrays to be fine. - string? format = structure.get_string("format"); - if (format == null || !format.has_prefix("GRAY")) { - is_color = true; - } - } - - // Don't allow grey-scale devices - if (!is_color) continue; - - if (device.protocol == DeviceProtocol.PIPEWIRE) { - pipewire_devices.add(device); - } else { - other_devices.add(device); - } - } - - // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux. - return pipewire_devices.size > 0 ? pipewire_devices : other_devices; - } - - private int get_max_fps(Device device) { - int fps = 0; - for (int i = 0; i < device.device.caps.get_size(); i++) { - unowned Gst.Structure structure = device.device.caps.get_structure(i); - - if (structure.has_field("framerate")) { - Value framerate = structure.get_value("framerate"); - if (framerate.type() == typeof(Gst.Fraction)) { - int num = Gst.Value.get_fraction_numerator(framerate); - int den = Gst.Value.get_fraction_denominator(framerate); - fps = int.max(fps, num / den); - } else if (framerate.type() == typeof(Gst.ValueList)) { - for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { - Value fraction = Gst.ValueList.get_value(framerate, j); - int num = Gst.Value.get_fraction_numerator(fraction); - int den = Gst.Value.get_fraction_denominator(fraction); - fps = int.max(fps, num / den); - } - } else { - debug("Unknown type for framerate %s on device %s", framerate.type_name(), device.display_name); - } - } - } - - debug("Max framerate for device %s: %d", device.display_name, fps); - return fps; - } - - public MediaDevice? get_preferred_device(string media, bool incoming) { - Gee.List devices = new ArrayList(); - foreach (MediaDevice media_device in get_devices(media, incoming)) { - if (media_device is Device) devices.add((Device)media_device); - } - if (devices.is_empty) { - warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media); - return null; - } - - // Take default if present - foreach (Device device in devices) { - if (device.is_default) { - debug("Using %s for %s %s as it's default", device.display_name, incoming ? "incoming" : "outgoing", media); - return device; - } - } - - if (media == "video") { - // Pick best FPS - int max_fps = -1; - Device? max_fps_device = null; - foreach (Device device in devices) { - int fps = get_max_fps(device); - if (fps > max_fps || max_fps_device == null) { - max_fps = fps; - max_fps_device = device; - } - } - debug("Using %s for %s %s as it has max FPS (%d)", max_fps_device.display_name, incoming ? "incoming" : "outgoing", media, max_fps); - return max_fps_device; - } else { - // Pick any - Device? device = devices.first(); - debug("Using %s for %s %s as it's first pick", device.display_name, incoming ? "incoming" : "outgoing", media); - return device; - } - } - - public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream? stream, bool incoming) { - Stream? plugin_stream = stream as Stream?; - if (plugin_stream == null) return null; - MediaDevice? device = incoming ? plugin_stream.output_device : plugin_stream.input_device; - return device ?? get_preferred_device(stream.media, incoming); - } - - public void dump_dot() { - if (pipe == null) return; - string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; - Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); - print(@"Stored pipe details as $name\n"); - } - - public void set_pause(Xmpp.Xep.JingleRtp.Stream? stream, bool pause) { - Stream? plugin_stream = stream as Stream?; - if (plugin_stream == null) return; - if (pause) { - plugin_stream.pause(); - } else { - plugin_stream.unpause(); - } - } - - public void set_device(Xmpp.Xep.JingleRtp.Stream? stream, MediaDevice? device) { - Device? real_device = device as Device?; - Stream? plugin_stream = stream as Stream?; - if (real_device == null || plugin_stream == null) return; - if (real_device.is_source) { - plugin_stream.input_device = real_device; - } else if (real_device.is_sink) { - plugin_stream.output_device = real_device; - } - } -} diff --git a/plugins/rtp/src/register_plugin.vala b/plugins/rtp/src/register_plugin.vala deleted file mode 100644 index a80137ea..00000000 --- a/plugins/rtp/src/register_plugin.vala +++ /dev/null @@ -1,3 +0,0 @@ -public Type register_plugin(Module module) { - return typeof (Dino.Plugins.Rtp.Plugin); -} diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala deleted file mode 100644 index 9a46d9ff..00000000 --- a/plugins/rtp/src/stream.vala +++ /dev/null @@ -1,851 +0,0 @@ -using Gee; -using Xmpp; - -public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { - public uint8 rtpid { get; private set; } - - public Plugin plugin { get; private set; } - public Gst.Pipeline pipe { get { - return plugin.pipe; - }} - public Gst.Element rtpbin { get { - return plugin.rtpbin; - }} - public CodecUtil codec_util { get { - return plugin.codec_util; - }} - private Gst.App.Sink send_rtp; - private Gst.App.Sink send_rtcp; - private Gst.App.Src recv_rtp; - private Gst.App.Src recv_rtcp; - private Gst.Element decode; - private Gst.RTP.BaseDepayload decode_depay; - private Gst.Element input; - private Gst.Pad input_pad; - private Gst.Element output; - private Gst.Element session; - - private Device _input_device; - public Device input_device { get { return _input_device; } set { - if (sending && !paused) { - var input = this.input; - set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null); - if (this._input_device != null) this._input_device.unlink(input); - } - this._input_device = value; - }} - private Device _output_device; - public Device output_device { get { return _output_device; } set { - if (output != null) remove_output(output); - if (value != null && receiving) add_output(value.link_sink()); - this._output_device = value; - }} - - public bool created { get; private set; default = false; } - public bool paused { get; private set; default = false; } - private bool push_recv_data = false; - private uint our_ssrc = Random.next_int(); - private int next_seqnum_offset = -1; - private uint32 next_timestamp_offset_base = 0; - private int64 next_timestamp_offset_stamp = 0; - private uint32 next_timestamp_offset { get { - if (next_timestamp_offset_base == 0) return 0; - int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp; - return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate); - } } - private uint32 participant_ssrc = 0; - - private Gst.Pad recv_rtcp_sink_pad; - private Gst.Pad recv_rtp_sink_pad; - private Gst.Pad recv_rtp_src_pad; - private Gst.Pad send_rtcp_src_pad; - private Gst.Pad send_rtp_sink_pad; - private Gst.Pad send_rtp_src_pad; - - private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session(); - - public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { - base(content); - this.plugin = plugin; - this.rtpid = plugin.next_free_id(); - - content.notify["senders"].connect_after(on_senders_changed); - } - - public void on_senders_changed() { - if (sending && input == null) { - input_device = input_device; - } - if (receiving && output == null) { - output_device = output_device; - } - } - - public override void create() { - plugin.pause(); - - // Create i/o if needed - - if (input == null && sending) { - input_device = input_device; - } - if (output == null && receiving && media == "audio") { - output_device = output_device; - } - - // Create app elements - send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink; - send_rtp.async = false; - send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); - send_rtp.emit_signals = true; - send_rtp.sync = true; - send_rtp.drop = true; - send_rtp.wait_on_eos = false; - send_rtp.new_sample.connect(on_new_sample); - send_rtp.new_serialized_event.connect(on_new_event); - send_rtp.connect("signal::eos", on_eos_static, this); - pipe.add(send_rtp); - - send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; - send_rtcp.async = false; - send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); - send_rtcp.emit_signals = true; - send_rtcp.sync = true; - send_rtcp.drop = true; - send_rtcp.wait_on_eos = false; - send_rtcp.new_sample.connect(on_new_sample); - send_rtcp.connect("signal::eos", on_eos_static, this); - pipe.add(send_rtcp); - - recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; - recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true); - recv_rtp.do_timestamp = true; - recv_rtp.format = Gst.Format.TIME; - recv_rtp.is_live = true; - pipe.add(recv_rtp); - - recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; - recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); - recv_rtcp.do_timestamp = true; - recv_rtcp.format = Gst.Format.TIME; - recv_rtcp.is_live = true; - pipe.add(recv_rtcp); - - // Connect RTCP - send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid"); - send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink")); - recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid"); - recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); - - // Connect input - send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); - if (input != null) { - input_pad = input.get_request_pad(@"src_$rtpid"); - input_pad.link(send_rtp_sink_pad); - } - - // Connect output - decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); - decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay"); - pipe.add(decode); - if (output != null) { - decode.link(output); - } - - // Connect RTP - recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid"); - recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad); - - created = true; - push_recv_data = true; - plugin.unpause(); - - GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); - if (session != null && remb_enabled) { - Object internal_session; - session.@get("internal-session", out internal_session); - if (internal_session != null) { - internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this); - } - Timeout.add(1000, () => remb_adjust()); - } - if (input_device != null && media == "video") { - input_device.update_bitrate(payload_type, target_send_bitrate); - } - } - - private int last_packets_lost = -1; - private uint64 last_packets_received = 0; - private uint64 last_octets_received = 0; - private uint max_target_receive_bitrate = 0; - private int64 last_remb_time = 0; - private bool remb_adjust() { - unowned Gst.Structure? stats; - if (session == null) { - debug("Session for %u finished, turning off remb adjustment", rtpid); - return Source.REMOVE; - } - session.get("stats", out stats); - if (stats == null) { - warning("No stats for session %u", rtpid); - return Source.REMOVE; - } - unowned ValueArray? source_stats; - stats.get("source-stats", typeof(ValueArray), out source_stats); - if (source_stats == null) { - warning("No source-stats for session %u", rtpid); - return Source.REMOVE; - } - - if (input_device == null) return Source.CONTINUE; - - foreach (Value value in source_stats.values) { - unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); - uint32 ssrc; - if (!source_stat.get_uint("ssrc", out ssrc)) continue; - if (ssrc == participant_ssrc) { - int packets_lost; - uint64 packets_received, octets_received; - source_stat.get_int("packets-lost", out packets_lost); - source_stat.get_uint64("packets-received", out packets_received); - source_stat.get_uint64("octets-received", out octets_received); - int new_lost = packets_lost - last_packets_lost; - if (new_lost < 0) new_lost = 0; - uint64 new_received = packets_received - last_packets_received; - if (packets_received < last_packets_received) new_received = 0; - uint64 new_octets = octets_received - last_octets_received; - if (octets_received < last_octets_received) octets_received = 0; - if (new_received == 0) continue; - last_packets_lost = packets_lost; - last_packets_received = packets_received; - last_octets_received = octets_received; - double loss_rate = (double)new_lost / (double)(new_lost + new_received); - uint new_target_receive_bitrate; - if (new_lost <= 0 || loss_rate < 0.02) { - new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); - } else if (loss_rate > 0.1) { - new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); - } else { - new_target_receive_bitrate = target_receive_bitrate; - } - if (last_remb_time == 0) { - last_remb_time = get_monotonic_time(); - } else { - int64 time_now = get_monotonic_time(); - int64 time_diff = time_now - last_remb_time; - last_remb_time = time_now; - uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); - new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); - max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate); - new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); - } - new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 - if (new_target_receive_bitrate != target_receive_bitrate) { - target_receive_bitrate = new_target_receive_bitrate; - uint8[] data = new uint8[] { - 143, 206, 0, 5, - 0, 0, 0, 0, - 0, 0, 0, 0, - 'R', 'E', 'M', 'B', - 1, 0, 0, 0, - 0, 0, 0, 0 - }; - data[4] = (uint8)((our_ssrc >> 24) & 0xff); - data[5] = (uint8)((our_ssrc >> 16) & 0xff); - data[6] = (uint8)((our_ssrc >> 8) & 0xff); - data[7] = (uint8)(our_ssrc & 0xff); - uint8 br_exp = 0; - uint32 br_mant = target_receive_bitrate * 1000; - uint8 bits = (uint8)Math.log2(br_mant); - if (bits > 16) { - br_exp = (uint8)bits - 16; - br_mant = br_mant >> br_exp; - } - data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); - data[18] = (uint8)((br_mant >> 8) & 0xff); - data[19] = (uint8)(br_mant & 0xff); - data[20] = (uint8)((ssrc >> 24) & 0xff); - data[21] = (uint8)((ssrc >> 16) & 0xff); - data[22] = (uint8)((ssrc >> 8) & 0xff); - data[23] = (uint8)(ssrc & 0xff); - encrypt_and_send_rtcp(data); - } - } - } - return Source.CONTINUE; - } - - private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { - if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) { - // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 - uint8[] data; - fci.extract_dup(0, fci.get_size(), out data); - if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; - uint8 br_exp = data[5] >> 2; - uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; - self.target_send_bitrate = (br_mant << br_exp) / 1000; - self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate); - } - } - - private void prepare_local_crypto() { - if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) { - crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); - debug("Setting up encryption with key params %s", local_crypto.key_params); - } - } - - bool flip = false; - uint8 rotation = 0; - - private bool on_new_event(Gst.App.Sink sink) { - if (sink == null || sink != send_rtp) { - return false; - } - Gst.MiniObject obj = sink.try_pull_object(0); - if (obj.type == typeof(Gst.Event)) { - unowned Gst.TagList tags; - if (((Gst.Event)obj).type == Gst.EventType.TAG) { - ((Gst.Event)obj).parse_tag(out tags); - Gst.Video.OrientationMethod orientation_method; - Gst.Video.Orientation.from_tag(tags, out orientation_method); - switch (orientation_method) { - case Gst.Video.OrientationMethod.IDENTITY: - case Gst.Video.OrientationMethod.VERT: - default: - rotation = 0; - break; - case Gst.Video.OrientationMethod.@90R: - case Gst.Video.OrientationMethod.UL_LR: - rotation = 1; - break; - case Gst.Video.OrientationMethod.@180: - case Gst.Video.OrientationMethod.HORIZ: - rotation = 2; - break; - case Gst.Video.OrientationMethod.@90L: - case Gst.Video.OrientationMethod.UR_LL: - rotation = 3; - break; - } - switch (orientation_method) { - case Gst.Video.OrientationMethod.IDENTITY: - case Gst.Video.OrientationMethod.@90R: - case Gst.Video.OrientationMethod.@180: - case Gst.Video.OrientationMethod.@90L: - default: - flip = false; - break; - case Gst.Video.OrientationMethod.VERT: - case Gst.Video.OrientationMethod.UL_LR: - case Gst.Video.OrientationMethod.HORIZ: - case Gst.Video.OrientationMethod.UR_LL: - flip = true; - break; - } - } - } - return false; - } - - - private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { - if (sink == null) { - debug("Sink is null"); - return Gst.FlowReturn.EOS; - } - if (sink != send_rtp && sink != send_rtcp) { - warning("unknown sample"); - return Gst.FlowReturn.NOT_SUPPORTED; - } - Gst.Sample sample = sink.pull_sample(); - Gst.Buffer buffer = sample.get_buffer(); - if (sink == send_rtp) { - uint buffer_ssrc = 0, buffer_seq = 0; - Gst.RTP.Buffer rtp_buffer; - if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { - buffer_ssrc = rtp_buffer.get_ssrc(); - buffer_seq = rtp_buffer.get_seq(); - next_seqnum_offset = rtp_buffer.get_seq() + 1; - next_timestamp_offset_base = rtp_buffer.get_timestamp(); - next_timestamp_offset_stamp = get_monotonic_time(); - rtp_buffer.unmap(); - } - if (our_ssrc != buffer_ssrc) { - warning_once("Sending RTP %s buffer seq %u with SSRC %u when our ssrc is %u", media, buffer_seq, buffer_ssrc, our_ssrc); - } - } - - if (sink == send_rtp) { - Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); - if (ext != null) { - buffer = (Gst.Buffer) buffer.make_writable(); - Gst.RTP.Buffer rtp_buffer; - if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.WRITE, out rtp_buffer)) { - uint8[] extension_data = new uint8[1]; - bool camera = false; - extension_data[0] = extension_data[0] | (rotation & 0x3); - if (flip) extension_data[0] = extension_data[0] | 0x4; - if (camera) extension_data[0] = extension_data[0] | 0x8; - rtp_buffer.add_extension_onebyte_header(ext.id, extension_data); - } - } - } - - prepare_local_crypto(); - - uint8[] data; - buffer.extract_dup(0, buffer.get_size(), out data); - if (sink == send_rtp) { - encrypt_and_send_rtp((owned) data); - } else if (sink == send_rtcp) { - encrypt_and_send_rtcp((owned) data); - } - return Gst.FlowReturn.OK; - } - - private void encrypt_and_send_rtp(owned uint8[] data) { - Bytes bytes; - if (crypto_session.has_encrypt) { - bytes = new Bytes.take(crypto_session.encrypt_rtp(data)); - } else { - bytes = new Bytes.take(data); - } - on_send_rtp_data(bytes); - } - - private void encrypt_and_send_rtcp(owned uint8[] data) { - Bytes bytes; - if (crypto_session.has_encrypt) { - bytes = new Bytes.take(crypto_session.encrypt_rtcp(data)); - } else { - bytes = new Bytes.take(data); - } - if (rtcp_mux) { - on_send_rtp_data(bytes); - } else { - on_send_rtcp_data(bytes); - } - } - - private static Gst.PadProbeReturn drop_probe() { - return Gst.PadProbeReturn.DROP; - } - - private static void on_eos_static(Gst.App.Sink sink, Stream self) { - debug("EOS on %s", sink.name); - if (sink == self.send_rtp) { - Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; }); - } else if (sink == self.send_rtcp) { - Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; }); - } - } - - private void on_send_rtp_eos() { - if (send_rtp_src_pad != null) { - send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); - send_rtp_src_pad = null; - } - send_rtp.set_locked_state(true); - send_rtp.set_state(Gst.State.NULL); - // This happens async, so pipe might be gone by now. - if (pipe != null) pipe.remove(send_rtp); - send_rtp = null; - debug("Stopped sending RTP for %u", rtpid); - } - - private void on_send_rtcp_eos() { - send_rtcp.set_locked_state(true); - send_rtcp.set_state(Gst.State.NULL); - // This happens async, so pipe might be gone by now. - if (pipe != null) pipe.remove(send_rtcp); - send_rtcp = null; - debug("Stopped sending RTCP for %u", rtpid); - } - - public override void destroy() { - // Stop network communication - push_recv_data = false; - if (recv_rtp != null) recv_rtp.end_of_stream(); - if (recv_rtcp != null) recv_rtcp.end_of_stream(); - if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample); - if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample); - - // Disconnect input device - if (input != null) { - input_pad.unlink(send_rtp_sink_pad); - input.release_request_pad(input_pad); - input_pad = null; - } - if (this._input_device != null) { - if (!paused) this._input_device.unlink(input); - this._input_device = null; - this.input = null; - } - - // Inject EOS - if (send_rtp_sink_pad != null) { - send_rtp_sink_pad.send_event(new Gst.Event.eos()); - } - - // Disconnect decode - if (recv_rtp_src_pad != null) { - recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); - recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); - } - - // Disconnect output - if (output != null) { - decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); - decode.unlink(output); - } - - // Disconnect output device - if (this._output_device != null) { - this._output_device.unlink(output); - this._output_device = null; - } - output = null; - - // Destroy decode - if (decode != null) { - decode.set_locked_state(true); - decode.set_state(Gst.State.NULL); - pipe.remove(decode); - decode = null; - decode_depay = null; - } - - // Disconnect and remove RTP input - if (recv_rtp != null) { - recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); - recv_rtp.set_locked_state(true); - recv_rtp.set_state(Gst.State.NULL); - pipe.remove(recv_rtp); - recv_rtp = null; - } - - // Disconnect and remove RTCP input - if (recv_rtcp != null) { - recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); - recv_rtcp.set_locked_state(true); - recv_rtcp.set_state(Gst.State.NULL); - pipe.remove(recv_rtcp); - recv_rtcp = null; - } - - // Release rtp pads - if (send_rtp_sink_pad != null) { - rtpbin.release_request_pad(send_rtp_sink_pad); - send_rtp_sink_pad = null; - } - if (recv_rtp_sink_pad != null) { - rtpbin.release_request_pad(recv_rtp_sink_pad); - recv_rtp_sink_pad = null; - } - if (send_rtcp_src_pad != null) { - rtpbin.release_request_pad(send_rtcp_src_pad); - send_rtcp_src_pad = null; - } - if (recv_rtcp_sink_pad != null) { - rtpbin.release_request_pad(recv_rtcp_sink_pad); - recv_rtcp_sink_pad = null; - } - } - - private void prepare_remote_crypto() { - if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) { - crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); - debug("Setting up decryption with key params %s", remote_crypto.key_params); - } - } - - private uint16 previous_incoming_video_orientation_degree = uint16.MAX; - public signal void incoming_video_orientation_changed(uint16 degree); - - public override void on_recv_rtp_data(Bytes bytes) { - if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { - on_recv_rtcp_data(bytes); - return; - } - - if (push_recv_data) { - prepare_remote_crypto(); - - Gst.Buffer buffer; - if (crypto_session.has_decrypt) { - try { - buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtp(bytes.get_data())); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return; - } - } else { - buffer = new Gst.Buffer.wrapped_bytes(bytes); - } - - Gst.RTP.Buffer rtp_buffer; - if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { - uint buffer_ssrc = rtp_buffer.get_ssrc(); - uint buffer_seq = rtp_buffer.get_seq(); - if (rtp_buffer.get_extension()) { - Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); - if (ext != null) { - unowned uint8[] extension_data; - if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) { - bool camera = (extension_data[0] & 0x8) > 0; - bool flip = (extension_data[0] & 0x4) > 0; - uint8 rotation = extension_data[0] & 0x3; - uint16 rotation_degree = uint16.MAX; - switch(rotation) { - case 0: rotation_degree = 0; break; - case 1: rotation_degree = 90; break; - case 2: rotation_degree = 180; break; - case 3: rotation_degree = 270; break; - } - if (rotation_degree != previous_incoming_video_orientation_degree) { - incoming_video_orientation_changed(rotation_degree); - previous_incoming_video_orientation_degree = rotation_degree; - } - } - } - } - rtp_buffer.unmap(); - } - - recv_rtp.push_buffer((owned) buffer); - } - } - - public override void on_recv_rtcp_data(Bytes bytes) { - if (push_recv_data) { - prepare_remote_crypto(); - - Gst.Buffer buffer; - if (crypto_session.has_decrypt) { - try { - buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtcp(bytes.get_data())); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return; - } - } else { - buffer = new Gst.Buffer.wrapped_bytes(bytes); - } - - recv_rtcp.push_buffer((owned) buffer); - } - } - - public override void on_rtp_ready() { - // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame. - // Send a full frame to let the counterpart display our video asap - rtpbin.send_event(new Gst.Event.custom( - Gst.EventType.CUSTOM_UPSTREAM, - new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null)) - ); - } - - public override void on_rtcp_ready() { - int rtp_session_id = (int) rtpid; - uint64 max_delay = int.MAX; - Object rtp_session; - bool rtp_sent; - GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session); - GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent); - debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); - } - - public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) { - debug("New ssrc %u with pad %s", ssrc, pad.name); - if (participant_ssrc != 0 && participant_ssrc != ssrc) { - warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc); - return; - } - participant_ssrc = ssrc; - recv_rtp_src_pad = pad; - if (decode != null) { - plugin.pause(); - debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name); - recv_rtp_src_pad.link(decode.get_static_pad("sink")); - plugin.unpause(); - } - } - - public void on_send_rtp_src_added(Gst.Pad pad) { - send_rtp_src_pad = pad; - if (send_rtp != null) { - plugin.pause(); - debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name); - send_rtp_src_pad.link(send_rtp.get_static_pad("sink")); - plugin.unpause(); - } - } - - public void set_input(Gst.Element? input) { - set_input_and_pause(input, paused); - } - - private void set_input_and_pause(Gst.Element? input, bool paused) { - if (created && this.input != null) { - this.input_pad.unlink(send_rtp_sink_pad); - this.input.release_request_pad(this.input_pad); - this.input_pad = null; - this.input = null; - } - - this.input = input; - this.paused = paused; - - if (created && sending && !paused && input != null) { - plugin.pause(); - input_pad = input.get_request_pad(@"src_$rtpid"); - input_pad.link(send_rtp_sink_pad); - plugin.unpause(); - } - } - - public void pause() { - if (paused) return; - var input = this.input; - set_input_and_pause(null, true); - if (input != null && input_device != null) input_device.unlink(input); - } - - public void unpause() { - if (!paused) return; - set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); - input_device.update_bitrate(payload_type, target_send_bitrate); - } - - public uint get_participant_ssrc(Xmpp.Jid participant) { - if (participant.equals(content.session.peer_full_jid)) { - return participant_ssrc; - } - return 0; - } - - ulong block_probe_handler_id = 0; - public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) { - if (output != null) { - critical("add_output() invoked more than once"); - return; - } - if (participant != null) { - critical("add_output() invoked with participant when not supported"); - return; - } - this.output = element; - if (created) { - plugin.pause(); - decode.link(element); - if (block_probe_handler_id != 0) { - decode.get_static_pad("src").remove_probe(block_probe_handler_id); - } - plugin.unpause(); - } - } - - public virtual void remove_output(Gst.Element element) { - if (output != element) { - critical("remove_output() invoked without prior add_output()"); - return; - } - if (created) { - block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); - decode.unlink(element); - } - if (this._output_device != null) { - this._output_device.unlink(element); - this._output_device = null; - } - this.output = null; - } -} - -public class Dino.Plugins.Rtp.VideoStream : Stream { - private Gee.List outputs = new ArrayList(); - private Gst.Element output_tee; - private Gst.Element rotate; - private ulong incoming_video_orientation_changed_handler; - - public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { - base(plugin, content); - if (media != "video") critical("VideoStream created for non-video media"); - } - - public override void create() { - incoming_video_orientation_changed_handler = incoming_video_orientation_changed.connect(on_video_orientation_changed); - plugin.pause(); - rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); - pipe.add(rotate); - output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid"); - output_tee.@set("allow-not-linked", true); - pipe.add(output_tee); - rotate.link(output_tee); - add_output(rotate); - base.create(); - foreach (Gst.Element output in outputs) { - output_tee.link(output); - } - plugin.unpause(); - } - - private void on_video_orientation_changed(uint16 degree) { - if (rotate != null) { - switch (degree) { - case 0: - rotate.@set("method", 0); - break; - case 90: - rotate.@set("method", 1); - break; - case 180: - rotate.@set("method", 2); - break; - case 270: - rotate.@set("method", 3); - break; - } - } - } - - public override void destroy() { - foreach (Gst.Element output in outputs) { - output_tee.unlink(output); - } - base.destroy(); - rotate.set_locked_state(true); - rotate.set_state(Gst.State.NULL); - rotate.unlink(output_tee); - pipe.remove(rotate); - rotate = null; - output_tee.set_locked_state(true); - output_tee.set_state(Gst.State.NULL); - pipe.remove(output_tee); - output_tee = null; - disconnect(incoming_video_orientation_changed_handler); - } - - public override void add_output(Gst.Element element, Xmpp.Jid? participant) { - if (element == output_tee || element == rotate) { - base.add_output(element); - return; - } - outputs.add(element); - if (output_tee != null) { - output_tee.link(element); - } - } - - public override void remove_output(Gst.Element element) { - if (element == output_tee || element == rotate) { - base.remove_output(element); - return; - } - outputs.remove(element); - if (output_tee != null) { - output_tee.unlink(element); - } - } -} diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala deleted file mode 100644 index af6eb14e..00000000 --- a/plugins/rtp/src/video_widget.vala +++ /dev/null @@ -1,293 +0,0 @@ -private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame); -[CCode (array_length_type = "size_t", type = "void*")] -private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame); - -public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object { - private Gdk.Paintable image; - private double pixel_aspect_ratio; - - public override Gdk.PaintableFlags get_flags() { - return 0; - } - - public void snapshot(Gdk.Snapshot snapshot, double width, double height) { - if (image != null) image.snapshot(snapshot, width, height); - } - - public override Gdk.Paintable get_current_image() { - if (image != null) return image; - return Gdk.Paintable.empty(0, 0); - } - - public override int get_intrinsic_width() { - if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width()); - return 0; - } - - public override int get_intrinsic_height() { - if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height()); - return 0; - } - - public override double get_intrinsic_aspect_ratio() { - if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio(); - return 0.0; - } - - public override void dispose() { - image = null; - base.dispose(); - } - - private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) { - if (paintable == image) return; - bool size_changed = image == null || - this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() || - image.get_intrinsic_height() != paintable.get_intrinsic_height() || - image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio(); - - if (image != null) this.image.dispose(); - this.image = paintable; - this.pixel_aspect_ratio = pixel_aspect_ratio; - - if (size_changed) invalidate_size(); - invalidate_contents(); - } - - public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) { - Idle.add(() => { - set_paintable(texture, pixel_aspect_ratio); - return Source.REMOVE; - }, Priority.DEFAULT); - } -} - -public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink { - internal Paintable paintable = new Paintable(); - private Gst.Video.Info info = new Gst.Video.Info(); - - class construct { - set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team "); - add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"))); - } - - construct { - set_drop_out_of_segment(false); - } - - public override bool set_info(Gst.Caps caps, Gst.Video.Info info) { - this.info = info; - return true; - } - - public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) { - if (buffer.pts != -1) { - start = buffer.pts; - if (buffer.duration != -1) { - end = start + buffer.duration; - } else if (info.fps_n > 0) { - end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n); - } - } - } - - public override Gst.Caps get_caps(Gst.Caps? filter) { - Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"); - - if (filter != null) { - return filter.intersect(caps, Gst.CapsIntersectMode.FIRST); - } else { - return caps; - } - } - - private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) { - switch (format) { - case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8; - case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8; - case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8; - case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8; - case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8; - case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8; - default: - warning("Unsupported video format: %s", format.to_string()); - return Gdk.MemoryFormat.A8R8G8B8; - } - } - - private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) { - Gst.Video.Frame frame = Gst.Video.Frame(); - Gdk.Texture texture; - - if (frame.map(info, buffer, Gst.MapFlags.READ)) { - unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame); - Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame)); - texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]); - pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d); - frame.unmap(); - } else { - texture = null; - } - return texture; - } - - private void queue_buffer(Gst.Buffer buf) { - double pixel_aspect_ratio; - Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio); - if (texture != null) { - paintable.queue_set_texture(texture, pixel_aspect_ratio); - } - } - - public override Gst.FlowReturn show_frame(Gst.Buffer buf) { - @lock.lock(); - queue_buffer(buf); - @lock.unlock(); - - return Gst.FlowReturn.OK; - } -} - -public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget { - private const int RECAPS_AFTER_CHANGE = 5; - private static uint last_id = 0; - - public uint id { get; private set; } - public Plugin plugin { get; private set; } - public Gst.Pipeline pipe { get { - return plugin.pipe; - }} - - private bool attached; - private Device? connected_device; - private Gst.Element? connected_device_element; - private Stream? connected_stream; - private Gst.Element prepare; - private Gst.Caps last_input_caps; - private Gst.Caps last_caps; - private int recaps_since_change; - private Sink sink; - private Gtk.Picture widget; - - private static uint active_widgets = 0; - - public static bool has_instances() { - return active_widgets > 0; - } - - private static void notify_weak(Object widget_object) { - if (active_widgets > 0) { - debug("Video widget %p destroyed. left=%u", widget_object, active_widgets); - active_widgets--; - } - if (active_widgets == 0 && widget_object is VideoWidget) { - ((VideoWidget)widget_object).plugin.destroy_call_pipe_if_unused(); - } - } - - public VideoWidget(Plugin plugin) { - this.plugin = plugin; - this.layout_manager = new Gtk.BinLayout(); - - id = last_id++; - sink = new Sink() { async = false, sync = true }; - widget = new Gtk.Picture.for_paintable(sink.paintable); - widget.insert_after(this, null); - active_widgets++; - debug("Video widget %p created. total=%u", this, active_widgets); - this.weak_ref(notify_weak); - } - - public void input_caps_changed(GLib.Object pad, ParamSpec spec) { - Gst.Caps? caps = ((Gst.Pad)pad).caps; - if (caps == null) { - debug("Input: No caps"); - return; - } - - int width, height; - caps.get_structure(0).get_int("width", out width); - caps.get_structure(0).get_int("height", out height); - debug("Input resolution changed: %ix%i", width, height); - // Invoke signal on GTK main loop as recipients are likely to use it for doing GTK operations - Idle.add(() => { - resolution_changed(width, height); - return Source.REMOVE; - }); - last_input_caps = caps; - } - - public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) { - if (sink == null) return; - detach(); - if (stream.media != "video") return; - connected_stream = stream as Stream?; - if (connected_stream == null) return; - plugin.pause(); - pipe.add(sink); - prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); - prepare.name = @"video_widget_$(id)_prepare"; - prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); - pipe.add(prepare); - connected_stream.add_output(prepare); - prepare.link(sink); - sink.set_locked_state(false); - plugin.unpause(); - attached = true; - } - - public void display_device(MediaDevice media_device) { - if (sink == null) return; - detach(); - connected_device = media_device as Device; - if (connected_device == null) return; - plugin.pause(); - pipe.add(sink); - prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); - prepare.name = @"video_widget_$(id)_prepare"; - if (prepare is Gst.Bin) { - ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed); - } else { - prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); - } - pipe.add(prepare); - connected_device_element = connected_device.link_source(); - connected_device_element.link(prepare); - prepare.link(sink); - sink.set_locked_state(false); - plugin.unpause(); - attached = true; - } - - public void detach() { - if (sink == null) return; - if (attached) { - debug("Detaching"); - if (connected_stream != null) { - connected_stream.remove_output(prepare); - connected_stream = null; - } - if (connected_device != null) { - connected_device_element.unlink(sink); - connected_device_element = null; - connected_device.unlink(); - connected_device = null; - } - prepare.set_locked_state(true); - prepare.set_state(Gst.State.NULL); - pipe.remove(prepare); - prepare = null; - sink.set_locked_state(true); - sink.set_state(Gst.State.NULL); - pipe.remove(sink); - attached = false; - } - } - - public override void dispose() { - detach(); - if (widget != null) widget.unparent(); - widget = null; - sink = null; - } -} diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala deleted file mode 100644 index 064684c5..00000000 --- a/plugins/rtp/src/voice_processor.vala +++ /dev/null @@ -1,209 +0,0 @@ -using Gst; - -namespace Dino.Plugins.Rtp { -public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); -} - -public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { - private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - public Audio.Info audio_info { get; private set; } - public signal void on_new_buffer(owned Buffer buffer); - public signal void on_new_delay(int delay); - private uint period_samples; - private uint period_size; - public int delay { get; private set; } - private Base.Adapter adapter = new Base.Adapter(); - - static construct { - add_static_pad_template(sink_template); - add_static_pad_template(src_template); - set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team "); - } - - construct { - set_passthrough(true); - } - - public override bool setup(Audio.Info info) { - audio_info = info; - period_samples = info.rate / 100; // 10ms buffers - period_size = period_samples * info.bpf; - return true; - } - - public override bool src_event(owned Event event) { - Query query = new Query.latency(); - if (event.type == EventType.LATENCY && srcpad != null && srcpad.query(query)) { - ClockTime upstream_latency; - query.parse_latency(null, out upstream_latency, null); - int delay = this.delay; - if (upstream_latency != CLOCK_TIME_NONE) { - delay = (int) (upstream_latency / MSECOND); - } else { - delay = 0; - } - if (delay != this.delay) { - debug("Delay adjusted from %ms to %dms", this.delay, delay); - this.delay = delay; - on_new_delay(delay); - } - } - return base.src_event((owned) event); - } - - public override FlowReturn transform_ip(Buffer buf) { - lock (adapter) { - adapter.push(adjust_to_running_time(this, buf)); - while (adapter.available() > period_size) { - on_new_buffer(adapter.take_buffer(period_size)); - } - } - return FlowReturn.OK; - } - - public override bool stop() { - adapter.clear(); - return true; - } -} - -public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { - private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - public Audio.Info audio_info { get; private set; } - private ulong process_outgoing_buffer_handler_id; - private ulong process_stream_delay_handler_id; - private uint adjust_delay_timeout_id; - private uint period_samples; - private uint period_size; - private Base.Adapter adapter = new Base.Adapter(); - private EchoProbe? echo_probe; - private Audio.StreamVolume? stream_volume; - private ClockTime last_reverse; - private void* native; - - static construct { - add_static_pad_template(sink_template); - add_static_pad_template(src_template); - set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team "); - } - - construct { - set_passthrough(false); - } - - public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { - this.echo_probe = echo_probe; - this.stream_volume = stream_volume; - } - - private static extern void* init_native(int stream_delay); - private static extern void destroy_native(void* native); - private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); - private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); - private static extern void adjust_stream_delay(void* native); - private static extern void set_stream_delay(void* native, int delay); - private static extern void notify_gain_level(void* native, int gain_level); - private static extern int get_suggested_gain_level(void* native); - private static extern bool get_stream_has_voice(void* native); - - public override bool setup(Audio.Info info) { - debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); - audio_info = info; - period_samples = info.rate / 100; // 10ms buffers - period_size = period_samples * info.bpf; - adapter.clear(); - return true; - } - - public override bool start() { - native = init_native(echo_probe.delay); - if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { - process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); - } - if (process_stream_delay_handler_id == 0 && echo_probe != null) { - process_stream_delay_handler_id = echo_probe.on_new_delay.connect(process_stream_delay); - } - if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { - stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; - } - return true; - } - - private bool adjust_delay() { - if (native != null) { - adjust_stream_delay(native); - return Source.CONTINUE; - } else { - adjust_delay_timeout_id = 0; - return Source.REMOVE; - } - } - - private void process_outgoing_buffer(owned Buffer buffer) { - if (buffer.pts != uint64.MAX) { - last_reverse = buffer.pts; - } - if (native != null) { - buffer = (Buffer) buffer.make_writable(); - analyze_reverse_stream(native, echo_probe.audio_info, buffer); - } - if (adjust_delay_timeout_id == 0 && echo_probe != null) { - adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); - } - } - - private void process_stream_delay(int stream_delay) { - if (native != null) { - set_stream_delay(native, stream_delay); - } - } - - public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { - lock (adapter) { - if (is_discont) { - adapter.clear(); - } - adapter.push(adjust_to_running_time(this, input)); - } - return FlowReturn.OK; - } - - public override FlowReturn generate_output(out Buffer output_buffer) { - lock (adapter) { - if (adapter.available() >= period_size) { - output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); - int old_gain_level = 0; - if (stream_volume != null) { - old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); - notify_gain_level(native, old_gain_level); - } - process_stream(native, audio_info, output_buffer); - if (stream_volume != null) { - int new_gain_level = get_suggested_gain_level(native); - if (old_gain_level != new_gain_level) { - debug("Gain: %i -> %i", old_gain_level, new_gain_level); - stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); - } - } - } - } - return FlowReturn.OK; - } - - public override bool stop() { - if (process_outgoing_buffer_handler_id != 0) { - echo_probe.disconnect(process_outgoing_buffer_handler_id); - process_outgoing_buffer_handler_id = 0; - } - if (adjust_delay_timeout_id != 0) { - Source.remove(adjust_delay_timeout_id); - adjust_delay_timeout_id = 0; - } - adapter.clear(); - destroy_native(native); - native = null; - return true; - } -} \ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp deleted file mode 100644 index 71cc8a54..00000000 --- a/plugins/rtp/src/voice_processor_native.cpp +++ /dev/null @@ -1,256 +0,0 @@ -#include -#include -#include -#if defined(WEBRTC0) -#include -#include -#include -#elif defined(WEBRTC1) || defined(WEBRTC2) -#include -#else -#error "Need to define WEBRTC0, WEBRTC1 or WEBRTC2" -#endif - -#define SAMPLE_RATE 48000 -#define SAMPLE_CHANNELS 1 - -struct _DinoPluginsRtpVoiceProcessorNative { -#if defined(WEBRTC0) - webrtc::AudioProcessing *apm; -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm; -#endif - gint stream_delay = 0; - gint last_median = 0; - gint last_poor_delays = 0; -}; - -extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { - GstBuffer *copy = gst_buffer_copy(buffer); - GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); - return copy; -} - -extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { - auto *native = new _DinoPluginsRtpVoiceProcessorNative(); - native->stream_delay = stream_delay; -#if defined(WEBRTC0) - webrtc::Config config; - config.Set(new webrtc::ExtendedFilter(true)); - config.Set(new webrtc::ExperimentalAgc(true, 85)); - native->apm = webrtc::AudioProcessing::Create(config); - - webrtc::AudioProcessing *apm = native->apm; - webrtc::ProcessingConfig pconfig; - pconfig.streams[webrtc::ProcessingConfig::kInputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - apm->Initialize(pconfig); - apm->high_pass_filter()->Enable(true); - apm->echo_cancellation()->enable_drift_compensation(false); - apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); - apm->echo_cancellation()->enable_delay_logging(true); - apm->echo_cancellation()->Enable(true); - apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); - apm->noise_suppression()->Enable(true); - apm->gain_control()->set_analog_level_limits(0, 255); - apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); - apm->gain_control()->set_target_level_dbfs(3); - apm->gain_control()->set_compression_gain_db(9); - apm->gain_control()->enable_limiter(true); - apm->gain_control()->Enable(true); - apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); - apm->voice_detection()->Enable(true); -#elif defined(WEBRTC1) || defined(WEBRTC2) - webrtc::AudioProcessing::Config config; - rtc::scoped_refptr apm = webrtc::AudioProcessingBuilder().Create(); - native->apm = apm; - config.high_pass_filter.enabled = true; - config.echo_canceller.enabled = true; - config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::Level::kModerate; - config.noise_suppression.enabled = true; - config.gain_controller1.target_level_dbfs = 3; - config.gain_controller1.compression_gain_db = 9; - config.gain_controller1.enable_limiter = true; - config.gain_controller1.enabled = true; -#ifdef WEBRTC1 - config.level_estimation.enabled = true; - config.voice_detection.enabled = true; -#endif - apm->ApplyConfig(config); -#endif - return native; -} - -extern "C" void -dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - int err; - -#if defined(WEBRTC0) - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); - webrtc::AudioProcessing *apm = native->apm; - GstMapInfo map; - gst_buffer_map(buffer, &map, GST_MAP_READ); - - webrtc::AudioFrame frame; - frame.num_channels_ = info->channels; - frame.sample_rate_hz_ = info->rate; - frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; - memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); - - apm->set_stream_delay_ms (native->stream_delay); - err = apm->AnalyzeReverseStream(&frame); - - gst_buffer_unmap(buffer, &map); -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm = native->apm; -#ifdef WEBRTC1 - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); -#else - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS); -#endif - GstAudioBuffer abuf; - if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) { - g_warning("voice_processor_native.cpp: analyze_reverse_stream: gst_audio_buffer_map failed"); - return; - } - - apm->set_stream_delay_ms (native->stream_delay); - auto * const data = (int16_t * const) abuf.planes[0]; - err = apm->ProcessReverseStream (data, config, config, data); - - gst_audio_buffer_unmap (&abuf); -#endif - if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); -} - -extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; -#if defined(WEBRTC0) - webrtc::AudioProcessing *apm = native->apm; - apm->gain_control()->set_stream_analog_level(gain_level); -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm = native->apm; - apm->set_stream_analog_level(gain_level); -#endif -} - -extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - int level = 0; -#if defined(WEBRTC0) - webrtc::AudioProcessing *apm = native->apm; - level = apm->gain_control()->stream_analog_level(); -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm = native->apm; - level = apm->recommended_stream_analog_level(); -#endif - return level; -} - -extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - bool has_voice = false; -#if defined(WEBRTC0) - webrtc::AudioProcessing *apm = native->apm; - has_voice = apm->voice_detection()->stream_has_voice(); -#elif defined(WEBRTC1) - rtc::scoped_refptr apm = native->apm; - webrtc::AudioProcessingStats stats = apm->GetStatistics (); - has_voice = stats.voice_detected.value_or(false); -#endif - return has_voice; -} - -extern "C" void dino_plugins_rtp_voice_processor_set_stream_delay(void *native_ptr, gint stream_delay) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - native->stream_delay = stream_delay; -} - -extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - int median, std, poor_delays; - float fraction_poor_delays; -#if defined(WEBRTC0) - webrtc::AudioProcessing *apm = native->apm; - apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); - poor_delays = (int)(fraction_poor_delays * 100.0); -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm = native->apm; - webrtc::AudioProcessingStats stats = apm->GetStatistics(); - median = stats.delay_median_ms.value_or(-1); - std = stats.delay_standard_deviation_ms.value_or(-1); - fraction_poor_delays = (float) stats.divergent_filter_fraction.value_or(-1.0); - poor_delays = (int) (fraction_poor_delays * 100.0); -#endif - if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; - g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); - native->last_median = median; - native->last_poor_delays = poor_delays; -#if defined(WEBRTC0) - if (poor_delays > 90) { - native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); - g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); - } -#endif -} - -extern "C" void -dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - int err; - -#if defined(WEBRTC0) - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); - webrtc::AudioProcessing *apm = native->apm; - GstMapInfo map; - gst_buffer_map(buffer, &map, GST_MAP_READWRITE); - - webrtc::AudioFrame frame; - frame.num_channels_ = info->channels; - frame.sample_rate_hz_ = info->rate; - frame.samples_per_channel_ = info->rate / 100; - memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); - - apm->set_stream_delay_ms(native->stream_delay); - err = apm->ProcessStream(&frame); - if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); - - gst_buffer_unmap(buffer, &map); -#elif defined(WEBRTC1) || defined(WEBRTC2) - rtc::scoped_refptr apm = native->apm; -#ifdef WEBRTC1 - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); -#else - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS); -#endif - GstAudioBuffer abuf; - if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) { - g_warning("voice_processor_native.cpp: process_stream: gst_audio_buffer_map failed"); - return; - } - - auto * const data = (int16_t * const) abuf.planes[0]; - err = apm->ProcessStream (data, config, config, data); - - gst_audio_buffer_unmap (&abuf); -#endif - if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); -} - -extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { - auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; -#if defined(WEBRTC0) - delete native->apm; - native->apm = NULL; -#elif defined(WEBRTC1) || defined(WEBRTC2) - native->apm = nullptr; -#endif - delete native; -} \ No newline at end of file -- cgit v1.3