Merge 'upstream/master'

1 files changed, 293 insertions, 0 deletions

diff --git a/main/src/plugins/rtp/video_widget.vala b/main/src/plugins/rtp/video_widget.vala
new file mode 100644
index 00000000..af6eb14e
--- /dev/null
+++ b/main/src/plugins/rtp/video_widget.vala
@@ -0,0 +1,293 @@
+private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame);
+[CCode (array_length_type = "size_t", type = "void*")]
+private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame);
+
+public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object {
+    private Gdk.Paintable image;
+    private double pixel_aspect_ratio;
+
+    public override Gdk.PaintableFlags get_flags() {
+        return 0;
+    }
+
+    public void snapshot(Gdk.Snapshot snapshot, double width, double height) {
+        if (image != null) image.snapshot(snapshot, width, height);
+    }
+
+    public override Gdk.Paintable get_current_image() {
+        if (image != null) return image;
+        return Gdk.Paintable.empty(0, 0);
+    }
+
+    public override int get_intrinsic_width() {
+        if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width());
+        return 0;
+    }
+
+    public override int get_intrinsic_height() {
+        if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height());
+        return 0;
+    }
+
+    public override double get_intrinsic_aspect_ratio() {
+        if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio();
+        return 0.0;
+    }
+
+    public override void dispose() {
+        image = null;
+        base.dispose();
+    }
+
+    private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) {
+        if (paintable == image) return;
+        bool size_changed = image == null ||
+                this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() ||
+                image.get_intrinsic_height() != paintable.get_intrinsic_height() ||
+                image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio();
+
+        if (image != null) this.image.dispose();
+        this.image = paintable;
+        this.pixel_aspect_ratio = pixel_aspect_ratio;
+
+        if (size_changed) invalidate_size();
+        invalidate_contents();
+    }
+
+    public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) {
+        Idle.add(() => {
+            set_paintable(texture, pixel_aspect_ratio);
+            return Source.REMOVE;
+        }, Priority.DEFAULT);
+    }
+}
+
+public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink {
+    internal Paintable paintable = new Paintable();
+    private Gst.Video.Info info = new Gst.Video.Info();
+
+    class construct {
+        set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team <team@dino.im>");
+        add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }")));
+    }
+
+    construct {
+        set_drop_out_of_segment(false);
+    }
+
+    public override bool set_info(Gst.Caps caps, Gst.Video.Info info) {
+        this.info = info;
+        return true;
+    }
+
+    public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) {
+        if (buffer.pts != -1) {
+            start = buffer.pts;
+            if (buffer.duration != -1) {
+                end = start + buffer.duration;
+            } else if (info.fps_n > 0) {
+                end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n);
+            }
+        }
+    }
+
+    public override Gst.Caps get_caps(Gst.Caps? filter) {
+        Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }");
+
+        if (filter != null) {
+            return filter.intersect(caps, Gst.CapsIntersectMode.FIRST);
+        } else {
+            return caps;
+        }
+    }
+
+    private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) {
+        switch (format) {
+            case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8;
+            case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8;
+            case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8;
+            case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8;
+            case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8;
+            case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8;
+            default:
+                warning("Unsupported video format: %s", format.to_string());
+                return Gdk.MemoryFormat.A8R8G8B8;
+        }
+    }
+
+    private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) {
+        Gst.Video.Frame frame = Gst.Video.Frame();
+        Gdk.Texture texture;
+
+        if (frame.map(info, buffer, Gst.MapFlags.READ)) {
+            unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame);
+            Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame));
+            texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]);
+            pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d);
+            frame.unmap();
+        } else {
+            texture = null;
+        }
+        return texture;
+    }
+
+    private void queue_buffer(Gst.Buffer buf) {
+        double pixel_aspect_ratio;
+        Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio);
+        if (texture != null) {
+            paintable.queue_set_texture(texture, pixel_aspect_ratio);
+        }
+    }
+
+    public override Gst.FlowReturn show_frame(Gst.Buffer buf) {
+        @lock.lock();
+        queue_buffer(buf);
+        @lock.unlock();
+
+        return Gst.FlowReturn.OK;
+    }
+}
+
+public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget {
+    private const int RECAPS_AFTER_CHANGE = 5;
+    private static uint last_id = 0;
+
+    public uint id { get; private set; }
+    public Plugin plugin { get; private set; }
+    public Gst.Pipeline pipe { get {
+        return plugin.pipe;
+    }}
+
+    private bool attached;
+    private Device? connected_device;
+    private Gst.Element? connected_device_element;
+    private Stream? connected_stream;
+    private Gst.Element prepare;
+    private Gst.Caps last_input_caps;
+    private Gst.Caps last_caps;
+    private int recaps_since_change;
+    private Sink sink;
+    private Gtk.Picture widget;
+
+    private static uint active_widgets = 0;
+
+    public static bool has_instances() {
+        return active_widgets > 0;
+    }
+
+    private static void notify_weak(Object widget_object) {
+        if (active_widgets > 0) {
+            debug("Video widget %p destroyed. left=%u", widget_object, active_widgets);
+            active_widgets--;
+        }
+        if (active_widgets == 0 && widget_object is VideoWidget) {
+            ((VideoWidget)widget_object).plugin.destroy_call_pipe_if_unused();
+        }
+    }
+
+    public VideoWidget(Plugin plugin) {
+        this.plugin = plugin;
+        this.layout_manager = new Gtk.BinLayout();
+
+        id = last_id++;
+        sink = new Sink() { async = false, sync = true };
+        widget = new Gtk.Picture.for_paintable(sink.paintable);
+        widget.insert_after(this, null);
+        active_widgets++;
+        debug("Video widget %p created. total=%u", this, active_widgets);
+        this.weak_ref(notify_weak);
+    }
+
+    public void input_caps_changed(GLib.Object pad, ParamSpec spec) {
+        Gst.Caps? caps = ((Gst.Pad)pad).caps;
+        if (caps == null) {
+            debug("Input: No caps");
+            return;
+        }
+
+        int width, height;
+        caps.get_structure(0).get_int("width", out width);
+        caps.get_structure(0).get_int("height", out height);
+        debug("Input resolution changed: %ix%i", width, height);
+        // Invoke signal on GTK main loop as recipients are likely to use it for doing GTK operations
+        Idle.add(() => {
+            resolution_changed(width, height);
+            return Source.REMOVE;
+        });
+        last_input_caps = caps;
+    }
+
+    public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) {
+        if (sink == null) return;
+        detach();
+        if (stream.media != "video") return;
+        connected_stream = stream as Stream?;
+        if (connected_stream == null) return;
+        plugin.pause();
+        pipe.add(sink);
+        prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true);
+        prepare.name = @"video_widget_$(id)_prepare";
+        prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+        pipe.add(prepare);
+        connected_stream.add_output(prepare);
+        prepare.link(sink);
+        sink.set_locked_state(false);
+        plugin.unpause();
+        attached = true;
+    }
+
+    public void display_device(MediaDevice media_device) {
+        if (sink == null) return;
+        detach();
+        connected_device = media_device as Device;
+        if (connected_device == null) return;
+        plugin.pause();
+        pipe.add(sink);
+        prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
+        prepare.name = @"video_widget_$(id)_prepare";
+        if (prepare is Gst.Bin) {
+            ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+        } else {
+            prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+        }
+        pipe.add(prepare);
+        connected_device_element = connected_device.link_source();
+        connected_device_element.link(prepare);
+        prepare.link(sink);
+        sink.set_locked_state(false);
+        plugin.unpause();
+        attached = true;
+    }
+
+    public void detach() {
+        if (sink == null) return;
+        if (attached) {
+            debug("Detaching");
+            if (connected_stream != null) {
+                connected_stream.remove_output(prepare);
+                connected_stream = null;
+            }
+            if (connected_device != null) {
+                connected_device_element.unlink(sink);
+                connected_device_element = null;
+                connected_device.unlink();
+                connected_device = null;
+            }
+            prepare.set_locked_state(true);
+            prepare.set_state(Gst.State.NULL);
+            pipe.remove(prepare);
+            prepare = null;
+            sink.set_locked_state(true);
+            sink.set_state(Gst.State.NULL);
+            pipe.remove(sink);
+            attached = false;
+        }
+    }
+
+    public override void dispose() {
+        detach();
+        if (widget != null) widget.unparent();
+        widget = null;
+        sink = null;
+    }
+}