aboutsummaryrefslogtreecommitdiff
path: root/main/src/plugins
diff options
context:
space:
mode:
authorMarvin W <git@larma.de>2026-10-04 14:42:16 +0200
committerMarvin W <git@larma.de>2026-10-23 16:59:59 +0200
commitdba577f39c489b9db29960f2bf5cc78ac4f29d7b (patch)
treea8354c9fc328c1442d5e8c5c2adec4b428c92f44 /main/src/plugins
parent2413442905e31f34cb0f77c9167003b3725760fe (diff)
Move OMEMO, OpenPGP and RTP plugins into main/libdino
Diffstat (limited to 'main/src/plugins')
-rw-r--r--main/src/plugins/omemo/dtls_srtp_verification_draft.vala268
-rw-r--r--main/src/plugins/omemo/file_transfer/file_decryptor.vala90
-rw-r--r--main/src/plugins/omemo/file_transfer/file_encryptor.vala76
-rw-r--r--main/src/plugins/omemo/jingle/jet_omemo.vala115
-rw-r--r--main/src/plugins/omemo/jingle/jingle_helper.vala53
-rw-r--r--main/src/plugins/omemo/logic/database.vala284
-rw-r--r--main/src/plugins/omemo/logic/decrypt.vala212
-rw-r--r--main/src/plugins/omemo/logic/encrypt.vala132
-rw-r--r--main/src/plugins/omemo/logic/manager.vala431
-rw-r--r--main/src/plugins/omemo/logic/pre_key_store.vala46
-rw-r--r--main/src/plugins/omemo/logic/session_store.vala50
-rw-r--r--main/src/plugins/omemo/logic/signed_pre_key_store.vala46
-rw-r--r--main/src/plugins/omemo/logic/trust_manager.vala142
-rw-r--r--main/src/plugins/omemo/plugin.vala87
-rw-r--r--main/src/plugins/omemo/protocol/bundle.vala88
-rw-r--r--main/src/plugins/omemo/protocol/message_flag.vala23
-rw-r--r--main/src/plugins/omemo/protocol/stream_module.vala332
-rw-r--r--main/src/plugins/omemo/trust_level.vala15
-rw-r--r--main/src/plugins/omemo/ui/bad_messages_populator.vala198
-rw-r--r--main/src/plugins/omemo/ui/call_encryption_entry.vala58
-rw-r--r--main/src/plugins/omemo/ui/contact_details_provider.vala28
-rw-r--r--main/src/plugins/omemo/ui/device_notification_populator.vala90
-rw-r--r--main/src/plugins/omemo/ui/encryption_list_entry.vala78
-rw-r--r--main/src/plugins/omemo/ui/encryption_preferences_entry.vala320
-rw-r--r--main/src/plugins/omemo/ui/manage_key_dialog.vala178
-rw-r--r--main/src/plugins/omemo/ui/own_notifications.vala35
-rw-r--r--main/src/plugins/omemo/ui/util.vala40
-rw-r--r--main/src/plugins/openpgp/contact_details_provider.vala51
-rw-r--r--main/src/plugins/openpgp/database.vala137
-rw-r--r--main/src/plugins/openpgp/encryption_list_entry.vala72
-rw-r--r--main/src/plugins/openpgp/encryption_preferences_entry.vala82
-rw-r--r--main/src/plugins/openpgp/file_transfer/file_decryptor.vala50
-rw-r--r--main/src/plugins/openpgp/file_transfer/file_encryptor.vala45
-rw-r--r--main/src/plugins/openpgp/gpgme_fix.c12
-rw-r--r--main/src/plugins/openpgp/gpgme_fix.h12
-rw-r--r--main/src/plugins/openpgp/gpgme_helper.vala185
-rw-r--r--main/src/plugins/openpgp/manager.vala123
-rw-r--r--main/src/plugins/openpgp/plugin.vala47
-rw-r--r--main/src/plugins/openpgp/stream_module.vala159
-rw-r--r--main/src/plugins/openpgp/util.vala29
-rw-r--r--main/src/plugins/rtp/codec_util.vala418
-rw-r--r--main/src/plugins/rtp/device.vala592
-rw-r--r--main/src/plugins/rtp/gst_fixes.c10
-rw-r--r--main/src/plugins/rtp/module.vala235
-rw-r--r--main/src/plugins/rtp/participant.vala39
-rw-r--r--main/src/plugins/rtp/plugin.vala504
-rw-r--r--main/src/plugins/rtp/stream.vala851
-rw-r--r--main/src/plugins/rtp/video_widget.vala293
-rw-r--r--main/src/plugins/rtp/voice_processor.vala209
-rw-r--r--main/src/plugins/rtp/voice_processor_native.cpp256
50 files changed, 7926 insertions, 0 deletions
diff --git a/main/src/plugins/omemo/dtls_srtp_verification_draft.vala b/main/src/plugins/omemo/dtls_srtp_verification_draft.vala
new file mode 100644
index 00000000..09617a2a
--- /dev/null
+++ b/main/src/plugins/omemo/dtls_srtp_verification_draft.vala
@@ -0,0 +1,268 @@
+using Omemo;
+using Gee;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft {
+ public const string NS_URI = "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification";
+
+ public class StreamModule : XmppStreamModule {
+
+ public static Xmpp.ModuleIdentity<StreamModule> IDENTITY = new Xmpp.ModuleIdentity<StreamModule>(NS_URI, "dtls_srtp_omemo_verification_draft");
+
+ private VerificationSendListener send_listener = new VerificationSendListener();
+ private HashMap<string, int> device_id_by_jingle_sid = new HashMap<string, int>();
+ private HashMap<string, int> device_id_by_muji_member = new HashMap<string, int>();
+ private HashMap<string, Gee.List<string>> content_names_by_jingle_sid = new HashMap<string, Gee.List<string>>();
+
+ private void on_preprocess_incoming_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) {
+ if (iq.type_ != Iq.Stanza.TYPE_SET) return;
+
+ Gee.List<StanzaNode> content_nodes = iq.stanza.get_deep_subnodes(Xep.Jingle.NS_URI + ":jingle", Xep.Jingle.NS_URI + ":content");
+ if (content_nodes.size == 0) return;
+
+ string? jingle_sid = iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "sid");
+ if (jingle_sid == null) return;
+
+ Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY);
+
+ foreach (StanzaNode content_node in content_nodes) {
+ string? content_name = content_node.get_attribute("name");
+ if (content_name == null) continue;
+ StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI);
+ if (transport_node == null) continue;
+ StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", NS_URI);
+ if (fingerprint_node == null) continue;
+ StanzaNode? encrypted_node = fingerprint_node.get_subnode("encrypted", Omemo.NS_URI);
+ if (encrypted_node == null) continue;
+
+ Xep.Omemo.ParsedData? parsed_data = decryptor.parse_node(encrypted_node);
+ if (parsed_data == null || parsed_data.ciphertext == null) continue;
+
+ if (device_id_by_jingle_sid.has_key(jingle_sid) && device_id_by_jingle_sid[jingle_sid] != parsed_data.sid) {
+ warning("Expected DTLS fingerprint to be OMEMO encrypted from %s %d, but it was from %d", iq.from.to_string(), device_id_by_jingle_sid[jingle_sid], parsed_data.sid);
+ }
+
+ foreach (Bytes encr_key in parsed_data.our_potential_encrypted_keys.keys) {
+ parsed_data.is_prekey = parsed_data.our_potential_encrypted_keys[encr_key];
+ parsed_data.encrypted_key = encr_key.get_data();
+
+ try {
+ uint8[] key = decryptor.decrypt_key(parsed_data, iq.from.bare_jid);
+ string cleartext = decryptor.decrypt(parsed_data.ciphertext, key, parsed_data.iv);
+
+ StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI).add_self_xmlns()
+ .put_node(new StanzaNode.text(cleartext));
+ string? hash_attr = fingerprint_node.get_attribute("hash", NS_URI);
+ string? setup_attr = fingerprint_node.get_attribute("setup", NS_URI);
+ if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr);
+ if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr);
+ transport_node.put_node(new_fingerprint_node);
+
+ device_id_by_jingle_sid[jingle_sid] = parsed_data.sid;
+ if (!content_names_by_jingle_sid.has_key(content_name)) {
+ content_names_by_jingle_sid[content_name] = new ArrayList<string>();
+ }
+ content_names_by_jingle_sid[content_name].add(content_name);
+
+ stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => {
+ Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res);
+ if (session == null || !session.contents_map.has_key(content_name)) return;
+ var encryption = new OmemoContentEncryption(NS_URI, "OMEMO", iq.from.bare_jid, device_id_by_jingle_sid[jingle_sid]);
+ session.contents_map[content_name].encryptions[NS_URI] = encryption;
+
+ if (iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "action") == "session-accept") {
+ session.additional_content_add_incoming.connect(on_content_add_received);
+ }
+ });
+
+ break;
+ } catch (Error e) {
+ debug("Decrypting message from %s/%d failed: %s", iq.from.bare_jid.to_string(), parsed_data.sid, e.message);
+ }
+ }
+ }
+ }
+
+ private void on_preprocess_outgoing_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) {
+ if (iq.type_ != Iq.Stanza.TYPE_SET) return;
+
+ StanzaNode? jingle_node = iq.stanza.get_subnode("jingle", Xep.Jingle.NS_URI);
+ if (jingle_node == null) return;
+
+ int device_id = -1;
+ string? sid = jingle_node.get_attribute("sid", Xep.Jingle.NS_URI);
+ if (sid != null && device_id_by_jingle_sid.has_key(sid)) {
+ device_id = device_id_by_jingle_sid[sid];
+ }
+
+ StanzaNode? muji_node = jingle_node.get_subnode("muji", Xep.Muji.NS_URI);
+ if (muji_node != null) {
+ string muji_room = muji_node.get_attribute("room");
+ try {
+ Jid muji_jid = Jid.from_string(muji_room);
+ if (device_id_by_muji_member.has_key(@"$(muji_jid.bare_jid)/$(iq.to)")) {
+ device_id = device_id_by_muji_member[@"$(muji_jid.bare_jid)/$(iq.to)"];
+ }
+ } catch (InvalidJidError e) {
+ // Ignore
+ }
+ }
+
+ if (device_id == -1) return;
+
+ Gee.List<StanzaNode> content_nodes = jingle_node.get_subnodes("content", Xep.Jingle.NS_URI);
+ if (content_nodes.size == 0) return;
+
+ foreach (StanzaNode content_node in content_nodes) {
+ StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI);
+ if (transport_node == null) continue;
+ StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI);
+ if (fingerprint_node == null) continue;
+ string fingerprint = fingerprint_node.get_deep_string_content();
+
+ StanzaNode? encrypted_node = null;
+ try {
+ Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY);
+ Xep.Omemo.EncryptionData enc_data = encryptor.encrypt_plaintext(fingerprint);
+ encryptor.encrypt_key(enc_data, iq.to.bare_jid, device_id);
+ encrypted_node = enc_data.get_encrypted_node();
+ } catch (Error e) {
+ warning("Error while OMEMO-encrypting call keys: %s", e.message);
+ return;
+ }
+
+ StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", NS_URI).add_self_xmlns().put_node(encrypted_node);
+ string? hash_attr = fingerprint_node.get_attribute("hash", Xep.JingleIceUdp.DTLS_NS_URI);
+ string? setup_attr = fingerprint_node.get_attribute("setup", Xep.JingleIceUdp.DTLS_NS_URI);
+ if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr);
+ if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr);
+ transport_node.put_node(new_fingerprint_node);
+
+ transport_node.sub_nodes.remove(fingerprint_node);
+ }
+ }
+
+ private void on_message_received(XmppStream stream, Xmpp.MessageStanza message) {
+ StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI);
+ if (proceed_node == null) return;
+
+ string? jingle_sid = proceed_node.get_attribute("id");
+ if (jingle_sid == null) return;
+
+ StanzaNode? device_node = proceed_node.get_subnode("device", NS_URI);
+ if (device_node == null) return;
+
+ int device_id = device_node.get_attribute_int("id", -1);
+ if (device_id == -1) return;
+
+ device_id_by_jingle_sid[jingle_sid] = device_id;
+ }
+
+ private void on_session_initiate_received(XmppStream stream, Xep.Jingle.Session session) {
+ if (device_id_by_jingle_sid.has_key(session.sid)) {
+ foreach (Xep.Jingle.Content content in session.contents) {
+ on_content_add_received(stream, content);
+ }
+ }
+ session.additional_content_add_incoming.connect(on_content_add_received);
+ }
+
+ private void on_content_add_received(XmppStream stream, Xep.Jingle.Content content) {
+ if (!content_names_by_jingle_sid.has_key(content.session.sid) || content_names_by_jingle_sid[content.session.sid].contains(content.content_name)) {
+ var encryption = new OmemoContentEncryption(NS_URI, "OMEMO", content.peer_full_jid.bare_jid, device_id_by_jingle_sid[content.session.sid]);
+ content.encryptions[encryption.encryption_ns] = encryption;
+ }
+ }
+
+ private void on_pre_send_presence_stanza(XmppStream stream, Presence.Stanza presence) {
+ StanzaNode? muji_node = presence.stanza.get_subnode("muji", Xep.Muji.NS_URI);
+ if (muji_node == null) return;
+
+ StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns()
+ .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string());
+ muji_node.put_node(device_node);
+ }
+
+ private void on_received_available(XmppStream stream, Presence.Stanza presence) {
+ StanzaNode? muji_node = presence.stanza.get_subnode("muji", Xep.Muji.NS_URI);
+ if (muji_node == null) return;
+
+ StanzaNode? device_node = muji_node.get_subnode("device", NS_URI);
+ if (device_node == null) return;
+
+ int device_id = device_node.get_attribute_int("id", -1);
+ if (device_id == -1) return;
+
+ StanzaNode? muc_x_node = presence.stanza.get_subnode("x", "http://jabber.org/protocol/muc#user");
+ if (muc_x_node == null) return;
+
+ StanzaNode? item_node = muc_x_node.get_subnode("item");
+ if (item_node == null) return;
+
+ Jid? real_jid = null;
+ try {
+ string jid_attribute = item_node.get_attribute("jid");
+ if (jid_attribute == null) return;
+ real_jid = Jid.from_string(jid_attribute);
+ } catch (InvalidJidError e) {
+ // Ignore
+ return;
+ }
+
+ device_id_by_muji_member[@"$(presence.from.bare_jid)/$(real_jid)"] = device_id;
+ }
+
+ public override void attach(XmppStream stream) {
+ stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.connect(on_message_received);
+ stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.connect(send_listener);
+ stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.connect(on_preprocess_incoming_iq_set_get);
+ stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.connect(on_preprocess_outgoing_iq_set_get);
+ stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.connect(on_session_initiate_received);
+ stream.get_module(Xmpp.Presence.Module.IDENTITY).pre_send_presence_stanza.connect(on_pre_send_presence_stanza);
+ stream.get_module(Xmpp.Presence.Module.IDENTITY).received_available.connect(on_received_available);
+ }
+
+ public override void detach(XmppStream stream) {
+ stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.disconnect(on_message_received);
+ stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.disconnect(send_listener);
+ stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.disconnect(on_preprocess_incoming_iq_set_get);
+ stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.disconnect(on_preprocess_outgoing_iq_set_get);
+ stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.disconnect(on_session_initiate_received);
+ stream.get_module(Xmpp.Presence.Module.IDENTITY).received_available.disconnect(on_received_available);
+ }
+
+ public override string get_ns() { return NS_URI; }
+
+ public override string get_id() { return IDENTITY.id; }
+ }
+
+ public class VerificationSendListener : StanzaListener<MessageStanza> {
+
+ private string[] after_actions_const = {};
+
+ public override string action_group { get { return "REWRITE_NODES"; } }
+ public override string[] after_actions { get { return after_actions_const; } }
+
+ public override async bool run(XmppStream stream, MessageStanza message) {
+ StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI);
+ if (proceed_node == null) return false;
+
+ StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns()
+ .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string());
+ proceed_node.put_node(device_node);
+ return false;
+ }
+ }
+
+ public class OmemoContentEncryption : Xep.Jingle.ContentEncryption {
+ public Jid jid { get; set; }
+ public int sid { get; set; }
+
+ public OmemoContentEncryption(string encryption_ns, string encryption_name, Jid jid, int sid) {
+ base(encryption_ns, encryption_name);
+ this.jid = jid;
+ this.sid = sid;
+ }
+ }
+}
+
diff --git a/main/src/plugins/omemo/file_transfer/file_decryptor.vala b/main/src/plugins/omemo/file_transfer/file_decryptor.vala
new file mode 100644
index 00000000..8ae0b96b
--- /dev/null
+++ b/main/src/plugins/omemo/file_transfer/file_decryptor.vala
@@ -0,0 +1,90 @@
+using Dino.Entities;
+
+using Dino.Crypto;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+public class OmemoHttpFileReceiveData : HttpFileReceiveData {
+ public string original_url;
+}
+
+public class OmemoFileDecryptor : FileDecryptor, Object {
+
+ private Regex url_regex = /^aesgcm:\/\/(.*)#(([A-Fa-f0-9]{2}){48}|([A-Fa-f0-9]{2}){44})$/;
+
+ public Encryption get_encryption() {
+ return Encryption.OMEMO;
+ }
+
+ public FileReceiveData prepare_get_meta_info(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) {
+ HttpFileReceiveData? http_receive_data = receive_data as HttpFileReceiveData;
+ if (http_receive_data == null) assert(false);
+ if ((receive_data as OmemoHttpFileReceiveData) != null) return receive_data;
+
+ var omemo_http_receive_data = new OmemoHttpFileReceiveData();
+ omemo_http_receive_data.url = aesgcm_to_https_link(http_receive_data.url);
+ omemo_http_receive_data.original_url = http_receive_data.url;
+
+ return omemo_http_receive_data;
+ }
+
+ public FileMeta prepare_download_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) {
+ if (file_meta.file_name != null) {
+ file_meta.file_name = file_meta.file_name.split("#")[0];
+ }
+ return file_meta;
+ }
+
+ public bool can_decrypt_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) {
+ HttpFileReceiveData? http_file_receive = receive_data as HttpFileReceiveData;
+ if (http_file_receive == null) return false;
+
+ return this.url_regex.match(http_file_receive.url) || (receive_data as OmemoHttpFileReceiveData) != null;
+ }
+
+ public async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError {
+ const uint KEY_SIZE = 32;
+ try {
+ OmemoHttpFileReceiveData? omemo_http_receive_data = receive_data as OmemoHttpFileReceiveData;
+ if (omemo_http_receive_data == null) assert(false);
+
+ // Decode IV and key
+ MatchInfo match_info;
+ this.url_regex.match(omemo_http_receive_data.original_url, 0, out match_info);
+ uint8[] iv_and_key = hex_to_bin(match_info.fetch(2).up());
+ uint8[] iv = iv_and_key[0:iv_and_key.length-KEY_SIZE];
+ uint8[] key = iv_and_key[iv_and_key.length-KEY_SIZE:iv_and_key.length];
+
+ file_transfer.encryption = Encryption.OMEMO;
+ debug("Decrypting file %s from %s", file_transfer.file_name, file_transfer.server_file_name);
+
+ SymmetricCipherDecryption cipher = create_symmetric_cipher_decryption("AES-GCM");
+ cipher.set_key(key);
+ cipher.set_iv(iv);
+ return new ConverterInputStream(encrypted_stream, new SymmetricCipherDecrypter((owned) cipher, 16));
+
+ } catch (Crypto.Error e) {
+ throw new FileReceiveError.DECRYPTION_FAILED("OMEMO file decryption error: %s".printf(e.message));
+ } catch (GLib.Error e) {
+ throw new FileReceiveError.DECRYPTION_FAILED("OMEMO file decryption error: %s".printf(e.message));
+ }
+ }
+
+ private uint8[] hex_to_bin(string hex) {
+ uint8[] bin = new uint8[hex.length / 2];
+ const string HEX = "0123456789ABCDEF";
+ for (int i = 0; i < hex.length / 2; i++) {
+ bin[i] = (uint8) (HEX.index_of_char(hex[i*2]) << 4) | HEX.index_of_char(hex[i*2+1]);
+ }
+ return bin;
+ }
+
+ private string aesgcm_to_https_link(string aesgcm_link) {
+ MatchInfo match_info;
+ this.url_regex.match(aesgcm_link, 0, out match_info);
+ return "https://" + match_info.fetch(1);
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/file_transfer/file_encryptor.vala b/main/src/plugins/omemo/file_transfer/file_encryptor.vala
new file mode 100644
index 00000000..5f1d6e69
--- /dev/null
+++ b/main/src/plugins/omemo/file_transfer/file_encryptor.vala
@@ -0,0 +1,76 @@
+using Gee;
+using Gtk;
+
+using Dino.Crypto;
+using Dino.Entities;
+using Xmpp;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+public class OmemoHttpFileMeta : HttpFileMeta {
+ public uint8[] iv;
+ public uint8[] key;
+}
+
+public class OmemoFileEncryptor : Dino.FileEncryptor, Object {
+
+ public bool can_encrypt_file(Conversation conversation, FileTransfer file_transfer) {
+ return file_transfer.encryption == Encryption.OMEMO;
+ }
+
+ public FileMeta encrypt_file(Conversation conversation, FileTransfer file_transfer) throws FileSendError {
+ const uint KEY_SIZE = 32;
+ const uint IV_SIZE = 12;
+ var omemo_http_file_meta = new OmemoHttpFileMeta();
+
+ try {
+ //Create a key and use it to encrypt the file
+ uint8[] iv = new uint8[IV_SIZE];
+ Crypto.randomize(iv);
+ uint8[] key = new uint8[KEY_SIZE];
+ Crypto.randomize(key);
+
+ SymmetricCipherEncryption cipher = create_symmetric_cipher_encryption("AES-GCM");
+ cipher.set_key(key);
+ cipher.set_iv(iv);
+
+ omemo_http_file_meta.iv = iv;
+ omemo_http_file_meta.key = key;
+ omemo_http_file_meta.size = file_transfer.size + 16;
+ omemo_http_file_meta.content_type = new FileContentType.from_mime_type("application/octet-stream");
+ file_transfer.input_stream = new ConverterInputStream(file_transfer.input_stream, new SymmetricCipherEncrypter((owned) cipher, 16));
+ } catch (Crypto.Error error) {
+ throw new FileSendError.ENCRYPTION_FAILED("OMEMO file encryption error: %s".printf(error.message));
+ } catch (GLib.Error error) {
+ throw new FileSendError.ENCRYPTION_FAILED("OMEMO file encryption error: %s".printf(error.message));
+ }
+
+ debug("Encrypting file %s as %s", file_transfer.file_name, file_transfer.server_file_name);
+
+ return omemo_http_file_meta;
+ }
+
+ public FileSendData? preprocess_send_file(Conversation conversation, FileTransfer file_transfer, FileSendData file_send_data, FileMeta file_meta) {
+ HttpFileSendData? send_data = file_send_data as HttpFileSendData;
+ if (send_data == null) return null;
+
+ OmemoHttpFileMeta? omemo_http_file_meta = file_meta as OmemoHttpFileMeta;
+ if (omemo_http_file_meta == null) return null;
+
+ // Convert iv and key to hex
+ string iv_and_key = "";
+ foreach (uint8 byte in omemo_http_file_meta.iv) iv_and_key += byte.to_string("%02x");
+ foreach (uint8 byte in omemo_http_file_meta.key) iv_and_key += byte.to_string("%02x");
+
+ string aesgcm_link = send_data.url_down + "#" + iv_and_key;
+ aesgcm_link = "aesgcm://" + aesgcm_link.substring(8); // replace https:// by aesgcm://
+
+ send_data.url_down = aesgcm_link;
+ send_data.encrypt_message = true;
+
+ return file_send_data;
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/jingle/jet_omemo.vala b/main/src/plugins/omemo/jingle/jet_omemo.vala
new file mode 100644
index 00000000..07d03f64
--- /dev/null
+++ b/main/src/plugins/omemo/jingle/jet_omemo.vala
@@ -0,0 +1,115 @@
+using Dino.Crypto;
+using Dino.Crypto.Omemo;
+using Dino;
+using Dino.Entities;
+using Gee;
+using Omemo;
+using Xmpp;
+using Xmpp.Xep;
+
+namespace Dino.Plugins.JetOmemo {
+
+private const string NS_URI = "urn:xmpp:jingle:jet-omemo:0";
+private const string AES_128_GCM_URI = "urn:xmpp:ciphers:aes-128-gcm-nopadding";
+
+public class Module : XmppStreamModule, Jet.EnvelopEncoding {
+ public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0396_jet_omemo");
+ const uint KEY_SIZE = 16;
+ const uint IV_SIZE = 12;
+
+ public override void attach(XmppStream stream) {
+ if (stream.get_module(Jet.Module.IDENTITY) != null) {
+ stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI);
+ stream.get_module(Jet.Module.IDENTITY).register_envelop_encoding(this);
+ stream.get_module(Jet.Module.IDENTITY).register_cipher(new AesGcmCipher(KEY_SIZE, IV_SIZE, AES_128_GCM_URI));
+ }
+ }
+
+ public override void detach(XmppStream stream) {
+ stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI);
+ }
+
+ public async bool is_available(XmppStream stream, Jid full_jid) {
+ bool? has_feature = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI);
+ if (has_feature == null || !(!)has_feature) {
+ return false;
+ }
+ return yield stream.get_module(Xep.Jet.Module.IDENTITY).is_available(stream, full_jid);
+ }
+
+ public string get_type_uri() {
+ return Omemo.NS_URI;
+ }
+
+ public Jet.TransportSecret decode_envolop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws Jingle.IqError {
+ StanzaNode? encrypted = security.get_subnode("encrypted", Omemo.NS_URI);
+ if (encrypted == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing encrypted element");
+
+ Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY);
+
+ Xmpp.Xep.Omemo.ParsedData? data = decryptor.parse_node(encrypted);
+ if (data == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: bad encrypted element");
+
+ foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) {
+ data.is_prekey = data.our_potential_encrypted_keys[encr_key];
+ data.encrypted_key = encr_key.get_data();
+
+ try {
+ uint8[] key = decryptor.decrypt_key(data, peer_full_jid.bare_jid);
+ return new Jet.TransportSecret(key, data.iv);
+ } catch (GLib.Error e) {
+ debug("Decrypting JET key from %s/%d failed: %s", peer_full_jid.bare_jid.to_string(), data.sid, e.message);
+ }
+ }
+ throw new Jingle.IqError.NOT_ACCEPTABLE("Not encrypted for targeted device");
+ }
+
+ public void encode_envelop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Jet.SecurityParameters security_params, StanzaNode security) {
+ Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store;
+
+ var encryption_data = new Xep.Omemo.EncryptionData(store.local_registration_id);
+ encryption_data.iv = security_params.secret.initialization_vector;
+ encryption_data.keytag = security_params.secret.transport_key;
+ Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY);
+ encryptor.encrypt_key_to_recipient(stream, encryption_data, peer_full_jid.bare_jid);
+
+ security.put_node(encryption_data.get_encrypted_node());
+ }
+
+ public override string get_ns() { return NS_URI; }
+ public override string get_id() { return IDENTITY.id; }
+}
+
+public class AesGcmCipher : Jet.Cipher, Object {
+ private uint key_size;
+ private uint default_iv_size;
+ private string uri;
+ public AesGcmCipher(uint key_size, uint default_iv_size, string uri) {
+ this.key_size = key_size;
+ this.default_iv_size = default_iv_size;
+ this.uri = uri;
+ }
+ public string get_cipher_uri() {
+ return uri;
+ }
+ public Jet.TransportSecret generate_random_secret() {
+ uint8[] iv = new uint8[default_iv_size];
+ Crypto.randomize(iv);
+ uint8[] key = new uint8[key_size];
+ Crypto.randomize(key);
+ return new Jet.TransportSecret(key, iv);
+ }
+ public InputStream wrap_input_stream(InputStream input, Jet.TransportSecret secret) requires (secret.transport_key.length == key_size) {
+ SymmetricCipherDecryption cipher = create_symmetric_cipher_decryption("AES-GCM");
+ cipher.set_key(secret.transport_key);
+ cipher.set_iv(secret.initialization_vector);
+ return new ConverterInputStream(input, new SymmetricCipherDecrypter((owned) cipher, 16));
+ }
+ public OutputStream wrap_output_stream(OutputStream output, Jet.TransportSecret secret) requires (secret.transport_key.length == key_size) {
+ SymmetricCipherEncryption cipher = create_symmetric_cipher_encryption("AES-GCM");
+ cipher.set_key(secret.transport_key);
+ cipher.set_iv(secret.initialization_vector);
+ return new ConverterOutputStream(output, new SymmetricCipherEncrypter((owned) cipher, 16));
+ }
+}
+}
diff --git a/main/src/plugins/omemo/jingle/jingle_helper.vala b/main/src/plugins/omemo/jingle/jingle_helper.vala
new file mode 100644
index 00000000..ec6c0eca
--- /dev/null
+++ b/main/src/plugins/omemo/jingle/jingle_helper.vala
@@ -0,0 +1,53 @@
+using Dino.Entities;
+using Xmpp;
+
+namespace Dino.Plugins.JetOmemo {
+public class EncryptionHelper : JingleFileEncryptionHelper, Object {
+ private StreamInteractor stream_interactor;
+
+ public EncryptionHelper(StreamInteractor stream_interactor) {
+ this.stream_interactor = stream_interactor;
+ }
+
+ public bool can_transfer(Conversation conversation) {
+ return true;
+ }
+
+ public async bool can_encrypt(Conversation conversation, FileTransfer file_transfer, Jid? full_jid) {
+ XmppStream? stream = stream_interactor.get_stream(conversation.account);
+ if (stream == null) return false;
+
+ Gee.List<Jid>? resources = stream.get_flag(Presence.Flag.IDENTITY).get_resources(conversation.counterpart);
+ if (resources == null) return false;
+
+ if (full_jid == null) {
+ foreach (Jid test_jid in resources) {
+ if (yield stream.get_module(Module.IDENTITY).is_available(stream, test_jid)) {
+ return true;
+ }
+ }
+ } else {
+ if (yield stream.get_module(Module.IDENTITY).is_available(stream, full_jid)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public string? get_precondition_name(Conversation conversation, FileTransfer file_transfer) {
+ return Xep.Jet.NS_URI;
+ }
+
+ public Object? get_precondition_options(Conversation conversation, FileTransfer file_transfer) {
+ return new Xep.Jet.Options(Omemo.NS_URI, AES_128_GCM_URI);
+ }
+
+ public Encryption get_encryption(Xmpp.Xep.JingleFileTransfer.FileTransfer jingle_transfer) {
+ Xep.Jet.SecurityParameters? security = jingle_transfer.security as Xep.Jet.SecurityParameters;
+ if (security != null && security.encoding.get_type_uri() == Omemo.NS_URI) {
+ return Encryption.OMEMO;
+ }
+ return Encryption.NONE;
+ }
+}
+}
diff --git a/main/src/plugins/omemo/logic/database.vala b/main/src/plugins/omemo/logic/database.vala
new file mode 100644
index 00000000..f2a92259
--- /dev/null
+++ b/main/src/plugins/omemo/logic/database.vala
@@ -0,0 +1,284 @@
+using Gee;
+
+using Dino.Entities;
+using Dino.Qlite;
+
+namespace Dino.Plugins.Omemo {
+
+public class Database : Qlite.Database {
+ private const int VERSION = 5;
+
+ public class IdentityMetaTable : Table {
+ //Default to provide backwards compatability
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true, min_version = 2, default = "-1" };
+ public Column<string> address_name = new Column.Text("address_name") { not_null = true };
+ public Column<int> device_id = new Column.Integer("device_id") { not_null = true };
+ public Column<string?> identity_key_public_base64 = new Column.Text("identity_key_public_base64");
+ public Column<bool> trusted_identity = new Column.BoolInt("trusted_identity") { default = "0", max_version = 1 };
+ public Column<int> trust_level = new Column.Integer("trust_level") { default = TrustLevel.UNKNOWN.to_string(), min_version = 2 };
+ public Column<bool> now_active = new Column.BoolInt("now_active") { default = "1" };
+ public Column<long> last_active = new Column.Long("last_active");
+ public Column<long> last_message_untrusted = new Column.Long("last_message_untrusted") { min_version = 5 };
+ public Column<long> last_message_undecryptable = new Column.Long("last_message_undecryptable") { min_version = 5 };
+
+ internal IdentityMetaTable(Database db) {
+ base(db, "identity_meta");
+ init({identity_id, address_name, device_id, identity_key_public_base64, trusted_identity, trust_level, now_active, last_active, last_message_untrusted, last_message_undecryptable});
+ index("identity_meta_idx", {identity_id, address_name, device_id}, true);
+ index("identity_meta_list_idx", {identity_id, address_name});
+ }
+
+ public QueryBuilder with_address(int identity_id, string address_name) {
+ return select().with(this.identity_id, "=", identity_id).with(this.address_name, "=", address_name);
+ }
+
+ public QueryBuilder get_with_device_id(int identity_id, int device_id) {
+ return select().with(this.identity_id, "=", identity_id).with(this.device_id, "=", device_id);
+ }
+
+ public void insert_device_list(int32 identity_id, string address_name, ArrayList<int32> devices) {
+ update().with(this.identity_id, "=", identity_id).with(this.address_name, "=", address_name).set(now_active, false).perform();
+ foreach (int32 device_id in devices) {
+ upsert()
+ .value(this.identity_id, identity_id, true)
+ .value(this.address_name, address_name, true)
+ .value(this.device_id, device_id, true)
+ .value(this.now_active, true)
+ .value(this.last_active, (long) new DateTime.now_utc().to_unix())
+ .perform();
+ }
+ }
+
+ public int64 insert_device_bundle(int32 identity_id, string address_name, int device_id, Bundle bundle, TrustLevel trust) {
+ if (bundle == null || bundle.identity_key == null) return -1;
+ // Do not replace identity_key if it was known before, it should never change!
+ string identity_key = Base64.encode(bundle.identity_key.serialize());
+ RowOption row = with_address(identity_id, address_name).with(this.device_id, "=", device_id).single().row();
+ if (row.is_present() && row[identity_key_public_base64] != null && row[identity_key_public_base64] != identity_key) {
+ critical("Tried to change the identity key for a known device id. Likely an attack.");
+ return -1;
+ }
+ return upsert()
+ .value(this.identity_id, identity_id, true)
+ .value(this.address_name, address_name, true)
+ .value(this.device_id, device_id, true)
+ .value(this.identity_key_public_base64, identity_key)
+ .value(this.trust_level, trust).perform();
+ }
+
+ public int64 insert_device_session(int32 identity_id, string address_name, int device_id, string identity_key, TrustLevel trust) {
+ RowOption row = with_address(identity_id, address_name).with(this.device_id, "=", device_id).single().row();
+ if (row.is_present() && row[identity_key_public_base64] != null && row[identity_key_public_base64] != identity_key) {
+ critical("Tried to change the identity key for a known device id. Likely an attack.");
+ return -1;
+ }
+ return upsert()
+ .value(this.identity_id, identity_id, true)
+ .value(this.address_name, address_name, true)
+ .value(this.device_id, device_id, true)
+ .value(this.identity_key_public_base64, identity_key)
+ .value(this.trust_level, trust).perform();
+ }
+
+ public void update_last_message_untrusted(int identity_id, int device_id, DateTime? time) {
+ var stmt = update()
+ .with(this.identity_id, "=", identity_id)
+ .with(this.device_id, "=", device_id);
+ if (time != null) {
+ stmt.set(last_message_untrusted, (long)time.to_unix());
+ } else {
+ stmt.set_null(last_message_untrusted);
+ }
+ stmt.perform();
+ }
+
+ public void update_last_message_undecryptable(int identity_id, int device_id, DateTime? time) {
+ var stmt = update()
+ .with(this.identity_id, "=", identity_id)
+ .with(this.device_id, "=", device_id);
+ if (time != null) {
+ stmt.set(last_message_undecryptable, (long)time.to_unix());
+ } else {
+ stmt.set_null(last_message_undecryptable);
+ }
+ stmt.perform();
+ }
+
+ public QueryBuilder get_trusted_devices(int identity_id, string address_name) {
+ return this.with_address(identity_id, address_name)
+ .with(this.trust_level, "!=", TrustLevel.UNTRUSTED)
+ .with(this.now_active, "=", true);
+ }
+
+ public QueryBuilder get_known_devices(int identity_id, string address_name) {
+ return this.with_address(identity_id, address_name)
+ .with(this.trust_level, "!=", TrustLevel.UNKNOWN)
+ .without_null(this.identity_key_public_base64);
+ }
+
+ public QueryBuilder get_unknown_devices(int identity_id, string address_name) {
+ return this.with_address(identity_id, address_name)
+ .with_null(this.identity_key_public_base64);
+ }
+
+ public QueryBuilder get_new_devices(int identity_id, string address_name) {
+ return this.with_address(identity_id, address_name)
+ .with(this.trust_level, "=", TrustLevel.UNKNOWN)
+ .without_null(this.identity_key_public_base64);
+ }
+
+ public Row? get_device(int identity_id, string address_name, int device_id) {
+ return this.with_address(identity_id, address_name)
+ .with(this.device_id, "=", device_id).single().row().inner;
+ }
+ }
+
+
+ public class TrustTable : Table {
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true };
+ public Column<string> address_name = new Column.Text("address_name");
+ public Column<bool> blind_trust = new Column.BoolInt("blind_trust") { default = "1" } ;
+
+ internal TrustTable(Database db) {
+ base(db, "trust");
+ init({identity_id, address_name, blind_trust});
+ index("trust_idx", {identity_id, address_name}, true);
+ }
+
+ public bool get_blind_trust(int32 identity_id, string address_name, bool def = false) {
+ RowOption row = this.select().with(this.identity_id, "=", identity_id)
+ .with(this.address_name, "=", address_name).single().row();
+ if (row.is_present()) return row[blind_trust];
+ return def;
+ }
+ }
+
+ public class IdentityTable : Table {
+ public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true };
+ public Column<int> account_id = new Column.Integer("account_id") { unique = true, not_null = true };
+ public Column<int> device_id = new Column.Integer("device_id") { not_null = true };
+ public Column<string> identity_key_private_base64 = new Column.NonNullText("identity_key_private_base64");
+ public Column<string> identity_key_public_base64 = new Column.NonNullText("identity_key_public_base64");
+
+ internal IdentityTable(Database db) {
+ base(db, "identity");
+ init({id, account_id, device_id, identity_key_private_base64, identity_key_public_base64});
+ }
+
+ public int get_id(int account_id) {
+ int id = -1;
+ Row? row = this.row_with(this.account_id, account_id).inner;
+ if (row != null) id = ((!)row)[this.id];
+ return id;
+ }
+ }
+
+ public class SignedPreKeyTable : Table {
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true };
+ public Column<int> signed_pre_key_id = new Column.Integer("signed_pre_key_id") { not_null = true };
+ public Column<string> record_base64 = new Column.NonNullText("record_base64");
+
+ internal SignedPreKeyTable(Database db) {
+ base(db, "signed_pre_key");
+ init({identity_id, signed_pre_key_id, record_base64});
+ unique({identity_id, signed_pre_key_id});
+ index("signed_pre_key_idx", {identity_id, signed_pre_key_id}, true);
+ }
+ }
+
+ public class PreKeyTable : Table {
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true };
+ public Column<int> pre_key_id = new Column.Integer("pre_key_id") { not_null = true };
+ public Column<string> record_base64 = new Column.NonNullText("record_base64");
+
+ internal PreKeyTable(Database db) {
+ base(db, "pre_key");
+ init({identity_id, pre_key_id, record_base64});
+ unique({identity_id, pre_key_id});
+ index("pre_key_idx", {identity_id, pre_key_id}, true);
+ }
+ }
+
+ public class SessionTable : Table {
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true };
+ public Column<string> address_name = new Column.NonNullText("name");
+ public Column<int> device_id = new Column.Integer("device_id") { not_null = true };
+ public Column<string> record_base64 = new Column.NonNullText("record_base64");
+
+ internal SessionTable(Database db) {
+ base(db, "session");
+ init({identity_id, address_name, device_id, record_base64});
+ unique({identity_id, address_name, device_id});
+ index("session_idx", {identity_id, address_name, device_id}, true);
+ }
+ }
+
+ public class ContentItemMetaTable : Table {
+ public Column<int> content_item_id = new Column.Integer("message_id") { primary_key = true };
+ public Column<int> identity_id = new Column.Integer("identity_id") { not_null = true };
+ public Column<string> address_name = new Column.Text("address_name") { not_null = true };
+ public Column<int> device_id = new Column.Integer("device_id") { not_null = true };
+ public Column<bool> trusted_when_received = new Column.BoolInt("trusted_when_received") { not_null = true, default = "1" };
+
+ internal ContentItemMetaTable(Database db) {
+ base(db, "content_item_meta");
+ init({content_item_id, identity_id, address_name, device_id, trusted_when_received});
+ index("content_item_meta_device_idx", {identity_id, device_id, address_name});
+ }
+
+ public RowOption with_content_item(ContentItem item) {
+ return row_with(content_item_id, item.id);
+ }
+
+ public QueryBuilder with_device(int identity_id, string address_name, int device_id) {
+ return select()
+ .with(this.identity_id, "=", identity_id)
+ .with(this.address_name, "=", address_name)
+ .with(this.device_id, "=", device_id);
+ }
+ }
+
+ public IdentityMetaTable identity_meta { get; private set; }
+ public TrustTable trust { get; private set; }
+ public IdentityTable identity { get; private set; }
+ public SignedPreKeyTable signed_pre_key { get; private set; }
+ public PreKeyTable pre_key { get; private set; }
+ public SessionTable session { get; private set; }
+ public ContentItemMetaTable content_item_meta { get; private set; }
+
+ public Database(string fileName) {
+ base(fileName, VERSION);
+ identity_meta = new IdentityMetaTable(this);
+ trust = new TrustTable(this);
+ identity = new IdentityTable(this);
+ signed_pre_key = new SignedPreKeyTable(this);
+ pre_key = new PreKeyTable(this);
+ session = new SessionTable(this);
+ content_item_meta = new ContentItemMetaTable(this);
+ init({identity_meta, trust, identity, signed_pre_key, pre_key, session, content_item_meta});
+
+ try {
+ exec("PRAGMA journal_mode = WAL");
+ exec("PRAGMA synchronous = NORMAL");
+ exec("PRAGMA secure_delete = ON");
+ } catch (Error e) {
+ error("Failed to set OMEMO database properties: %s", e.message);
+ }
+ }
+
+ public override void migrate(long oldVersion) {
+ if(oldVersion == 1) {
+ try {
+ exec("DROP INDEX identity_meta_idx");
+ exec("DROP INDEX identity_meta_list_idx");
+ exec("CREATE UNIQUE INDEX identity_meta_idx ON identity_meta (identity_id, address_name, device_id)");
+ exec("CREATE INDEX identity_meta_list_idx ON identity_meta (identity_id, address_name)");
+ } catch (Error e) {
+ stderr.printf("Failed to migrate OMEMO database\n");
+ Process.exit(-1);
+ }
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/logic/decrypt.vala b/main/src/plugins/omemo/logic/decrypt.vala
new file mode 100644
index 00000000..135ac42c
--- /dev/null
+++ b/main/src/plugins/omemo/logic/decrypt.vala
@@ -0,0 +1,212 @@
+using Gee;
+
+using Dino.Crypto.Omemo;
+using Dino.Entities;
+using Dino.Qlite;
+using Omemo;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+ public class OmemoDecryptor : Xep.Omemo.OmemoDecryptor {
+
+ private Account account;
+ private Store store;
+ private Database db;
+ private StreamInteractor stream_interactor;
+ private TrustManager trust_manager;
+
+ public override uint32 own_device_id { get { return store.local_registration_id; }}
+
+ public OmemoDecryptor(Account account, StreamInteractor stream_interactor, TrustManager trust_manager, Database db, Store store) {
+ this.account = account;
+ this.stream_interactor = stream_interactor;
+ this.trust_manager = trust_manager;
+ this.db = db;
+ this.store = store;
+ }
+
+ public bool decrypt_message(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) {
+ StanzaNode? encrypted_node = stanza.stanza.get_subnode("encrypted", NS_URI);
+ if (encrypted_node == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false;
+
+ if (!Plugin.ensure_context()) return false;
+ int identity_id = db.identity.get_id(conversation.account.id);
+
+ MessageFlag flag = new MessageFlag();
+ stanza.add_flag(flag);
+
+ Xep.Omemo.ParsedData? data = parse_node(encrypted_node);
+ if (data == null) return false;
+
+
+ foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) {
+ data.is_prekey = data.our_potential_encrypted_keys[encr_key];
+ data.encrypted_key = encr_key.get_data();
+ Gee.List<Jid> possible_jids = get_potential_message_jids(message, data, identity_id);
+ if (possible_jids.size == 0) {
+ debug("Received message from unknown entity with device id %d", data.sid);
+ }
+
+ foreach (Jid possible_jid in possible_jids) {
+ try {
+ uint8[] key = decrypt_key(data, possible_jid);
+ if (data.ciphertext != null) {
+ string cleartext = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, data.iv, data.ciphertext));
+ message.body = cleartext;
+ }
+
+ // If we figured out which real jid a message comes from due to decryption working, save it
+ if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) {
+ message.real_jid = possible_jid;
+ }
+
+ message.encryption = Encryption.OMEMO;
+
+ trust_manager.message_device_id_map[message] = data.sid;
+ return true;
+ } catch (Error e) {
+ debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), data.sid, e.message);
+ }
+ }
+ }
+
+ if (
+ data.ciphertext != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok
+ data.our_potential_encrypted_keys.size == 0 && // The message was not encrypted to us
+ stream_interactor.module_manager.get_module(message.account, StreamModule.IDENTITY).store.local_registration_id != data.sid // Message from this device. Never encrypted to itself.
+ ) {
+ db.identity_meta.update_last_message_undecryptable(identity_id, data.sid, message.time);
+ trust_manager.bad_message_state_updated(conversation.account, message.from, data.sid);
+ }
+
+ debug("Received OMEMO encryped message that could not be decrypted.");
+ return false;
+ }
+
+ public Gee.List<Jid> get_potential_message_jids(Entities.Message message, Xmpp.Xep.Omemo.ParsedData data, int identity_id) {
+ Gee.List<Jid> possible_jids = new ArrayList<Jid>();
+ if (message.type_ == Message.Type.CHAT) {
+ possible_jids.add(message.from.bare_jid);
+ } else {
+ if (message.real_jid != null) {
+ possible_jids.add(message.real_jid.bare_jid);
+ } else if (data.is_prekey) {
+ // pre key messages do store the identity key, so we can use that to find the real jid
+ PreKeyOmemoMessage msg = Plugin.get_context().deserialize_signal_pre_key_message(data.encrypted_key);
+ string identity_key = Base64.encode(msg.identity_key.serialize());
+ foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) {
+ try {
+ possible_jids.add(Jid.from_string(row[db.identity_meta.address_name]));
+ } catch (InvalidJidError e) {
+ warning("Ignoring invalid jid from database: %s", e.message);
+ }
+ }
+ } else {
+ // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id
+ foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid)) {
+ try {
+ possible_jids.add(Jid.from_string(row[db.identity_meta.address_name]));
+ } catch (InvalidJidError e) {
+ warning("Ignoring invalid jid from database: %s", e.message);
+ }
+ }
+ }
+ }
+ return possible_jids;
+ }
+
+ public override uint8[] decrypt_key(Xmpp.Xep.Omemo.ParsedData data, Jid from_jid) throws GLib.Error {
+ int sid = data.sid;
+ uint8[] ciphertext = data.ciphertext;
+ uint8[] encrypted_key = data.encrypted_key;
+
+ Address address = new Address(from_jid.to_string(), sid);
+ uint8[] key;
+
+ if (data.is_prekey) {
+ int identity_id = db.identity.get_id(account.id);
+ PreKeyOmemoMessage msg = Plugin.get_context().deserialize_signal_pre_key_message(encrypted_key);
+ string identity_key = Base64.encode(msg.identity_key.serialize());
+
+ bool ok = update_db_for_prekey(identity_id, identity_key, from_jid, sid);
+ if (!ok) throw new GLib.Error(-1, 0, "Failed updating db for prekey");
+
+ debug("Starting new session for decryption with device from %s/%d", from_jid.to_string(), sid);
+ SessionCipher cipher = store.create_session_cipher(address);
+ key = cipher.decrypt_pre_key_message(msg);
+ // TODO: Finish session
+ } else {
+ debug("Continuing session for decryption with device from %s/%d", from_jid.to_string(), sid);
+ OmemoMessage msg = Plugin.get_context().deserialize_signal_message(encrypted_key);
+ SessionCipher cipher = store.create_session_cipher(address);
+ key = cipher.decrypt_message(msg);
+ }
+
+ if (key.length >= 32) {
+ int authtaglength = key.length - 16;
+ uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength];
+ uint8[] new_key = new uint8[16];
+ Memory.copy(new_ciphertext, ciphertext, ciphertext.length);
+ Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength);
+ Memory.copy(new_key, key, 16);
+ data.ciphertext = new_ciphertext;
+ key = new_key;
+ }
+
+ return key;
+ }
+
+ public override string decrypt(uint8[] ciphertext, uint8[] key, uint8[] iv) throws GLib.Error {
+ return arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext));
+ }
+
+ private bool update_db_for_prekey(int identity_id, string identity_key, Jid from_jid, int sid) {
+ Row? device = db.identity_meta.get_device(identity_id, from_jid.to_string(), sid);
+ if (device != null && device[db.identity_meta.identity_key_public_base64] != null) {
+ if (device[db.identity_meta.identity_key_public_base64] != identity_key) {
+ critical("Tried to use a different identity key for a known device id.");
+ return false;
+ }
+ } else {
+ debug("Learn new device from incoming message from %s/%d", from_jid.to_string(), sid);
+ bool blind_trust = db.trust.get_blind_trust(identity_id, from_jid.to_string(), true);
+ if (db.identity_meta.insert_device_session(identity_id, from_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) {
+ critical("Failed learning a device.");
+ return false;
+ }
+
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (device == null && stream != null) {
+ stream.get_module(StreamModule.IDENTITY).request_user_devicelist.begin(stream, from_jid);
+ }
+ }
+ return true;
+ }
+
+ private string arr_to_str(uint8[] arr) {
+ // null-terminate the array
+ uint8[] rarr = new uint8[arr.length+1];
+ Memory.copy(rarr, arr, arr.length);
+ return (string)rarr;
+ }
+ }
+
+ public class DecryptMessageListener : MessageListener {
+ public string[] after_actions_const = new string[]{ };
+ public override string action_group { get { return "DECRYPT"; } }
+ public override string[] after_actions { get { return after_actions_const; } }
+
+ private HashMap<Account, OmemoDecryptor> decryptors;
+
+ public DecryptMessageListener(HashMap<Account, OmemoDecryptor> decryptors) {
+ this.decryptors = decryptors;
+ }
+
+ public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) {
+ decryptors[message.account].decrypt_message(message, stanza, conversation);
+ return false;
+ }
+ }
+}
+
diff --git a/main/src/plugins/omemo/logic/encrypt.vala b/main/src/plugins/omemo/logic/encrypt.vala
new file mode 100644
index 00000000..8994d00d
--- /dev/null
+++ b/main/src/plugins/omemo/logic/encrypt.vala
@@ -0,0 +1,132 @@
+using Gee;
+using Omemo;
+using Dino.Crypto.Omemo;
+using Dino.Entities;
+using Xmpp;
+using Xmpp.Xep.Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+ public class OmemoEncryptor : Xep.Omemo.OmemoEncryptor {
+
+ private Account account;
+ private Store store;
+ private TrustManager trust_manager;
+
+ public override uint32 own_device_id { get { return store.local_registration_id; }}
+
+ public OmemoEncryptor(Account account, TrustManager trust_manager, Store store) {
+ this.account = account;
+ this.trust_manager = trust_manager;
+ this.store = store;
+ }
+
+ public override Xep.Omemo.EncryptionData encrypt_plaintext(string plaintext) throws GLib.Error {
+ const uint KEY_SIZE = 16;
+ const uint IV_SIZE = 12;
+
+ //Create a key and use it to encrypt the message
+ uint8[] key = new uint8[KEY_SIZE];
+ Crypto.randomize(key);
+ uint8[] iv = new uint8[IV_SIZE];
+ Crypto.randomize(iv);
+
+ uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, plaintext.data);
+ uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length - 16];
+ uint8[] tag = aes_encrypt_result[aes_encrypt_result.length - 16:aes_encrypt_result.length];
+ uint8[] keytag = new uint8[key.length + tag.length];
+ Memory.copy(keytag, key, key.length);
+ Memory.copy((uint8*)keytag + key.length, tag, tag.length);
+
+ var ret = new Xep.Omemo.EncryptionData(own_device_id);
+ ret.ciphertext = ciphertext;
+ ret.keytag = keytag;
+ ret.iv = iv;
+ return ret;
+ }
+
+ public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) {
+
+ EncryptState status = new EncryptState();
+ if (!Plugin.ensure_context()) return status;
+ if (message.to == null) return status;
+
+ try {
+ EncryptionData enc_data = encrypt_plaintext(message.body);
+ status = encrypt_key_to_recipients(enc_data, self_jid, recipients, stream);
+
+ message.stanza.put_node(enc_data.get_encrypted_node());
+ Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO");
+ message.body = "[This message is OMEMO encrypted]";
+ status.encrypted = true;
+ } catch (Error e) {
+ warning(@"error while encrypting message: $(e.message)\n");
+ message.body = "[OMEMO encryption failed]";
+ status.encrypted = false;
+ }
+ return status;
+ }
+
+ internal EncryptState encrypt_key_to_recipients(EncryptionData enc_data, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) throws Error {
+ EncryptState status = new EncryptState();
+
+ //Check we have the bundles and device lists needed to send the message
+ if (!trust_manager.is_known_address(account, self_jid)) return status;
+ status.own_list = true;
+ status.own_devices = trust_manager.get_trusted_devices(account, self_jid).size;
+ status.other_waiting_lists = 0;
+ status.other_devices = 0;
+ foreach (Jid recipient in recipients) {
+ if (!trust_manager.is_known_address(account, recipient)) {
+ status.other_waiting_lists++;
+ }
+ if (status.other_waiting_lists > 0) return status;
+ status.other_devices += trust_manager.get_trusted_devices(account, recipient).size;
+ }
+ if (status.own_devices == 0 || status.other_devices == 0) return status;
+
+
+ //Encrypt the key for each recipient's device individually
+ foreach (Jid recipient in recipients) {
+ EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, recipient);
+ status.add_result(enc_res, false);
+ }
+
+ // Encrypt the key for each own device
+ EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, self_jid);
+ status.add_result(enc_res, true);
+
+ return status;
+ }
+
+ public override EncryptionResult encrypt_key_to_recipient(XmppStream stream, Xep.Omemo.EncryptionData enc_data, Jid recipient) throws GLib.Error {
+ var result = new EncryptionResult();
+ StreamModule module = stream.get_module(StreamModule.IDENTITY);
+
+ foreach(int32 device_id in trust_manager.get_trusted_devices(account, recipient)) {
+ if (module.is_ignored_device(recipient, device_id)) {
+ result.lost++;
+ continue;
+ }
+ try {
+ encrypt_key(enc_data, recipient, device_id);
+ result.success++;
+ } catch (Error e) {
+ if (e.code == ErrorCode.UNKNOWN) result.unknown++;
+ else result.failure++;
+ }
+ }
+ return result;
+ }
+
+ public override void encrypt_key(Xep.Omemo.EncryptionData encryption_data, Jid jid, int32 device_id) throws GLib.Error {
+ Address address = new Address(jid.to_string(), device_id);
+ SessionCipher cipher = store.create_session_cipher(address);
+ CiphertextMessage device_key = cipher.encrypt(encryption_data.keytag);
+ address.device_id = 0;
+ debug("Created encrypted key for %s/%d", jid.to_string(), device_id);
+
+ encryption_data.add_device_key(device_id, device_key.serialized, device_key.type == CiphertextType.PREKEY);
+ }
+ }
+}
diff --git a/main/src/plugins/omemo/logic/manager.vala b/main/src/plugins/omemo/logic/manager.vala
new file mode 100644
index 00000000..358db4e3
--- /dev/null
+++ b/main/src/plugins/omemo/logic/manager.vala
@@ -0,0 +1,431 @@
+using Gee;
+
+using Dino.Crypto.Omemo;
+using Dino.Entities;
+using Dino.Qlite;
+using Omemo;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class Manager : StreamInteractionModule, Object {
+ public static ModuleIdentity<Manager> IDENTITY = new ModuleIdentity<Manager>("omemo_manager");
+ public string id { get { return IDENTITY.id; } }
+
+ private StreamInteractor stream_interactor;
+ private Database db;
+ private TrustManager trust_manager;
+ private HashMap<Account, OmemoEncryptor> encryptors;
+ private Map<Entities.Message, MessageState> message_states = new HashMap<Entities.Message, MessageState>(Entities.Message.hash_func, Entities.Message.equals_func);
+
+ private class MessageState {
+ public Entities.Message msg { get; private set; }
+ public Xep.Omemo.EncryptState last_try { get; private set; }
+ public int waiting_other_sessions { get; set; }
+ public int waiting_own_sessions { get; set; }
+ public bool waiting_own_devicelist { get; set; }
+ public int waiting_other_devicelists { get; set; }
+ public bool force_next_attempt { get; set; }
+ public bool will_send_now { get; private set; }
+ public bool active_send_attempt { get; set; }
+
+ public MessageState(Entities.Message msg, Xep.Omemo.EncryptState last_try) {
+ update_from_encrypt_status(msg, last_try);
+ }
+
+ public void update_from_encrypt_status(Entities.Message msg, Xep.Omemo.EncryptState new_try) {
+ this.msg = msg;
+ this.last_try = new_try;
+ this.waiting_other_sessions = new_try.other_unknown;
+ this.waiting_own_sessions = new_try.own_unknown;
+ this.waiting_own_devicelist = !new_try.own_list;
+ this.waiting_other_devicelists = new_try.other_waiting_lists;
+ this.active_send_attempt = false;
+ will_send_now = false;
+ if (new_try.other_failure > 0 || (new_try.other_lost == new_try.other_devices && new_try.other_devices > 0)) {
+ msg.marked = Entities.Message.Marked.WONTSEND;
+ } else if (new_try.other_unknown > 0 || new_try.own_unknown > 0 || new_try.other_waiting_lists > 0 || !new_try.own_list || new_try.own_devices == 0) {
+ msg.marked = Entities.Message.Marked.UNSENT;
+ } else if (!new_try.encrypted) {
+ msg.marked = Entities.Message.Marked.WONTSEND;
+ } else {
+ will_send_now = true;
+ }
+ }
+
+ public bool should_retry_now() {
+ return !waiting_own_devicelist && waiting_other_devicelists <= 0 && waiting_other_sessions <= 0 && waiting_own_sessions <= 0 && !active_send_attempt;
+ }
+
+ public string to_string() {
+ return @"MessageState (msg=$(msg.stanza_id), send=$will_send_now, $last_try)";
+ }
+ }
+
+ private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) {
+ this.stream_interactor = stream_interactor;
+ this.db = db;
+ this.trust_manager = trust_manager;
+ this.encryptors = encryptors;
+
+ stream_interactor.account_added.connect(on_account_added);
+ stream_interactor.stream_negotiated.connect(on_stream_negotiated);
+ stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(on_pre_message_send);
+ stream_interactor.get_module(RosterManager.IDENTITY).mutual_subscription.connect(on_mutual_subscription);
+ }
+
+ public void clear_device_list(Account account) {
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (stream == null) return;
+
+ stream.get_module(StreamModule.IDENTITY).clear_device_list(stream);
+ }
+
+ private Gee.List<Jid> get_occupants(Jid jid, Account account){
+ Gee.List<Jid> occupants = new ArrayList<Jid>(Jid.equals_bare_func);
+ if(!stream_interactor.get_module(MucManager.IDENTITY).is_groupchat(jid, account)){
+ occupants.add(jid);
+ }
+ Gee.List<Jid>? occupant_jids = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(jid, account);
+ if(occupant_jids == null) {
+ return occupants;
+ }
+ foreach (Jid occupant in occupant_jids) {
+ if(!occupant.equals(account.bare_jid)){
+ occupants.add(occupant.bare_jid);
+ }
+ }
+ return occupants;
+ }
+
+ private void on_pre_message_send(Entities.Message message, Xmpp.MessageStanza message_stanza, Conversation conversation) {
+ if (message.encryption == Encryption.OMEMO) {
+ if (message.type_ == Message.Type.GROUPCHAT_PM) {
+ message.marked = Message.Marked.WONTSEND;
+ return;
+ }
+ XmppStream? stream = stream_interactor.get_stream(conversation.account);
+ if (stream == null) {
+ message.marked = Entities.Message.Marked.UNSENT;
+ return;
+ }
+ StreamModule? module_ = ((!)stream).get_module(StreamModule.IDENTITY);
+ if (module_ == null) {
+ message.marked = Entities.Message.Marked.UNSENT;
+ return;
+ }
+ StreamModule module = (!)module_;
+
+ //Get a list of everyone for whom the message should be encrypted
+ Gee.List<Jid> recipients;
+ if (message_stanza.type_ == MessageStanza.TYPE_GROUPCHAT) {
+ recipients = get_occupants((!)message.to.bare_jid, conversation.account);
+ if (recipients.size == 0) {
+ message.marked = Entities.Message.Marked.WONTSEND;
+ return;
+ }
+ } else {
+ recipients = new ArrayList<Jid>(Jid.equals_bare_func);
+ recipients.add(message_stanza.to);
+ }
+
+ //Attempt to encrypt the message
+ Xep.Omemo.EncryptState enc_state = encryptors[conversation.account].encrypt(message_stanza, conversation.account.bare_jid, recipients, stream);
+ MessageState state;
+ lock (message_states) {
+ if (message_states.has_key(message)) {
+ state = message_states.get(message);
+ state.update_from_encrypt_status(message, enc_state);
+ if (state.will_send_now) {
+ debug("sending message delayed: %s", state.to_string());
+ }
+ } else {
+ state = new MessageState(message, enc_state);
+ message_states[message] = state;
+ }
+ if (state.will_send_now) {
+ message_states.unset(message);
+ }
+ }
+
+ //Encryption failed - need to fetch more information
+ if (!state.will_send_now) {
+ if (message.marked == Entities.Message.Marked.WONTSEND) {
+ debug("retracting message %s", state.to_string());
+ message_states.unset(message);
+ } else {
+ debug("delaying message %s", state.to_string());
+
+ if (state.waiting_own_sessions > 0) {
+ module.fetch_bundles((!)stream, conversation.account.bare_jid, trust_manager.get_trusted_devices(conversation.account, conversation.account.bare_jid));
+ }
+ if (state.waiting_other_sessions > 0 && message.counterpart != null) {
+ foreach(Jid jid in get_occupants(((!)message.counterpart).bare_jid, conversation.account)) {
+ module.fetch_bundles((!)stream, jid, trust_manager.get_trusted_devices(conversation.account, jid));
+ }
+ }
+ if (state.waiting_other_devicelists > 0 && message.counterpart != null) {
+ foreach(Jid jid in get_occupants(((!)message.counterpart).bare_jid, conversation.account)) {
+ module.request_user_devicelist.begin((!)stream, jid);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void on_mutual_subscription(Account account, Jid jid) {
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if(stream == null) return;
+
+ stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).request_user_devicelist.begin((!)stream, jid);
+ }
+
+ private void on_stream_negotiated(Account account, XmppStream stream) {
+ StreamModule module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY);
+ if (module != null) {
+ module.request_user_devicelist.begin(stream, account.bare_jid);
+ }
+ }
+
+ private void on_account_added(Account account) {
+ StreamModule module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY);
+ if (module != null) {
+ module.device_list_loaded.connect((jid, devices) => on_device_list_loaded(account, jid, devices));
+ module.bundle_fetched.connect((jid, device_id, bundle) => on_bundle_fetched(account, jid, device_id, bundle));
+ module.bundle_fetch_failed.connect((jid) => continue_message_sending(account, jid));
+ }
+ initialize_store.begin(account);
+ }
+
+ private void on_device_list_loaded(Account account, Jid jid, ArrayList<int32> device_list) {
+ debug("received device list for %s from %s", account.bare_jid.to_string(), jid.to_string());
+
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (stream == null) {
+ return;
+ }
+ StreamModule? module = ((!)stream).get_module(StreamModule.IDENTITY);
+ if (module == null) {
+ return;
+ }
+
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return;
+
+ //Update meta database
+ db.identity_meta.insert_device_list(identity_id, jid.bare_jid.to_string(), device_list);
+
+ //Fetch the bundle for each new device
+ int inc = 0;
+ foreach (Row row in db.identity_meta.get_unknown_devices(identity_id, jid.bare_jid.to_string())) {
+ try {
+ module.fetch_bundle(stream, Jid.from_string(row[db.identity_meta.address_name]), row[db.identity_meta.device_id], false);
+ inc++;
+ } catch (InvalidJidError e) {
+ warning("Ignoring device with invalid Jid: %s", e.message);
+ }
+ }
+ if (inc > 0) {
+ debug("new bundles %i/%i for %s", inc, device_list.size, jid.to_string());
+ }
+
+ //Create an entry for the jid in the account table if one does not exist already
+ if (db.trust.select().with(db.trust.identity_id, "=", identity_id).with(db.trust.address_name, "=", jid.bare_jid.to_string()).count() == 0) {
+ db.trust.insert().value(db.trust.identity_id, identity_id).value(db.trust.address_name, jid.bare_jid.to_string()).value(db.trust.blind_trust, true).perform();
+ }
+
+ //Get all messages that needed the devicelist and determine if we can now send them
+ HashSet<Entities.Message> send_now = new HashSet<Entities.Message>();
+ lock (message_states) {
+ foreach (Entities.Message msg in message_states.keys) {
+ if (!msg.account.equals(account)) continue;
+ Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account);
+ MessageState state = message_states[msg];
+ if (account.bare_jid.equals(jid)) {
+ state.waiting_own_devicelist = false;
+ } else if (msg.counterpart != null && occupants.contains(jid)) {
+ state.waiting_other_devicelists--;
+ }
+ if (state.should_retry_now()) {
+ send_now.add(msg);
+ state.active_send_attempt = true;
+ }
+ }
+ }
+ foreach (Entities.Message msg in send_now) {
+ if (msg.counterpart == null) continue;
+ Entities.Conversation? conv = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(((!)msg.counterpart), account);
+ if (conv == null) continue;
+ stream_interactor.get_module(MessageProcessor.IDENTITY).send_xmpp_message(msg, (!)conv, true);
+ }
+
+ }
+
+ private void on_bundle_fetched(Account account, Jid jid, int32 device_id, Bundle bundle) {
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return;
+
+ bool blind_trust = db.trust.get_blind_trust(identity_id, jid.bare_jid.to_string(), true);
+
+ //If we don't blindly trust new devices and we haven't seen this key before then don't trust it
+ bool untrust = !(blind_trust || db.identity_meta.with_address(identity_id, jid.bare_jid.to_string())
+ .with(db.identity_meta.device_id, "=", device_id)
+ .with(db.identity_meta.identity_key_public_base64, "=", Base64.encode(bundle.identity_key.serialize()))
+ .single().row().is_present());
+
+ //Get trust information from the database if the device id is known
+ Row device = db.identity_meta.get_device(identity_id, jid.bare_jid.to_string(), device_id);
+ TrustLevel trusted = TrustLevel.UNKNOWN;
+ if (device != null) {
+ trusted = (TrustLevel) device[db.identity_meta.trust_level];
+ }
+
+ if(untrust) {
+ trusted = TrustLevel.UNKNOWN;
+ } else if (blind_trust && trusted == TrustLevel.UNKNOWN) {
+ trusted = TrustLevel.TRUSTED;
+ }
+
+ //Update the database with the appropriate trust information
+ db.identity_meta.insert_device_bundle(identity_id, jid.bare_jid.to_string(), device_id, bundle, trusted);
+
+ if (should_start_session(account, jid)) {
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (stream != null) {
+ StreamModule? module = ((!)stream).get_module(StreamModule.IDENTITY);
+ if (module != null) {
+ module.start_session(stream, jid, device_id, bundle);
+ }
+ }
+ }
+ continue_message_sending(account, jid);
+ }
+
+ private bool should_start_session(Account account, Jid jid) {
+ lock (message_states) {
+ foreach (Entities.Message msg in message_states.keys) {
+ if (!msg.account.equals(account)) continue;
+ Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account);
+ if (account.bare_jid.equals(jid) || (msg.counterpart != null && (msg.counterpart.equals_bare(jid) || occupants.contains(jid)))) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void continue_message_sending(Account account, Jid jid) {
+ //Get all messages waiting and determine if they can now be sent
+ HashSet<Entities.Message> send_now = new HashSet<Entities.Message>();
+ lock (message_states) {
+ foreach (Entities.Message msg in message_states.keys) {
+ if (!msg.account.equals(account)) continue;
+ Gee.List<Jid> occupants = get_occupants(msg.counterpart.bare_jid, account);
+
+ MessageState state = message_states[msg];
+
+ if (account.bare_jid.equals(jid)) {
+ state.waiting_own_sessions--;
+ } else if (msg.counterpart != null && (msg.counterpart.equals_bare(jid) || occupants.contains(jid))) {
+ state.waiting_other_sessions--;
+ }
+ if (state.should_retry_now()){
+ send_now.add(msg);
+ state.active_send_attempt = true;
+ }
+ }
+ }
+ foreach (Entities.Message msg in send_now) {
+ if (msg.counterpart == null) continue;
+ Entities.Conversation? conv = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation((!)msg.counterpart, account);
+ if (conv == null) continue;
+ stream_interactor.get_module(MessageProcessor.IDENTITY).send_xmpp_message(msg, (!)conv, true);
+ }
+ }
+
+ private async void initialize_store(Account account) {
+ // If the account is not yet persisted, wait for that and then continue - without identity.account_id the entry isn't worth much.
+ if (account.id == -1) {
+ account.notify["id"].connect(() => initialize_store.callback());
+ yield;
+ }
+ StreamModule? module = stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY);
+ if (module == null) return;
+ Store store = module.store;
+ Qlite.Row? row = db.identity.row_with(db.identity.account_id, account.id).inner;
+ int identity_id = -1;
+ bool publish_identity = false;
+
+ if (row == null) {
+ // OMEMO not yet initialized, starting with empty base
+ publish_identity = true;
+ try {
+ store.identity_key_store.local_registration_id = Random.int_range(1, int32.MAX);
+
+ ECKeyPair key_pair = Plugin.get_context().generate_key_pair();
+ store.identity_key_store.identity_key_private = new Bytes(key_pair.private.serialize());
+ store.identity_key_store.identity_key_public = new Bytes(key_pair.public.serialize());
+
+ identity_id = (int) db.identity.upsert()
+ .value(db.identity.account_id, account.id, true)
+ .value(db.identity.device_id, (int) store.local_registration_id)
+ .value(db.identity.identity_key_private_base64, Base64.encode(store.identity_key_store.identity_key_private.get_data()))
+ .value(db.identity.identity_key_public_base64, Base64.encode(store.identity_key_store.identity_key_public.get_data()))
+ .perform();
+ } catch (Error e) {
+ // Ignore error
+ }
+ } else {
+ store.identity_key_store.local_registration_id = ((!)row)[db.identity.device_id];
+ store.identity_key_store.identity_key_private = new Bytes(Base64.decode(((!)row)[db.identity.identity_key_private_base64]));
+ store.identity_key_store.identity_key_public = new Bytes(Base64.decode(((!)row)[db.identity.identity_key_public_base64]));
+ identity_id = ((!)row)[db.identity.id];
+ }
+
+ if (identity_id >= 0) {
+ store.signed_pre_key_store = new BackedSignedPreKeyStore(db, identity_id);
+ store.pre_key_store = new BackedPreKeyStore(db, identity_id);
+ store.session_store = new BackedSessionStore(db, identity_id);
+ } else {
+ warning("store for %s is not persisted!", account.bare_jid.to_string());
+ }
+
+ // Generated new device ID, ensure this gets added to the devicelist
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (stream != null) {
+ module.request_user_devicelist.begin((!)stream, account.bare_jid);
+ }
+ }
+
+ public async bool ensure_get_keys_for_conversation(Conversation conversation) {
+ if (stream_interactor.get_module(MucManager.IDENTITY).is_private_room(conversation.account, conversation.counterpart)) {
+ foreach (Jid offline_member in stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account)) {
+ bool ok = yield ensure_get_keys_for_jid(conversation.account, offline_member);
+ if (!ok) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ return yield ensure_get_keys_for_jid(conversation.account, conversation.counterpart.bare_jid);
+ }
+
+ public async bool ensure_get_keys_for_jid(Account account, Jid jid) {
+ if (trust_manager.is_known_address(account, jid)) return true;
+ XmppStream? stream = stream_interactor.get_stream(account);
+ if (stream != null) {
+ var device_list = yield stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).request_user_devicelist(stream, jid);
+ return device_list.size > 0;
+ }
+ return true; // TODO wait for stream?
+ }
+
+ public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) {
+ Manager m = new Manager(stream_interactor, db, trust_manager, encryptors);
+ stream_interactor.add_module(m);
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/logic/pre_key_store.vala b/main/src/plugins/omemo/logic/pre_key_store.vala
new file mode 100644
index 00000000..968ec119
--- /dev/null
+++ b/main/src/plugins/omemo/logic/pre_key_store.vala
@@ -0,0 +1,46 @@
+using Dino.Crypto.Omemo;
+using Dino.Qlite;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+private class BackedPreKeyStore : SimplePreKeyStore {
+ private Database db;
+ private int identity_id;
+
+ public BackedPreKeyStore(Database db, int identity_id) {
+ this.db = db;
+ this.identity_id = identity_id;
+ init();
+ }
+
+ private void init() {
+ try {
+ foreach (Row row in db.pre_key.select().with(db.pre_key.identity_id, "=", identity_id)) {
+ store_pre_key(row[db.pre_key.pre_key_id], Base64.decode(row[db.pre_key.record_base64]));
+ }
+ } catch (Error e) {
+ warning("Error while initializing pre key store: %s", e.message);
+ }
+
+ pre_key_stored.connect(on_pre_key_stored);
+ pre_key_deleted.connect(on_pre_key_deleted);
+ }
+
+ public void on_pre_key_stored(PreKeyStore.Key key) {
+ db.pre_key.upsert()
+ .value(db.pre_key.identity_id, identity_id, true)
+ .value(db.pre_key.pre_key_id, (int) key.key_id, true)
+ .value(db.pre_key.record_base64, Base64.encode(key.record))
+ .perform();
+ }
+
+ public void on_pre_key_deleted(PreKeyStore.Key key) {
+ db.pre_key.delete()
+ .with(db.pre_key.identity_id, "=", identity_id)
+ .with(db.pre_key.pre_key_id, "=", (int) key.key_id)
+ .perform();
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/logic/session_store.vala b/main/src/plugins/omemo/logic/session_store.vala
new file mode 100644
index 00000000..4232e6be
--- /dev/null
+++ b/main/src/plugins/omemo/logic/session_store.vala
@@ -0,0 +1,50 @@
+using Dino.Crypto.Omemo;
+using Dino.Qlite;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+private class BackedSessionStore : SimpleSessionStore {
+ private Database db;
+ private int identity_id;
+
+ public BackedSessionStore(Database db, int identity_id) {
+ this.db = db;
+ this.identity_id = identity_id;
+ init();
+ }
+
+ private void init() {
+ try {
+ foreach (Row row in db.session.select().with(db.session.identity_id, "=", identity_id)) {
+ Address addr = new Address(row[db.session.address_name], row[db.session.device_id]);
+ store_session(addr, Base64.decode(row[db.session.record_base64]));
+ addr.device_id = 0;
+ }
+ } catch (Error e) {
+ print("Error while initializing session store: %s", e.message);
+ }
+
+ session_stored.connect(on_session_stored);
+ session_removed.connect(on_session_deleted);
+ }
+
+ public void on_session_stored(SessionStore.Session session) {
+ db.session.upsert()
+ .value(db.session.identity_id, identity_id, true)
+ .value(db.session.address_name, session.name, true)
+ .value(db.session.device_id, session.device_id, true)
+ .value(db.session.record_base64, Base64.encode(session.record))
+ .perform();
+ }
+
+ public void on_session_deleted(SessionStore.Session session) {
+ db.session.delete()
+ .with(db.session.identity_id, "=", identity_id)
+ .with(db.session.address_name, "=", session.name)
+ .with(db.session.device_id, "=", session.device_id)
+ .perform();
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/logic/signed_pre_key_store.vala b/main/src/plugins/omemo/logic/signed_pre_key_store.vala
new file mode 100644
index 00000000..373a0060
--- /dev/null
+++ b/main/src/plugins/omemo/logic/signed_pre_key_store.vala
@@ -0,0 +1,46 @@
+using Dino.Crypto.Omemo;
+using Dino.Qlite;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+private class BackedSignedPreKeyStore : SimpleSignedPreKeyStore {
+ private Database db;
+ private int identity_id;
+
+ public BackedSignedPreKeyStore(Database db, int identity_id) {
+ this.db = db;
+ this.identity_id = identity_id;
+ init();
+ }
+
+ private void init() {
+ try {
+ foreach (Row row in db.signed_pre_key.select().with(db.signed_pre_key.identity_id, "=", identity_id)) {
+ store_signed_pre_key(row[db.signed_pre_key.signed_pre_key_id], Base64.decode(row[db.signed_pre_key.record_base64]));
+ }
+ } catch (Error e) {
+ print("Error while initializing signed pre key store: %s", e.message);
+ }
+
+ signed_pre_key_stored.connect(on_signed_pre_key_stored);
+ signed_pre_key_deleted.connect(on_signed_pre_key_deleted);
+ }
+
+ public void on_signed_pre_key_stored(SignedPreKeyStore.Key key) {
+ db.signed_pre_key.upsert()
+ .value(db.signed_pre_key.identity_id, identity_id, true)
+ .value(db.signed_pre_key.signed_pre_key_id, (int) key.key_id, true)
+ .value(db.signed_pre_key.record_base64, Base64.encode(key.record))
+ .perform();
+ }
+
+ public void on_signed_pre_key_deleted(SignedPreKeyStore.Key key) {
+ db.signed_pre_key.delete()
+ .with(db.signed_pre_key.identity_id, "=", identity_id)
+ .with(db.signed_pre_key.signed_pre_key_id, "=", (int) key.key_id)
+ .perform();
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/logic/trust_manager.vala b/main/src/plugins/omemo/logic/trust_manager.vala
new file mode 100644
index 00000000..f17ade99
--- /dev/null
+++ b/main/src/plugins/omemo/logic/trust_manager.vala
@@ -0,0 +1,142 @@
+using Gee;
+
+using Dino.Entities;
+using Dino.Qlite;
+using Omemo;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class TrustManager {
+
+ public signal void bad_message_state_updated(Account account, Jid jid, int device_id);
+
+ private StreamInteractor stream_interactor;
+ private Database db;
+ private TagMessageListener tag_message_listener;
+
+ public HashMap<Message, int> message_device_id_map = new HashMap<Message, int>(Message.hash_func, Message.equals_func);
+
+ public TrustManager(StreamInteractor stream_interactor, Database db) {
+ this.stream_interactor = stream_interactor;
+ this.db = db;
+
+ tag_message_listener = new TagMessageListener(stream_interactor, this, db, message_device_id_map);
+ stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(tag_message_listener);
+ }
+
+ public void set_blind_trust(Account account, Jid jid, bool blind_trust) {
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return;
+ db.trust.update()
+ .with(db.trust.identity_id, "=", identity_id)
+ .with(db.trust.address_name, "=", jid.bare_jid.to_string())
+ .set(db.trust.blind_trust, blind_trust).perform();
+ }
+
+ public void set_device_trust(Account account, Jid jid, int device_id, TrustLevel trust_level) {
+ int identity_id = db.identity.get_id(account.id);
+ db.identity_meta.update()
+ .with(db.identity_meta.identity_id, "=", identity_id)
+ .with(db.identity_meta.address_name, "=", jid.bare_jid.to_string())
+ .with(db.identity_meta.device_id, "=", device_id)
+ .set(db.identity_meta.trust_level, trust_level).perform();
+
+ // Hide messages from untrusted or unknown devices
+ string selection = null;
+ string[] selection_args = {};
+ var app_db = Application.get_default().db;
+ foreach (Row row in db.content_item_meta.with_device(identity_id, jid.bare_jid.to_string(), device_id).with(db.content_item_meta.trusted_when_received, "=", false)) {
+ if (selection == null) {
+ selection = @"$(app_db.content_item.id) = ?";
+ } else {
+ selection += @" OR $(app_db.content_item.id) = ?";
+ }
+ selection_args += row[db.content_item_meta.content_item_id].to_string();
+ }
+ if (selection != null) {
+ app_db.content_item.update()
+ .set(app_db.content_item.hide, trust_level == TrustLevel.UNTRUSTED || trust_level == TrustLevel.UNKNOWN)
+ .where(selection, selection_args)
+ .perform();
+ }
+
+ if (trust_level == TrustLevel.TRUSTED) {
+ db.identity_meta.update_last_message_untrusted(identity_id, device_id, null);
+ bad_message_state_updated(account, jid, device_id);
+ }
+ }
+
+ public bool is_known_address(Account account, Jid jid) {
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return false;
+ return db.identity_meta.with_address(identity_id, jid.to_string()).with(db.identity_meta.last_active, ">", 0).count() > 0;
+ }
+
+ public Gee.List<int32> get_trusted_devices(Account account, Jid jid) {
+ Gee.List<int32> devices = new ArrayList<int32>();
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return devices;
+ foreach (Row device in db.identity_meta.get_trusted_devices(identity_id, jid.bare_jid.to_string())) {
+ if(device[db.identity_meta.trust_level] != TrustLevel.UNKNOWN || device[db.identity_meta.identity_key_public_base64] == null)
+ devices.add(device[db.identity_meta.device_id]);
+ }
+ return devices;
+ }
+
+ private class TagMessageListener : MessageListener {
+ public string[] after_actions_const = new string[]{ "STORE" };
+ public override string action_group { get { return "DECRYPT_TAG"; } }
+ public override string[] after_actions { get { return after_actions_const; } }
+
+ private StreamInteractor stream_interactor;
+ private TrustManager trust_manager;
+ private Database db;
+ private HashMap<Message, int> message_device_id_map;
+
+ public TagMessageListener(StreamInteractor stream_interactor, TrustManager trust_manager, Database db, HashMap<Message, int> message_device_id_map) {
+ this.stream_interactor = stream_interactor;
+ this.trust_manager = trust_manager;
+ this.db = db;
+ this.message_device_id_map = message_device_id_map;
+ }
+
+ public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) {
+ int device_id = 0;
+ if (message_device_id_map.has_key(message)) {
+ device_id = message_device_id_map[message];
+ message_device_id_map.unset(message);
+ }
+
+ // TODO: Handling of files
+
+ ContentItem? content_item = stream_interactor.get_module(ContentItemStore.IDENTITY).get_item_by_foreign(conversation, 1, message.id);
+
+ if (content_item != null && device_id != 0) {
+ Jid jid = content_item.jid;
+ if (conversation.type_ == Conversation.Type.GROUPCHAT) {
+ jid = message.real_jid;
+ }
+
+ int identity_id = db.identity.get_id(conversation.account.id);
+ TrustLevel trust_level = (TrustLevel) db.identity_meta.get_device(identity_id, jid.bare_jid.to_string(), device_id)[db.identity_meta.trust_level];
+ if (trust_level == TrustLevel.UNTRUSTED || trust_level == TrustLevel.UNKNOWN) {
+ stream_interactor.get_module(ContentItemStore.IDENTITY).set_item_hide(content_item, true);
+ db.identity_meta.update_last_message_untrusted(identity_id, device_id, message.time);
+ trust_manager.bad_message_state_updated(conversation.account, jid, device_id);
+ }
+
+ db.content_item_meta.insert()
+ .value(db.content_item_meta.content_item_id, content_item.id)
+ .value(db.content_item_meta.identity_id, identity_id)
+ .value(db.content_item_meta.address_name, jid.bare_jid.to_string())
+ .value(db.content_item_meta.device_id, device_id)
+ .value(db.content_item_meta.trusted_when_received, trust_level != TrustLevel.UNTRUSTED)
+ .perform();
+ }
+ return false;
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/plugin.vala b/main/src/plugins/omemo/plugin.vala
new file mode 100644
index 00000000..34528bde
--- /dev/null
+++ b/main/src/plugins/omemo/plugin.vala
@@ -0,0 +1,87 @@
+using Gee;
+using Dino.Crypto.Omemo;
+using Dino.Entities;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+public class Plugin : RootInterface, Object {
+ public const bool DEBUG = false;
+ private static Context? _context;
+ public static Context get_context() {
+ assert(_context != null);
+ return (!)_context;
+ }
+ public static bool ensure_context() {
+ lock(_context) {
+ try {
+ if (_context == null) {
+ _context = new Context(DEBUG);
+ }
+ return true;
+ } catch (Error e) {
+ warning("Error initializing libomemo-c Context %s", e.message);
+ return false;
+ }
+ }
+ }
+
+ public Dino.Application app;
+ public Database db;
+ public EncryptionListEntry list_entry;
+ public ContactDetailsProvider contact_details_provider;
+ public DeviceNotificationPopulator device_notification_populator;
+ public OwnNotifications own_notifications;
+ public TrustManager trust_manager;
+ public HashMap<Account, OmemoDecryptor> decryptors = new HashMap<Account, OmemoDecryptor>(Account.hash_func, Account.equals_func);
+ public HashMap<Account, OmemoEncryptor> encryptors = new HashMap<Account, OmemoEncryptor>(Account.hash_func, Account.equals_func);
+
+ public void registered(Dino.Application app) {
+ ensure_context();
+ this.app = app;
+ this.db = new Database(Path.build_filename(Application.get_storage_dir(), "omemo.db"));
+ this.list_entry = new EncryptionListEntry(this);
+ this.contact_details_provider = new ContactDetailsProvider(this);
+ this.device_notification_populator = new DeviceNotificationPopulator(this, this.app.stream_interactor);
+ this.trust_manager = new TrustManager(this.app.stream_interactor, this.db);
+
+ this.app.plugin_registry.register_encryption_list_entry(list_entry);
+ this.app.plugin_registry.register_encryption_preferences_entry(new OmemoPreferencesEntry(this));
+ this.app.plugin_registry.register_contact_details_entry(contact_details_provider);
+ this.app.plugin_registry.register_notification_populator(device_notification_populator);
+ this.app.plugin_registry.register_conversation_addition_populator(new BadMessagesPopulator(this.app.stream_interactor, this));
+ this.app.plugin_registry.register_call_entryption_entry(DtlsSrtpVerificationDraft.NS_URI, new CallEncryptionEntry(db));
+
+ this.app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => {
+ Store store = Plugin.get_context().create_store();
+ list.add(new StreamModule(store));
+ decryptors[account] = new OmemoDecryptor(account, app.stream_interactor, trust_manager, db, store);
+ list.add(decryptors[account]);
+ encryptors[account] = new OmemoEncryptor(account, trust_manager, store);
+ list.add(encryptors[account]);
+ list.add(new JetOmemo.Module());
+ list.add(new DtlsSrtpVerificationDraft.StreamModule());
+ this.own_notifications = new OwnNotifications(this, this.app.stream_interactor, account);
+ });
+
+ app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(new DecryptMessageListener(decryptors));
+ app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new OmemoFileDecryptor());
+ app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new OmemoFileEncryptor());
+ JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.OMEMO, new JetOmemo.EncryptionHelper(app.stream_interactor));
+
+ Manager.start(this.app.stream_interactor, db, trust_manager, encryptors);
+ }
+
+ public void shutdown() {
+ // Nothing to do
+ }
+
+ public bool has_new_devices(Account account, Xmpp.Jid jid) {
+ int identity_id = db.identity.get_id(account.id);
+ if (identity_id < 0) return false;
+
+ return db.identity_meta.get_new_devices(identity_id, jid.bare_jid.to_string()).count() > 0;
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/protocol/bundle.vala b/main/src/plugins/omemo/protocol/bundle.vala
new file mode 100644
index 00000000..b7e2b9cf
--- /dev/null
+++ b/main/src/plugins/omemo/protocol/bundle.vala
@@ -0,0 +1,88 @@
+using Gee;
+using Omemo;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class Bundle {
+ public StanzaNode? node;
+
+ public Bundle(StanzaNode? node) {
+ this.node = node;
+ assert(Plugin.ensure_context());
+ }
+
+ public int32 signed_pre_key_id { get {
+ if (node == null) return -1;
+ string? id = ((!)node).get_deep_attribute("signedPreKeyPublic", "signedPreKeyId");
+ if (id == null) return -1;
+ return int.parse((!)id);
+ }}
+
+ public ECPublicKey? signed_pre_key { owned get {
+ if (node == null) return null;
+ string? key = ((!)node).get_deep_string_content("signedPreKeyPublic");
+ if (key == null) return null;
+ try {
+ return Plugin.get_context().decode_public_key(Base64.decode((!)key));
+ } catch (Error e) {
+ return null;
+ }
+ }}
+
+ public uint8[]? signed_pre_key_signature { owned get {
+ if (node == null) return null;
+ string? sig = ((!)node).get_deep_string_content("signedPreKeySignature");
+ if (sig == null) return null;
+ return Base64.decode((!)sig);
+ }}
+
+ public ECPublicKey? identity_key { owned get {
+ if (node == null) return null;
+ string? key = ((!)node).get_deep_string_content("identityKey");
+ if (key == null) return null;
+ try {
+ return Plugin.get_context().decode_public_key(Base64.decode((!)key));
+ } catch (Error e) {
+ return null;
+ }
+ }}
+
+ public ArrayList<PreKey> pre_keys { owned get {
+ ArrayList<PreKey> list = new ArrayList<PreKey>();
+ if (node == null || ((!)node).get_subnode("prekeys") == null) return list;
+ ((!)node).get_deep_subnodes("prekeys", "preKeyPublic")
+ .filter((node) => ((!)node).get_attribute("preKeyId") != null)
+ .map<PreKey>(PreKey.create)
+ .foreach((key) => list.add(key));
+ return list;
+ }}
+
+ public class PreKey {
+ private StanzaNode node;
+
+ public static PreKey create(owned StanzaNode node) {
+ return new PreKey(node);
+ }
+
+ public PreKey(StanzaNode node) {
+ this.node = node;
+ }
+
+ public int32 key_id { get {
+ return int.parse(node.get_attribute("preKeyId") ?? "-1");
+ }}
+
+ public ECPublicKey? key { owned get {
+ string? key = node.get_string_content();
+ if (key == null) return null;
+ try {
+ return Plugin.get_context().decode_public_key(Base64.decode((!)key));
+ } catch (Error e) {
+ return null;
+ }
+ }}
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/protocol/message_flag.vala b/main/src/plugins/omemo/protocol/message_flag.vala
new file mode 100644
index 00000000..ba9ea16e
--- /dev/null
+++ b/main/src/plugins/omemo/protocol/message_flag.vala
@@ -0,0 +1,23 @@
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class MessageFlag : Xmpp.MessageFlag {
+ public const string id = "omemo";
+
+ public bool decrypted = false;
+
+ public static MessageFlag? get_flag(MessageStanza message) {
+ return (MessageFlag) message.get_flag(NS_URI, id);
+ }
+
+ public override string get_ns() {
+ return NS_URI;
+ }
+
+ public override string get_id() {
+ return id;
+ }
+}
+
+} \ No newline at end of file
diff --git a/main/src/plugins/omemo/protocol/stream_module.vala b/main/src/plugins/omemo/protocol/stream_module.vala
new file mode 100644
index 00000000..695c3508
--- /dev/null
+++ b/main/src/plugins/omemo/protocol/stream_module.vala
@@ -0,0 +1,332 @@
+using Gee;
+using Dino.Crypto.Omemo;
+using Xmpp;
+using Xmpp.Xep;
+using Omemo;
+
+namespace Dino.Plugins.Omemo {
+
+internal const string NS_URI = "eu.siacs.conversations.axolotl";
+private const string NODE_DEVICELIST = NS_URI + ".devicelist";
+private const string NODE_BUNDLES = NS_URI + ".bundles";
+private const string NODE_VERIFICATION = NS_URI + ".verification";
+
+private const int NUM_KEYS_TO_PUBLISH = 100;
+
+public class StreamModule : XmppStreamModule {
+ public static Xmpp.ModuleIdentity<StreamModule> IDENTITY = new Xmpp.ModuleIdentity<StreamModule>(NS_URI, "omemo_module");
+ private static TimeSpan IGNORE_TIME = TimeSpan.MINUTE;
+
+ public Store store { public get; private set; }
+ private ConcurrentSet<string> active_bundle_requests = new ConcurrentSet<string>();
+ private HashMap<Jid, Future<ArrayList<int32>>> active_devicelist_requests = new HashMap<Jid, Future<ArrayList<int32>>>(Jid.hash_func, Jid.equals_func);
+ private Map<string, DateTime> device_ignore_time = new HashMap<string, DateTime>();
+
+ public signal void device_list_loaded(Jid jid, ArrayList<int32> devices);
+ public signal void bundle_fetched(Jid jid, int device_id, Bundle bundle);
+ public signal void bundle_fetch_failed(Jid jid, int device_id);
+
+ public StreamModule(Store store) {
+ this.store = store;
+ }
+
+ public override void attach(XmppStream stream) {
+ stream.get_module(Pubsub.Module.IDENTITY).add_filtered_notification(stream, NODE_DEVICELIST,
+ (stream, jid, id, node) => parse_device_list(stream, jid, id, node), null, null);
+ }
+
+ public override void detach(XmppStream stream) {
+ stream.get_module(Pubsub.Module.IDENTITY).remove_filtered_notification(stream, NODE_DEVICELIST);
+ }
+
+ public async ArrayList<int32> request_user_devicelist(XmppStream stream, Jid jid) {
+ var future = active_devicelist_requests[jid];
+ if (future == null) {
+ var promise = new Promise<ArrayList<int32>?>();
+ future = promise.future;
+ active_devicelist_requests[jid] = future;
+
+ stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid, NODE_DEVICELIST, (stream, jid, id, node) => {
+ ArrayList<int32> device_list = parse_device_list(stream, jid, id, node);
+ promise.set_value(device_list);
+ active_devicelist_requests.unset(jid);
+ });
+ }
+
+ try {
+ ArrayList<int32> device_list = yield future.wait_async();
+ return device_list;
+ } catch (FutureError error) {
+ warning("Future error when waiting for device list: %s", error.message);
+ return new ArrayList<int32>();
+ }
+ }
+
+ public ArrayList<int32> parse_device_list(XmppStream stream, Jid jid, string? id, StanzaNode? node_) {
+ ArrayList<int32> device_list = new ArrayList<int32>();
+
+ StanzaNode node = node_ ?? new StanzaNode.build("list", NS_URI).add_self_xmlns();
+ Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid;
+ if (my_jid == null) return device_list;
+ if (jid.equals_bare(my_jid) && store.local_registration_id != 0) {
+ bool am_on_devicelist = false;
+ foreach (StanzaNode device_node in node.get_subnodes("device")) {
+ int device_id = device_node.get_attribute_int("id");
+ if (store.local_registration_id == device_id) {
+ am_on_devicelist = true;
+ }
+ }
+ if (!am_on_devicelist) {
+ debug("Not on device list, adding id");
+ node.put_node(new StanzaNode.build("device", NS_URI).put_attribute("id", store.local_registration_id.to_string()));
+ stream.get_module(Pubsub.Module.IDENTITY).publish.begin(stream, jid, NODE_DEVICELIST, id, node, null, true, () => {
+ try_make_node_public.begin(stream, NODE_DEVICELIST);
+ });
+ }
+ publish_bundles_if_needed(stream, jid);
+ }
+
+ foreach (StanzaNode device_node in node.get_subnodes("device")) {
+ device_list.add(device_node.get_attribute_int("id"));
+ }
+ device_list_loaded(jid, device_list);
+
+ return device_list;
+ }
+
+ public void fetch_bundles(XmppStream stream, Jid jid, Gee.List<int32> devices) {
+ Address address = new Address(jid.bare_jid.to_string(), 0);
+ foreach(int32 device_id in devices) {
+ if (!is_ignored_device(jid, device_id)) {
+ address.device_id = device_id;
+ try {
+ if (!store.contains_session(address)) {
+ fetch_bundle(stream, jid, device_id);
+ }
+ } catch (Error e) {
+ // Ignore
+ }
+ }
+ }
+ address.device_id = 0; // TODO: Hack to have address obj live longer
+ }
+
+ public void fetch_bundle(XmppStream stream, Jid jid, int device_id, bool ignore_if_non_present = true) {
+ if (active_bundle_requests.add(jid.bare_jid.to_string() + @":$device_id")) {
+ debug("Asking for bundle for %s/%d", jid.bare_jid.to_string(), device_id);
+ stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid.bare_jid, @"$NODE_BUNDLES:$device_id", (stream, jid, id, node) => {
+ on_other_bundle_result(stream, jid, device_id, id, node, ignore_if_non_present);
+ });
+ }
+ }
+
+ public void ignore_device(Jid jid, int32 device_id) {
+ if (device_id <= 0) return;
+ lock (device_ignore_time) {
+ device_ignore_time[jid.bare_jid.to_string() + @":$device_id"] = new DateTime.now_utc();
+ }
+ }
+
+ public void unignore_device(Jid jid, int32 device_id) {
+ if (device_id <= 0) return;
+ lock (device_ignore_time) {
+ device_ignore_time.unset(jid.bare_jid.to_string() + @":$device_id");
+ }
+ }
+
+ public bool is_ignored_device(Jid jid, int32 device_id) {
+ if (device_id <= 0) return true;
+ lock (device_ignore_time) {
+ string id = jid.bare_jid.to_string() + @":$device_id";
+ if (device_ignore_time.has_key(id)) {
+ return new DateTime.now_utc().difference(device_ignore_time[id]) < IGNORE_TIME;
+ }
+ }
+ return false;
+ }
+
+ public void clear_device_list(XmppStream stream) {
+ stream.get_module(Pubsub.Module.IDENTITY).delete_node(stream, null, NODE_DEVICELIST);
+ }
+
+ private void on_other_bundle_result(XmppStream stream, Jid jid, int device_id, string? id, StanzaNode? node, bool ignore_if_non_present) {
+ if (node == null) {
+ // Device not registered, shouldn't exist
+ if (ignore_if_non_present) {
+ debug("Ignoring device %s/%d: No bundle", jid.bare_jid.to_string(), device_id);
+ stream.get_module(IDENTITY).ignore_device(jid, device_id);
+ }
+ bundle_fetch_failed(jid, device_id);
+ } else {
+ Bundle bundle = new Bundle(node);
+ stream.get_module(IDENTITY).unignore_device(jid, device_id);
+ debug("Received bundle for %s/%d: %s", jid.bare_jid.to_string(), device_id, Base64.encode(bundle.identity_key.serialize()));
+ bundle_fetched(jid, device_id, bundle);
+ }
+ stream.get_module(IDENTITY).active_bundle_requests.remove(jid.bare_jid.to_string() + @":$device_id");
+ }
+
+ public bool start_session(XmppStream stream, Jid jid, int32 device_id, Bundle bundle) {
+ bool fail = false;
+ int32 signed_pre_key_id = bundle.signed_pre_key_id;
+ ECPublicKey? signed_pre_key = bundle.signed_pre_key;
+ uint8[] signed_pre_key_signature = bundle.signed_pre_key_signature;
+ ECPublicKey? identity_key = bundle.identity_key;
+
+ ArrayList<Bundle.PreKey> pre_keys = bundle.pre_keys;
+ if (signed_pre_key_id < 0 || signed_pre_key == null || identity_key == null || pre_keys.size == 0) {
+ fail = true;
+ } else {
+ int pre_key_idx = Random.int_range(0, pre_keys.size);
+ int32 pre_key_id = pre_keys[pre_key_idx].key_id;
+ ECPublicKey? pre_key = pre_keys[pre_key_idx].key;
+ if (pre_key_id < 0 || pre_key == null) {
+ fail = true;
+ } else {
+ Address address = new Address(jid.bare_jid.to_string(), device_id);
+ try {
+ if (store.contains_session(address)) {
+ return false;
+ }
+ debug("Starting new session for encryption with %s/%d", jid.bare_jid.to_string(), device_id);
+ SessionBuilder builder = store.create_session_builder(address);
+ builder.process_pre_key_bundle(create_pre_key_bundle(device_id, device_id, pre_key_id, pre_key, signed_pre_key_id, signed_pre_key, signed_pre_key_signature, identity_key));
+ } catch (Error e) {
+ debug("Can't create session with %s/%d: %s", jid.bare_jid.to_string(), device_id, e.message);
+ fail = true;
+ }
+ address.device_id = 0; // TODO: Hack to have address obj live longer
+ }
+ }
+ if (fail) {
+ debug("Ignoring device %s/%d: Bad bundle: %s", jid.bare_jid.to_string(), device_id, bundle.node.to_string());
+ stream.get_module(IDENTITY).ignore_device(jid, device_id);
+ }
+ return true;
+ }
+
+ public void publish_bundles_if_needed(XmppStream stream, Jid jid) {
+ if (active_bundle_requests.add(jid.bare_jid.to_string() + @":$(store.local_registration_id)")) {
+ stream.get_module(Pubsub.Module.IDENTITY).request(stream, jid, @"$NODE_BUNDLES:$(store.local_registration_id)", on_self_bundle_result);
+ }
+ }
+
+ private void on_self_bundle_result(XmppStream stream, Jid jid, string? id, StanzaNode? node) {
+ if (!Plugin.ensure_context()) return;
+ Map<int, ECPublicKey> keys = new HashMap<int, ECPublicKey>();
+ ECPublicKey? identity_key = null;
+ int32 signed_pre_key_id = -1;
+ ECPublicKey? signed_pre_key = null;
+ SignedPreKeyRecord? signed_pre_key_record = null;
+ bool changed = false;
+ if (node == null) {
+ identity_key = store.identity_key_pair.public;
+ changed = true;
+ } else {
+ Bundle bundle = new Bundle(node);
+ foreach (Bundle.PreKey prekey in bundle.pre_keys) {
+ ECPublicKey? key = prekey.key;
+ if (key != null) {
+ keys[prekey.key_id] = (!)key;
+ }
+ }
+ identity_key = bundle.identity_key;
+ signed_pre_key_id = bundle.signed_pre_key_id;
+ signed_pre_key = bundle.signed_pre_key;
+ }
+
+ try {
+ // Validate IdentityKey
+ if (identity_key == null || store.identity_key_pair.public.compare((!)identity_key) != 0) {
+ changed = true;
+ }
+ IdentityKeyPair identity_key_pair = store.identity_key_pair;
+
+ // Validate signedPreKeyRecord + ID
+ if (signed_pre_key == null || signed_pre_key_id == -1 || !store.contains_signed_pre_key(signed_pre_key_id) || store.load_signed_pre_key(signed_pre_key_id).key_pair.public.compare((!)signed_pre_key) != 0) {
+ signed_pre_key_id = Random.int_range(1, int32.MAX); // TODO: No random, use ordered number
+ signed_pre_key_record = Plugin.get_context().generate_signed_pre_key(identity_key_pair, signed_pre_key_id);
+ store.store_signed_pre_key((!)signed_pre_key_record);
+ changed = true;
+ } else {
+ signed_pre_key_record = store.load_signed_pre_key(signed_pre_key_id);
+ }
+
+ // Validate PreKeys
+ Set<PreKeyRecord> pre_key_records = new HashSet<PreKeyRecord>();
+ foreach (var entry in keys.entries) {
+ if (store.contains_pre_key(entry.key)) {
+ PreKeyRecord record = store.load_pre_key(entry.key);
+ if (record.key_pair.public.compare(entry.value) == 0) {
+ pre_key_records.add(record);
+ }
+ }
+ }
+ int new_keys = NUM_KEYS_TO_PUBLISH - pre_key_records.size;
+ if (new_keys > 0) {
+ int32 next_id = Random.int_range(1, int32.MAX); // TODO: No random, use ordered number
+ Set<PreKeyRecord> new_records = Plugin.get_context().generate_pre_keys((uint)next_id, (uint)new_keys);
+ pre_key_records.add_all(new_records);
+ foreach (PreKeyRecord record in new_records) {
+ store.store_pre_key(record);
+ }
+ changed = true;
+ }
+
+ if (changed) {
+ publish_bundles.begin(stream, (!)signed_pre_key_record, identity_key_pair, pre_key_records, (int32) store.local_registration_id);
+ }
+ } catch (Error e) {
+ warning(@"Unexpected error while publishing bundle: $(e.message)\n");
+ }
+ stream.get_module(IDENTITY).active_bundle_requests.remove(jid.bare_jid.to_string() + @":$(store.local_registration_id)");
+ }
+
+ public async void publish_bundles(XmppStream stream, SignedPreKeyRecord signed_pre_key_record, IdentityKeyPair identity_key_pair, Set<PreKeyRecord> pre_key_records, int32 device_id) throws Error {
+ ECKeyPair tmp;
+ StanzaNode bundle = new StanzaNode.build("bundle", NS_URI)
+ .add_self_xmlns()
+ .put_node(new StanzaNode.build("signedPreKeyPublic", NS_URI)
+ .put_attribute("signedPreKeyId", signed_pre_key_record.id.to_string())
+ .put_node(new StanzaNode.text(Base64.encode((tmp = signed_pre_key_record.key_pair).public.serialize()))))
+ .put_node(new StanzaNode.build("signedPreKeySignature", NS_URI)
+ .put_node(new StanzaNode.text(Base64.encode(signed_pre_key_record.signature))))
+ .put_node(new StanzaNode.build("identityKey", NS_URI)
+ .put_node(new StanzaNode.text(Base64.encode(identity_key_pair.public.serialize()))));
+ StanzaNode prekeys = new StanzaNode.build("prekeys", NS_URI);
+ foreach (PreKeyRecord pre_key_record in pre_key_records) {
+ prekeys.put_node(new StanzaNode.build("preKeyPublic", NS_URI)
+ .put_attribute("preKeyId", pre_key_record.id.to_string())
+ .put_node(new StanzaNode.text(Base64.encode(pre_key_record.key_pair.public.serialize()))));
+ }
+ bundle.put_node(prekeys);
+
+ string node_id = @"$NODE_BUNDLES:$device_id";
+ yield stream.get_module(Pubsub.Module.IDENTITY).publish(stream, null, node_id, "1", bundle);
+ yield try_make_node_public(stream, node_id);
+
+ }
+
+ private async void try_make_node_public(XmppStream stream, string node_id) {
+ DataForms.DataForm? data_form = yield stream.get_module(Pubsub.Module.IDENTITY).request_node_config(stream, null, node_id);
+ if (data_form == null) return;
+
+ foreach (DataForms.DataForm.Field field in data_form.fields) {
+ if (field.var == "pubsub#access_model" && field.get_value_string() != Pubsub.ACCESS_MODEL_OPEN) {
+ field.set_value_string(Pubsub.ACCESS_MODEL_OPEN);
+ yield stream.get_module(Pubsub.Module.IDENTITY).submit_node_config(stream, data_form, node_id);
+ break;
+ }
+ }
+ }
+
+ public override string get_ns() {
+ return NS_URI;
+ }
+
+ public override string get_id() {
+ return IDENTITY.id;
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/trust_level.vala b/main/src/plugins/omemo/trust_level.vala
new file mode 100644
index 00000000..afcfb0fb
--- /dev/null
+++ b/main/src/plugins/omemo/trust_level.vala
@@ -0,0 +1,15 @@
+namespace Dino.Plugins.Omemo {
+
+public enum TrustLevel {
+ VERIFIED,
+ TRUSTED,
+ UNTRUSTED,
+ UNKNOWN;
+
+ public string to_string() {
+ int val = this;
+ return val.to_string();
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/ui/bad_messages_populator.vala b/main/src/plugins/omemo/ui/bad_messages_populator.vala
new file mode 100644
index 00000000..c291a422
--- /dev/null
+++ b/main/src/plugins/omemo/ui/bad_messages_populator.vala
@@ -0,0 +1,198 @@
+using Gee;
+using Gtk;
+
+using Dino.Entities;
+using Dino.Qlite;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public enum BadnessType {
+ UNTRUSTED,
+ UNDECRYPTABLE
+}
+
+public class BadMessagesPopulator : Plugins.ConversationItemPopulator, Plugins.ConversationAdditionPopulator, Object {
+
+ public string id { get { return "bad_omemo_messages"; } }
+
+ private StreamInteractor stream_interactor;
+ private Plugin plugin;
+ private Database db;
+
+ private Conversation? current_conversation;
+ private Plugins.ConversationItemCollection? item_collection;
+
+ private Gee.List<BadMessageItem> bad_items = new ArrayList<BadMessageItem>();
+
+ public BadMessagesPopulator(StreamInteractor stream_interactor, Plugin plugin) {
+ this.stream_interactor = stream_interactor;
+ this.plugin = plugin;
+ this.db = plugin.db;
+
+ plugin.trust_manager.bad_message_state_updated.connect((account, jid, device_id) => {
+ clear_state();
+ init_state();
+ });
+ }
+
+ private void init_state() {
+ if (current_conversation == null) return;
+ if (current_conversation.type_ == Conversation.Type.GROUPCHAT_PM) return;
+
+ var qry = db.identity_meta.select()
+ .join_with(db.identity, db.identity.id, db.identity_meta.identity_id)
+ .with(db.identity.account_id, "=", current_conversation.account.id)
+ .where("last_message_untrusted is not NULL OR last_message_undecryptable is not NULL");
+
+ switch (current_conversation.type_) {
+ case Conversation.Type.CHAT:
+ qry.with(db.identity_meta.address_name, "=", current_conversation.counterpart.to_string());
+ break;
+ case Conversation.Type.GROUPCHAT:
+ bool is_private = stream_interactor.get_module(MucManager.IDENTITY).is_private_room(current_conversation.account, current_conversation.counterpart);
+ if (!is_private) return;
+
+ var list = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(current_conversation.counterpart, current_conversation.account);
+ if (list == null || list.is_empty) return;
+
+ var selection = new StringBuilder();
+ string[] selection_args = {};
+ foreach (Jid jid in list) {
+ if (selection.len == 0) {
+ selection.append(@" ($(db.identity_meta.address_name) = ?");
+ } else {
+ selection.append(@" OR $(db.identity_meta.address_name) = ?");
+ }
+ selection_args += jid.to_string();
+ }
+ selection.append(")");
+ qry.where(selection.str, selection_args);
+ break;
+ case Conversation.Type.GROUPCHAT_PM:
+ break;
+ }
+
+ foreach (Row row in qry) {
+ Jid jid = Jid.from_string(row[db.identity_meta.address_name]);
+ if (!db.identity_meta.last_message_untrusted.is_null(row)) {
+ DateTime time = new DateTime.from_unix_utc(row[db.identity_meta.last_message_untrusted]);
+ var item = new BadMessageItem(plugin, current_conversation, jid, time, BadnessType.UNTRUSTED);
+ bad_items.add(item);
+ item_collection.insert_item(item);
+ }
+ if (!db.identity_meta.last_message_undecryptable.is_null(row)) {
+ DateTime time = new DateTime.from_unix_utc(row[db.identity_meta.last_message_undecryptable]);
+ var item = new BadMessageItem(plugin, current_conversation, jid, time, BadnessType.UNDECRYPTABLE);
+ bad_items.add(item);
+ item_collection.insert_item(item);
+ }
+ }
+ }
+
+ private void clear_state() {
+ foreach (BadMessageItem bad_item in bad_items) {
+ item_collection.remove_item(bad_item);
+ }
+ bad_items.clear();
+ }
+
+ public void init(Conversation conversation, Plugins.ConversationItemCollection item_collection, Plugins.WidgetType type) {
+ current_conversation = conversation;
+ this.item_collection = item_collection;
+
+ init_state();
+ }
+
+ public void close(Conversation conversation) {
+ clear_state();
+ }
+
+ public void populate_timespan(Conversation conversation, DateTime after, DateTime before) { }
+}
+
+public class BadMessageItem : Plugins.MetaConversationItem {
+
+ private Plugin plugin;
+ private Conversation conversation;
+ private Jid problem_jid;
+ private BadnessType badness_type;
+
+ public BadMessageItem(Plugin plugin, Conversation conversation, Jid jid, DateTime date, BadnessType badness_type) {
+ this.plugin = plugin;
+ this.conversation = conversation;
+ this.problem_jid = jid;
+ this.time = date;
+ this.badness_type = badness_type;
+ }
+
+ public override Object? get_widget(Plugins.ConversationItemWidgetInterface outer, Plugins.WidgetType widget_type) {
+ return new BadMessagesWidget(plugin, conversation, problem_jid, badness_type);
+ }
+
+ public override Gee.List<Plugins.MessageAction>? get_item_actions(Plugins.WidgetType type) { return null; }
+}
+
+public class BadMessagesWidget : Box {
+ private Plugin plugin;
+ private Conversation conversation;
+ private Jid jid;
+ private Label label;
+
+ public BadMessagesWidget(Plugin plugin, Conversation conversation, Jid jid, BadnessType badness_type) {
+ Object(orientation:Orientation.HORIZONTAL, spacing:5);
+
+ this.plugin = plugin;
+ this.conversation = conversation;
+ this.jid = jid;
+ this.halign = Align.CENTER;
+ this.visible = true;
+
+ string who = "";
+ if (conversation.type_ == Conversation.Type.CHAT) {
+ who = Dino.get_participant_display_name(plugin.app.stream_interactor, conversation, jid);
+ } else if (conversation.type_ == Conversation.Type.GROUPCHAT) {
+ who = jid.to_string();
+ // `jid` is a real JID. In MUCs, try to show nicks instead (given that the JID is currently online)
+ var occupants = plugin.app.stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account);
+ if (occupants == null) return;
+ foreach (Jid occupant in occupants) {
+ if (jid.equals_bare(plugin.app.stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(occupant, conversation.account))) {
+ who = occupant.resourcepart;
+ }
+ }
+ }
+
+ string warning_text = "";
+ if (badness_type == BadnessType.UNTRUSTED) {
+ warning_text = _("%s has been using an untrusted device. You won't see messages from devices that you do not trust.").printf(who) +
+ " <a href=\"\">%s</a>".printf(_("Manage devices"));
+ } else {
+ warning_text += _("%s does not trust this device. That means, you might be missing messages.").printf(who);
+ }
+ label = new Label(warning_text) { margin_start=70, margin_end=70, justify=Justification.CENTER, use_markup=true, selectable=true, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true };
+ label.add_css_class("dim-label");
+ this.append(label);
+
+ if (badness_type == BadnessType.UNTRUSTED) {
+ label.activate_link.connect(on_label_activate_link);
+ }
+ }
+
+ private bool on_label_activate_link() {
+ var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.string("encryption")});
+ GLib.Application.get_default().activate_action("open-conversation-details", variant);
+ return false;
+ }
+
+ public override void dispose() {
+ if (label != null) {
+ label.unparent();
+ label.dispose();
+ label = null;
+ }
+ base.dispose();
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/ui/call_encryption_entry.vala b/main/src/plugins/omemo/ui/call_encryption_entry.vala
new file mode 100644
index 00000000..04f0443c
--- /dev/null
+++ b/main/src/plugins/omemo/ui/call_encryption_entry.vala
@@ -0,0 +1,58 @@
+using Gtk;
+
+using Dino.Entities;
+using Dino.Qlite;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+ public class CallEncryptionEntry : Plugins.CallEncryptionEntry, Object {
+ private Database db;
+
+ public CallEncryptionEntry(Database db) {
+ this.db = db;
+ }
+
+ public Plugins.CallEncryptionWidget? get_widget(Account account, Xmpp.Xep.Jingle.ContentEncryption encryption) {
+ DtlsSrtpVerificationDraft.OmemoContentEncryption? omemo_encryption = encryption as DtlsSrtpVerificationDraft.OmemoContentEncryption;
+ if (omemo_encryption == null) return null;
+
+ int identity_id = db.identity.get_id(account.id);
+ Row? device = db.identity_meta.get_device(identity_id, omemo_encryption.jid.to_string(), omemo_encryption.sid);
+ if (device == null) return null;
+ TrustLevel trust = (TrustLevel) device[db.identity_meta.trust_level];
+
+ return new CallEncryptionWidget(trust);
+ }
+ }
+
+ public class CallEncryptionWidget : Plugins.CallEncryptionWidget, Object {
+
+ string? title = null;
+ string? icon = null;
+ bool should_show_keys = false;
+
+ public CallEncryptionWidget(TrustLevel trust) {
+ if (trust == TrustLevel.VERIFIED) {
+ title = "This call is <b>encrypted and verified</b> with OMEMO.";
+ icon = "dino-security-high-symbolic";
+ should_show_keys = false;
+ } else {
+ title = "This call is encrypted with OMEMO.";
+ should_show_keys = true;
+ }
+ }
+
+ public string? get_title() {
+ return title;
+ }
+
+ public string? get_icon_name() {
+ return icon;
+ }
+
+ public bool show_keys() {
+ return should_show_keys;
+ }
+ }
+}
diff --git a/main/src/plugins/omemo/ui/contact_details_provider.vala b/main/src/plugins/omemo/ui/contact_details_provider.vala
new file mode 100644
index 00000000..1b61f31d
--- /dev/null
+++ b/main/src/plugins/omemo/ui/contact_details_provider.vala
@@ -0,0 +1,28 @@
+using Gtk;
+using Gee;
+using Dino.Entities;
+
+namespace Dino.Plugins.Omemo {
+
+public class ContactDetailsProvider : Plugins.ContactDetailsProvider, Object {
+ public string id { get { return "omemo_info"; } }
+ public string tab { get { return "encryption"; } }
+
+ private Plugin plugin;
+
+ public ContactDetailsProvider(Plugin plugin) {
+ this.plugin = plugin;
+ }
+
+ public void populate(Conversation conversation, Plugins.ContactDetails contact_details, WidgetType type) { }
+
+ public Object? get_widget(Conversation conversation) {
+ if (conversation.type_ != Conversation.Type.CHAT) return null;
+
+ var widget = new OmemoPreferencesWidget(plugin);
+ widget.set_jid(conversation.account, conversation.counterpart);
+ return widget;
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/ui/device_notification_populator.vala b/main/src/plugins/omemo/ui/device_notification_populator.vala
new file mode 100644
index 00000000..7829d89c
--- /dev/null
+++ b/main/src/plugins/omemo/ui/device_notification_populator.vala
@@ -0,0 +1,90 @@
+using Dino.Entities;
+using Xmpp;
+using Gtk;
+
+namespace Dino.Plugins.Omemo {
+
+public class DeviceNotificationPopulator : NotificationPopulator, Object {
+
+ public string id { get { return "device_notification"; } }
+
+ private StreamInteractor? stream_interactor;
+ private Plugin plugin;
+ private Conversation? current_conversation;
+ private NotificationCollection? notification_collection;
+ private ConversationNotification notification;
+
+ public DeviceNotificationPopulator(Plugin plugin, StreamInteractor stream_interactor) {
+ this.stream_interactor = stream_interactor;
+ this.plugin = plugin;
+
+ stream_interactor.account_added.connect(on_account_added);
+ }
+
+ public void init(Conversation conversation, NotificationCollection notification_collection, Plugins.WidgetType type) {
+ current_conversation = conversation;
+ this.notification_collection = notification_collection;
+ if (plugin.has_new_devices(conversation.account, conversation.counterpart) && conversation.type_ == Conversation.Type.CHAT) {
+ display_notification();
+ }
+ }
+
+ public void close(Conversation conversation) {
+ notification = null;
+ }
+
+ private void display_notification() {
+ if (notification == null) {
+ notification = new ConversationNotification(plugin, current_conversation);
+ notification.should_hide.connect(should_hide);
+ notification_collection.add_meta_notification(notification);
+ }
+ }
+
+ public void should_hide() {
+ if (!plugin.has_new_devices(current_conversation.account, current_conversation.counterpart) && notification != null){
+ notification_collection.remove_meta_notification(notification);
+ notification = null;
+ }
+ }
+
+ private void on_account_added(Account account) {
+ stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).bundle_fetched.connect_after((jid, device_id, bundle) => {
+ if (current_conversation != null && jid.equals(current_conversation.counterpart) && plugin.has_new_devices(current_conversation.account, current_conversation.counterpart)) {
+ display_notification();
+ }
+ });
+ }
+}
+
+private class ConversationNotification : MetaConversationNotification {
+ private Widget widget;
+ private Plugin plugin;
+ private Jid jid;
+ private Account account;
+ public signal void should_hide();
+
+ public ConversationNotification(Plugin plugin, Conversation conversation) {
+ this.plugin = plugin;
+ this.jid = jid;
+ this.account = account;
+
+ Box box = new Box(Orientation.HORIZONTAL, 5);
+ Button manage_button = new Button.with_label(_("Manage"));
+ manage_button.clicked.connect(() => {
+ manage_button.activate();
+
+ var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.string("encryption")});
+ GLib.Application.get_default().activate_action("open-conversation-details", variant);
+ });
+ box.append(new Label(_("This contact has new devices")) { margin_end=10 });
+ box.append(manage_button);
+ widget = box;
+ }
+
+ public override Object? get_widget(WidgetType type) {
+ return widget;
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/ui/encryption_list_entry.vala b/main/src/plugins/omemo/ui/encryption_list_entry.vala
new file mode 100644
index 00000000..dfec813a
--- /dev/null
+++ b/main/src/plugins/omemo/ui/encryption_list_entry.vala
@@ -0,0 +1,78 @@
+using Gtk;
+
+using Dino.Entities;
+using Dino.Qlite;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class EncryptionListEntry : Plugins.EncryptionListEntry, Object {
+ private Plugin plugin;
+ private Database db;
+
+ public EncryptionListEntry(Plugin plugin) {
+ this.plugin = plugin;
+ this.db = plugin.db;
+ }
+
+ public Entities.Encryption encryption { get {
+ return Entities.Encryption.OMEMO;
+ }}
+
+ public string name { get {
+ return "OMEMO";
+ }}
+
+ public Object? get_encryption_icon(Entities.Conversation conversation, ContentItem content_item) {
+ return null;
+ }
+
+ public string? get_encryption_icon_name(Entities.Conversation conversation, ContentItem content_item) {
+ if (content_item.encryption != encryption) return null;
+
+ RowOption row = db.content_item_meta.select( { db.identity_meta.trust_level } ).with(db.content_item_meta.content_item_id, "=", content_item.id)
+ .join_on(db.identity_meta, @"$(db.identity_meta.address_name)=$(db.content_item_meta.address_name) AND $(db.identity_meta.device_id)=$(db.content_item_meta.device_id)")
+ .single().row();
+
+
+ if (row.is_present() && (TrustLevel) row[db.identity_meta.trust_level] == TrustLevel.VERIFIED) {
+ return "dino-security-high-symbolic";
+ }
+ return null;
+ }
+
+ public void encryption_activated(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) {
+ encryption_activated_async.begin(conversation, input_status_callback);
+ }
+
+ public async void encryption_activated_async(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) {
+ if (conversation.type_ == Conversation.Type.GROUPCHAT_PM) {
+ input_status_callback(new Plugins.InputFieldStatus("Can't use encryption in a groupchat private message.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ return;
+ }
+ MucManager muc_manager = plugin.app.stream_interactor.get_module(MucManager.IDENTITY);
+ Manager omemo_manager = plugin.app.stream_interactor.get_module(Manager.IDENTITY);
+
+ if (muc_manager.is_private_room(conversation.account, conversation.counterpart)) {
+ var offline_members = muc_manager.get_offline_members(conversation.counterpart, conversation.account);
+ if (offline_members == null) {
+ // We don't store offline members yet, and it'll be null if we're offline
+ return;
+ }
+ foreach (Jid offline_member in offline_members) {
+ bool ok = yield omemo_manager.ensure_get_keys_for_jid(conversation.account, offline_member);
+ if (!ok) {
+ input_status_callback(new Plugins.InputFieldStatus("A member does not support OMEMO: %s".printf(offline_member.to_string()), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ return;
+ }
+ }
+ return;
+ }
+
+ if (!(yield omemo_manager.ensure_get_keys_for_jid(conversation.account, conversation.counterpart.bare_jid))) {
+ input_status_callback(new Plugins.InputFieldStatus("This contact does not support %s encryption".printf("OMEMO"), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ }
+ }
+
+}
+}
diff --git a/main/src/plugins/omemo/ui/encryption_preferences_entry.vala b/main/src/plugins/omemo/ui/encryption_preferences_entry.vala
new file mode 100644
index 00000000..55faecac
--- /dev/null
+++ b/main/src/plugins/omemo/ui/encryption_preferences_entry.vala
@@ -0,0 +1,320 @@
+using Gee;
+using Gtk;
+using Qrencode;
+
+using Dino.Crypto.Omemo;
+using Dino.Entities;
+using Dino.Qlite;
+using Omemo;
+using Xmpp;
+
+namespace Dino.Plugins.Omemo {
+
+public class OmemoPreferencesEntry : Plugins.EncryptionPreferencesEntry {
+
+ OmemoPreferencesWidget widget;
+ Plugin plugin;
+
+ public OmemoPreferencesEntry(Plugin plugin) {
+ this.plugin = plugin;
+ }
+
+ public override Object? get_widget(Account account, WidgetType type) {
+ if (type != WidgetType.GTK4) return null;
+ var widget = new OmemoPreferencesWidget(plugin);
+ widget.set_jid(account, account.bare_jid);
+ return widget;
+ }
+
+ public override string id { get { return "omemo_preferences_entryption"; }}
+}
+
+[GtkTemplate (ui = "/im/dino/Dino/encryption_preferences_entry.ui")]
+public class OmemoPreferencesWidget : Adw.PreferencesGroup {
+ private Plugin plugin;
+ private Account account;
+ private Jid jid;
+ private int identity_id = 0;
+ private Store store;
+ private Set<uint32> displayed_ids = new HashSet<uint32>();
+
+ [GtkChild] private unowned Adw.SwitchRow automatically_accept_new_row;
+ [GtkChild] private unowned Adw.SwitchRow encrypt_by_default_row;
+ [GtkChild] private unowned Label new_keys_label;
+
+ [GtkChild] private unowned Adw.PreferencesGroup keys_preferences_group;
+ [GtkChild] private unowned ListBox new_keys_listbox;
+ [GtkChild] private unowned Picture qrcode_picture;
+ [GtkChild] private unowned Popover qrcode_popover;
+
+ private ArrayList<Widget> keys_preferences_group_children = new ArrayList<Widget>();
+
+ construct {
+ // If we set the strings in the .ui file, they don't get translated
+ encrypt_by_default_row.title = _("OMEMO by default");
+ encrypt_by_default_row.subtitle = _("Enable OMEMO encryption for new conversations");
+ automatically_accept_new_row.title = _("Encrypt to new devices");
+ automatically_accept_new_row.subtitle = _("Automatically encrypt to new devices from this contact.");
+ new_keys_label.label = _("New keys");
+ }
+
+ public OmemoPreferencesWidget(Plugin plugin) {
+ this.plugin = plugin;
+ }
+
+ public void set_jid(Account account, Jid jid) {
+ this.account = account;
+ this.jid = jid;
+ this.identity_id = plugin.db.identity.get_id(account.id);
+ if (identity_id <= 0) {
+ warning("OmemoPreferencesWidget missing identity_id");
+ return;
+ }
+
+ automatically_accept_new_row.active = plugin.db.trust.get_blind_trust(identity_id, jid.bare_jid.to_string(), true);
+ automatically_accept_new_row.notify["active"].connect((obj, _) => { on_auto_accept_toggled(((Adw.SwitchRow) obj).active); });
+
+ encrypt_by_default_row.visible = account.bare_jid.equals_bare(jid);
+ encrypt_by_default_row.active = plugin.app.settings.get_default_encryption(account) != Encryption.NONE;
+ encrypt_by_default_row.notify["active"].connect((obj, _) => { on_omemo_by_default_toggled(((Adw.SwitchRow) obj).active); });
+
+ Dino.Application? app = Application.get_default() as Dino.Application;
+ if (app != null) {
+ store = app.stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).store;
+ }
+
+ redraw_key_list();
+
+ // Check for unknown devices
+ fetch_unknown_bundles();
+ }
+
+ private void redraw_key_list() {
+ // Remove current widgets
+ foreach (var widget in keys_preferences_group_children) {
+ keys_preferences_group.remove(widget);
+ }
+ keys_preferences_group_children.clear();
+
+ // Dialog opened from the account settings menu
+ // Show the fingerprint for this device separately with buttons for a qrcode and to copy
+ if(jid.equals(account.bare_jid)) {
+ automatically_accept_new_row.subtitle = _("New encryption keys from your other devices will be accepted automatically.");
+ add_own_fingerprint();
+ }
+
+ //Show the normal devicelist
+ var own_id = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.device_id];
+ foreach (Row device in plugin.db.identity_meta.get_known_devices(identity_id, jid.to_string())) {
+ if(jid.equals(account.bare_jid) && device[plugin.db.identity_meta.device_id] == own_id) {
+ // If this is our own account, don't show this device twice (did it separately already)
+ continue;
+ }
+ add_fingerprint(device, (TrustLevel) device[plugin.db.identity_meta.trust_level]);
+ }
+
+ //Show any new devices for which the user must decide whether to accept or reject
+ foreach (Row device in plugin.db.identity_meta.get_new_devices(identity_id, jid.to_string())) {
+ add_new_fingerprint(device);
+ }
+ }
+
+ private static string escape_for_iri_path_segment(string s) {
+ // from RFC 3986, 2.2. Reserved Characters:
+ string SUB_DELIMS = "!$&'()*+,;=";
+ // from RFC 3986, 3.3. Path (pchar without unreserved and pct-encoded):
+ string ALLOWED_RESERVED_CHARS = SUB_DELIMS + ":@";
+ return GLib.Uri.escape_string(s, ALLOWED_RESERVED_CHARS, true);
+ }
+
+ private void fetch_unknown_bundles() {
+ Dino.Application app = Application.get_default() as Dino.Application;
+ XmppStream? stream = app.stream_interactor.get_stream(account);
+ if (stream == null) return;
+ StreamModule? module = stream.get_module(StreamModule.IDENTITY);
+ if (module == null) return;
+ module.bundle_fetched.connect_after((bundle_jid, device_id, bundle) => {
+ if (bundle_jid.equals(jid) && !displayed_ids.contains(device_id)) {
+ redraw_key_list();
+ }
+ });
+ foreach (Row device in plugin.db.identity_meta.get_unknown_devices(identity_id, jid.to_string())) {
+ try {
+ module.fetch_bundle(stream, Jid.from_string(device[plugin.db.identity_meta.address_name]), device[plugin.db.identity_meta.device_id], false);
+ } catch (InvalidJidError e) {
+ warning("Ignoring device with invalid Jid: %s", e.message);
+ }
+ }
+ }
+
+ private void add_own_fingerprint() {
+ string own_b64 = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.identity_key_public_base64];
+ string fingerprint = fingerprint_from_base64(own_b64);
+
+ var own_action_box = new Box(Orientation.HORIZONTAL, 6);
+ var show_qrcode_button = new MenuButton() { icon_name="dino-qr-code-symbolic", valign=Align.CENTER };
+ own_action_box.append(show_qrcode_button);
+ var copy_button = new Button() { icon_name="dino-edit-copy-symbolic", valign=Align.CENTER };
+ copy_button.clicked.connect(() => { copy_button.get_clipboard().set_text(fingerprint); });
+ own_action_box.append(copy_button);
+
+ Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true };
+ action_row.title = _("This device");
+ action_row.subtitle = fingerprint_markup(fingerprint_from_base64(own_b64));
+ action_row.add_suffix(own_action_box);
+ add_key_row(action_row);
+
+ // Create and set QR code popover
+ int sid = plugin.db.identity.row_with(plugin.db.identity.account_id, account.id)[plugin.db.identity.device_id];
+ var iri_query = @"omemo-sid-$(sid)=$(fingerprint)";
+ string iri = GLib.Uri.join(UriFlags.NONE, "xmpp", null, null, 0, jid.to_string(), iri_query, null);
+
+ const int QUIET_ZONE_MODULES = 4; // MUST be at least 4
+ const int MODULE_SIZE_PX = 4; // arbitrary
+ var qr_paintable = new QRcode(iri, 2)
+ .to_paintable(MODULE_SIZE_PX * qrcode_picture.scale_factor);
+ qrcode_picture.paintable = qr_paintable;
+ qrcode_picture.margin_top = qrcode_picture.margin_end =
+ qrcode_picture.margin_bottom = qrcode_picture.margin_start = QUIET_ZONE_MODULES * MODULE_SIZE_PX;
+ qrcode_popover.add_css_class("qrcode-container");
+
+ show_qrcode_button.popover = qrcode_popover;
+ }
+
+ private void add_fingerprint(Row device, TrustLevel trust) {
+ string key_base64 = device[plugin.db.identity_meta.identity_key_public_base64];
+ bool key_active = device[plugin.db.identity_meta.now_active];
+ if (store != null) {
+ try {
+ Address address = new Address(jid.to_string(), device[plugin.db.identity_meta.device_id]);
+ SessionRecord? session = null;
+ if (store.contains_session(address)) {
+ session = store.load_session(address);
+ string session_key_base64 = Base64.encode(session.state.remote_identity_key.serialize());
+ if (key_base64 != session_key_base64) {
+ critical("Session and database identity key mismatch!");
+ key_base64 = session_key_base64;
+ }
+ }
+ } catch (Error e) {
+ print("Error while reading session store: %s", e.message);
+ }
+ }
+
+ if (device[plugin.db.identity_meta.now_active]) {
+ Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true };
+ action_row.activated.connect(() => {
+ Row updated_device = plugin.db.identity_meta.get_device(device[plugin.db.identity_meta.identity_id], device[plugin.db.identity_meta.address_name], device[plugin.db.identity_meta.device_id]);
+ ManageKeyDialog manage_dialog = new ManageKeyDialog(updated_device, plugin.db);
+ manage_dialog.set_transient_for((Gtk.Window) get_root());
+ manage_dialog.present();
+ manage_dialog.response.connect((response) => {
+ update_stored_trust(response, updated_device);
+ redraw_key_list();
+ });
+ });
+ action_row.activatable = true;
+ action_row.title = account.bare_jid.equals_bare(jid) ? _("Other device") : _("Device");
+ action_row.subtitle = fingerprint_markup(fingerprint_from_base64(key_base64));
+ string trust_str = _("Accepted");
+ switch(trust) {
+ case TrustLevel.UNTRUSTED:
+ trust_str = _("Rejected");
+ break;
+ case TrustLevel.VERIFIED:
+ trust_str = _("Verified");
+ break;
+ }
+
+ action_row.add_suffix(new Label(trust_str));
+ add_key_row(action_row);
+ }
+ displayed_ids.add(device[plugin.db.identity_meta.device_id]);
+ }
+
+ private bool on_auto_accept_toggled(bool active) {
+ plugin.trust_manager.set_blind_trust(account, jid, active);
+
+ if (active) {
+ int identity_id = plugin.db.identity.get_id(account.id);
+ if (identity_id < 0) return false;
+
+ foreach (Row device in plugin.db.identity_meta.get_new_devices(identity_id, jid.to_string())) {
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED);
+ add_fingerprint(device, TrustLevel.TRUSTED);
+ }
+ }
+ return false;
+ }
+
+ private bool on_omemo_by_default_toggled(bool active) {
+ var encryption_value = active ? Encryption.OMEMO : Encryption.NONE;
+ plugin.app.settings.set_default_encryption(account, encryption_value);
+ return false;
+ }
+
+ private void update_stored_trust(int response, Row device) {
+ switch (response) {
+ case TrustLevel.TRUSTED:
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED);
+ break;
+ case TrustLevel.UNTRUSTED:
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.UNTRUSTED);
+ break;
+ case TrustLevel.VERIFIED:
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.VERIFIED);
+ plugin.trust_manager.set_blind_trust(account, jid, false);
+ automatically_accept_new_row.active = false;
+ break;
+ }
+ }
+
+ private void add_new_fingerprint(Row device) {
+ Adw.ActionRow action_row = new Adw.ActionRow() { use_markup = true };
+ action_row.title = _("New device");
+ action_row.subtitle = fingerprint_markup(fingerprint_from_base64(device[plugin.db.identity_meta.identity_key_public_base64]));
+
+ Button accept_button = new Button() { visible = true, valign = Align.CENTER, hexpand = true };
+ accept_button.set_icon_name("dino-check-plain-symbolic"); // using .image = sets .image-button. Together with .suggested/destructive action that breaks the button Adwaita
+ accept_button.add_css_class("suggested-action");
+ accept_button.tooltip_text = _("Accept key");
+
+ Button reject_button = new Button() { visible = true, valign = Align.CENTER, hexpand = true };
+ reject_button.set_icon_name("action-unavailable-symbolic");
+ reject_button.add_css_class("destructive-action");
+ reject_button.tooltip_text = _("Reject key");
+
+ accept_button.clicked.connect(() => {
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.TRUSTED);
+ add_fingerprint(device, TrustLevel.TRUSTED);
+ remove_key_row(action_row);
+ });
+
+ reject_button.clicked.connect(() => {
+ plugin.trust_manager.set_device_trust(account, jid, device[plugin.db.identity_meta.device_id], TrustLevel.UNTRUSTED);
+ add_fingerprint(device, TrustLevel.UNTRUSTED);
+ remove_key_row(action_row);
+ });
+
+ Box control_box = new Box(Gtk.Orientation.HORIZONTAL, 0) { visible = true, hexpand = true };
+ control_box.append(accept_button);
+ control_box.append(reject_button);
+ control_box.add_css_class("linked"); // .linked: Visually link the accept / reject buttons
+
+ action_row.add_suffix(control_box);
+
+ add_key_row(action_row);
+ displayed_ids.add(device[plugin.db.identity_meta.device_id]);
+ }
+
+ private void add_key_row(Adw.PreferencesRow widget) {
+ keys_preferences_group.add(widget);
+ keys_preferences_group_children.add(widget);
+ }
+
+ private void remove_key_row(Adw.PreferencesRow widget) {
+ keys_preferences_group.remove(widget);
+ keys_preferences_group_children.remove(widget);
+ }
+}
+} \ No newline at end of file
diff --git a/main/src/plugins/omemo/ui/manage_key_dialog.vala b/main/src/plugins/omemo/ui/manage_key_dialog.vala
new file mode 100644
index 00000000..65033999
--- /dev/null
+++ b/main/src/plugins/omemo/ui/manage_key_dialog.vala
@@ -0,0 +1,178 @@
+using Gtk;
+
+using Dino.Qlite;
+
+namespace Dino.Plugins.Omemo {
+
+[GtkTemplate (ui = "/im/dino/Dino/manage_key_dialog.ui")]
+public class ManageKeyDialog : Gtk.Dialog {
+
+ [GtkChild] private unowned Stack manage_stack;
+
+ [GtkChild] private unowned Button cancel_button;
+ [GtkChild] private unowned Button ok_button;
+
+ [GtkChild] private unowned Label main_desc_label;
+ [GtkChild] private unowned ListBox main_action_list;
+
+ [GtkChild] private unowned Image confirm_image;
+ [GtkChild] private unowned Label confirm_title_label;
+ [GtkChild] private unowned Label confirm_desc_label;
+
+ [GtkChild] private unowned Label verify_label;
+ [GtkChild] private unowned Label compare_fingerprint_label;
+ [GtkChild] private unowned Button verify_yes_button;
+ [GtkChild] private unowned Button verify_no_button;
+
+ private Row device;
+ private Database db;
+
+ private bool return_to_main;
+ private int current_response;
+
+ construct {
+ // If we set the strings in the .ui file, they don't get translated
+ this.title = _("Manage Key");
+ compare_fingerprint_label.label = _("Compare the fingerprint, character by character, with the one shown on your contact's device.");
+ verify_no_button.label = _("Fingerprints differ");
+ verify_yes_button.label = _("Fingerprints match");
+ cancel_button.label = _("Cancel");
+ ok_button.label = _("Confirm");
+ }
+
+ public ManageKeyDialog(Row device, Database db) {
+ Object(use_header_bar : 1);
+
+ this.device = device;
+ this.db = db;
+
+ setup_main_screen();
+ setup_verify_screen();
+
+ cancel_button.clicked.connect(handle_cancel);
+ ok_button.clicked.connect(() => {
+ response(current_response);
+ close();
+ });
+
+ verify_yes_button.clicked.connect(() => {
+ confirm_image.set_from_icon_name("security-high-symbolic");
+ confirm_title_label.label = _("Verify key");
+ confirm_desc_label.set_markup(_("Future messages sent by %s from the device that uses this key will be highlighted accordingly in the chat window.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ manage_stack.set_visible_child_name("confirm");
+ ok_button.sensitive = true;
+ return_to_main = false;
+ current_response = TrustLevel.VERIFIED;
+ });
+
+ verify_no_button.clicked.connect(() => {
+ return_to_main = false;
+ confirm_image.set_from_icon_name("dino-dialog-warning-symbolic");
+ confirm_title_label.label = _("Fingerprints do not match");
+ confirm_desc_label.set_markup(_("Please verify that you are comparing the correct fingerprint. If fingerprints do not match, %s's account may be compromised and you should consider rejecting this key.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ manage_stack.set_visible_child_name("confirm");
+ });
+ }
+
+ private void handle_cancel() {
+ if (manage_stack.get_visible_child_name() == "main") close();
+
+ if (manage_stack.get_visible_child_name() == "verify") {
+ manage_stack.set_visible_child_name("main");
+ cancel_button.label = _("Cancel");
+ }
+
+ if (manage_stack.get_visible_child_name() == "confirm") {
+ if (return_to_main) {
+ manage_stack.set_visible_child_name("main");
+ cancel_button.label = _("Cancel");
+ } else {
+ manage_stack.set_visible_child_name("verify");
+ }
+ }
+
+ ok_button.sensitive = false;
+ }
+
+ private Box make_action_box(string title, string desc){
+ Box box = new Box(Orientation.VERTICAL, 0) { visible = true, margin_start = 20, margin_end = 20, margin_top = 14, margin_bottom = 14 };
+ Label lbl_title = new Label(title) { visible = true, halign = Align.START };
+ Label lbl_desc = new Label(desc) { visible = true, xalign = 0, wrap = true, max_width_chars = 40 };
+
+ Pango.AttrList title_attrs = new Pango.AttrList();
+ title_attrs.insert(Pango.attr_scale_new(1.1));
+ lbl_title.attributes = title_attrs;
+ Pango.AttrList desc_attrs = new Pango.AttrList();
+ desc_attrs.insert(Pango.attr_scale_new(0.8));
+ lbl_desc.attributes = desc_attrs;
+ lbl_desc.add_css_class("dim-label");
+
+ box.append(lbl_title);
+ box.append(lbl_desc);
+
+ return box;
+ }
+
+ private void setup_main_screen() {
+ main_action_list.set_header_func((row, before_row) => {
+ if (row.get_header() == null && before_row != null) {
+ row.set_header(new Separator(Orientation.HORIZONTAL));
+ }
+ });
+
+ ListBoxRow verify_row = new ListBoxRow() { visible = true };
+ verify_row.set_child(make_action_box(_("Verify key fingerprint"), _("Compare this key's fingerprint with the fingerprint displayed on the contact's device.")));
+ ListBoxRow reject_row = new ListBoxRow() { visible = true };
+ reject_row.set_child(make_action_box(_("Reject key"), _("Block encrypted communication with the contact's device that uses this key.")));
+ ListBoxRow accept_row = new ListBoxRow() {visible = true };
+ accept_row.set_child(make_action_box(_("Accept key"), _("Allow encrypted communication with the contact's device that uses this key.")));
+
+ switch((TrustLevel) device[db.identity_meta.trust_level]) {
+ case TrustLevel.TRUSTED:
+ main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#1A63D9'>"+_("accepted")+"</span>")+" "+_("This means it can be used by %s to receive and send encrypted messages.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ main_action_list.append(verify_row);
+ main_action_list.append(reject_row);
+ break;
+ case TrustLevel.VERIFIED:
+ main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#1A63D9'>"+_("verified")+"</span>")+" "+_("This means it can be used by %s to receive and send encrypted messages.").printf(@"<b>$(device[db.identity_meta.address_name])</b>") + " " + _("Additionally it has been verified to match the key on the contact's device."));
+ main_action_list.append(reject_row);
+ break;
+ case TrustLevel.UNTRUSTED:
+ main_desc_label.set_markup(_("This key is currently %s.").printf("<span color='#D91900'>"+_("rejected")+"</span>")+" "+_("This means it cannot be used by %s to decipher your messages, and you won't see messages encrypted with it.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ main_action_list.append(accept_row);
+ break;
+ }
+
+ //Row clicked - go to appropriate screen
+ main_action_list.row_activated.connect((row) => {
+ if(row == verify_row) {
+ manage_stack.set_visible_child_name("verify");
+ } else if (row == reject_row) {
+ confirm_image.set_from_icon_name("action-unavailable-symbolic");
+ confirm_title_label.label = _("Reject key");
+ confirm_desc_label.set_markup(_("You won't see encrypted messages from the device of %s that uses this key. Conversely, that device won't be able to decipher your messages anymore.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ manage_stack.set_visible_child_name("confirm");
+ ok_button.sensitive = true;
+ return_to_main = true;
+ current_response = TrustLevel.UNTRUSTED;
+ } else if (row == accept_row) {
+ confirm_image.set_from_icon_name("dino-check-plain-symbolic");
+ confirm_title_label.label = _("Accept key");
+ confirm_desc_label.set_markup(_("You will be able to exchange encrypted messages with the device of %s that uses this key.").printf(@"<b>$(device[db.identity_meta.address_name])</b>"));
+ manage_stack.set_visible_child_name("confirm");
+ ok_button.sensitive = true;
+ return_to_main = true;
+ current_response = TrustLevel.TRUSTED;
+ }
+ cancel_button.label = _("Back");
+ });
+
+ manage_stack.set_visible_child_name("main");
+ }
+
+ private void setup_verify_screen() {
+ verify_label.set_markup(fingerprint_markup(fingerprint_from_base64(device[db.identity_meta.identity_key_public_base64])));
+ }
+}
+
+}
diff --git a/main/src/plugins/omemo/ui/own_notifications.vala b/main/src/plugins/omemo/ui/own_notifications.vala
new file mode 100644
index 00000000..7ff10e30
--- /dev/null
+++ b/main/src/plugins/omemo/ui/own_notifications.vala
@@ -0,0 +1,35 @@
+using Dino.Entities;
+using Xmpp;
+using Gtk;
+
+namespace Dino.Plugins.Omemo {
+
+public class OwnNotifications {
+
+ private StreamInteractor stream_interactor;
+ private Plugin plugin;
+ private Account account;
+
+ public OwnNotifications (Plugin plugin, StreamInteractor stream_interactor, Account account) {
+ this.stream_interactor = (!)stream_interactor;
+ this.plugin = plugin;
+ this.account = account;
+ stream_interactor.module_manager.get_module(account, StreamModule.IDENTITY).bundle_fetched.connect_after((jid, device_id, bundle) => {
+ if (jid.equals(account.bare_jid) && plugin.has_new_devices(account, account.bare_jid)) {
+ display_notification();
+ }
+ });
+
+ if (plugin.has_new_devices(account, account.bare_jid)) {
+ display_notification();
+ }
+ }
+
+ private void display_notification() {
+ Notification notification = new Notification(_("OMEMO trust decision required"));
+ notification.set_default_action_and_target_value("app.own-keys", new Variant.int32(account.id));
+ notification.set_body(_("Did you add a new device for account %s?").printf(@"$(account.bare_jid.to_string())"));
+ plugin.app.send_notification(account.id.to_string()+"-new-device", notification);
+ }
+}
+}
diff --git a/main/src/plugins/omemo/ui/util.vala b/main/src/plugins/omemo/ui/util.vala
new file mode 100644
index 00000000..e250ff4d
--- /dev/null
+++ b/main/src/plugins/omemo/ui/util.vala
@@ -0,0 +1,40 @@
+using Xmpp.Util;
+
+namespace Dino.Plugins.Omemo {
+
+public static string fingerprint_from_base64(string b64) {
+ uint8[] arr = Base64.decode(b64);
+
+ arr = arr[1:arr.length];
+ string s = "";
+ foreach (uint8 i in arr) {
+ string tmp = i.to_string("%x");
+ if (tmp.length == 1) tmp = "0" + tmp;
+ s = s + tmp;
+ }
+
+ return s;
+}
+
+public static string fingerprint_markup(string s) {
+ return "<span font_family='monospace' font='9'>" + format_fingerprint(s) + "</span>";
+}
+
+public static string format_fingerprint(string s) {
+ string markup = "";
+ for (int i = 0; i < s.length; i += 4) {
+ string four_chars = s.substring(i, 4).down();
+
+ if (i % 32 == 0 && i != 0) markup += "\n";
+ markup += four_chars;
+ if (i % 16 == 12 && i % 32 != 28) {
+ markup += " ";
+ }
+ if (i % 8 == 4 && i % 16 != 12) {
+ markup += "\u00a0"; // Non-breaking space
+ }
+ }
+ return markup;
+}
+
+}
diff --git a/main/src/plugins/openpgp/contact_details_provider.vala b/main/src/plugins/openpgp/contact_details_provider.vala
new file mode 100644
index 00000000..353c0b85
--- /dev/null
+++ b/main/src/plugins/openpgp/contact_details_provider.vala
@@ -0,0 +1,51 @@
+using Gtk;
+
+using Dino.Entities;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class ContactDetailsProvider : Plugins.ContactDetailsProvider, Object {
+ public string id { get { return "pgp_info"; } }
+ public string tab { get { return "encryption"; } }
+
+ private StreamInteractor stream_interactor;
+
+ public ContactDetailsProvider(StreamInteractor stream_interactor) {
+ this.stream_interactor = stream_interactor;
+ }
+
+ public void populate(Conversation conversation, Plugins.ContactDetails contact_details, WidgetType type) { }
+
+ public Object? get_widget(Conversation conversation) {
+ var preferences_group = new Adw.PreferencesGroup() { title="OpenPGP" };
+
+ if (conversation.type_ != Conversation.Type.CHAT) return null;
+
+ string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, conversation.counterpart);
+ if (key_id == null) return null;
+
+ Gee.List<GPG.Key>? keys = null;
+ try {
+ keys = GPGHelper.get_keylist(key_id);
+ } catch (Error e) { }
+
+ var str = "";
+ if (keys != null && keys.size > 0) {
+ str = markup_id(keys[0].fpr, true);
+ } else {
+ str = _("Key not in keychain") + "\n" + markup_id(key_id, false);
+ }
+
+ var view = new Adw.ActionRow() {
+ title = _("Fingerprint"),
+ subtitle = str,
+ subtitle_selectable = true
+ };
+
+ preferences_group.add(view);
+
+ return preferences_group;
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/database.vala b/main/src/plugins/openpgp/database.vala
new file mode 100644
index 00000000..4c05571e
--- /dev/null
+++ b/main/src/plugins/openpgp/database.vala
@@ -0,0 +1,137 @@
+using Dino.Entities;
+using Dino.Qlite;
+using Xmpp;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class Database : Qlite.Database {
+ private const int VERSION = 1;
+
+ public class AccountSetting : Table {
+ public Column<int> account_id = new Column.Integer("account_id") { primary_key = true };
+ public Column<string> key = new Column.Text("key") { not_null = true };
+ public Column<string> signed_data = new Column.Text("signed_data") { min_version = 1 };
+ public Column<string> sig = new Column.Text("sig") { min_version = 1 };
+
+ internal AccountSetting(Database db) {
+ base(db, "account_setting");
+ init({account_id, key, signed_data, sig});
+ }
+ }
+
+ public class ContactKey : Table {
+ public Column<string> jid = new Column.Text("jid") { primary_key = true };
+ public Column<string> key = new Column.Text("key");
+ public Column<string> sig = new Column.Text("sig") { min_version = 1 };
+ public Column<string> signed_data = new Column.Text("signed_data") { min_version = 1 };
+
+ internal ContactKey(Database db) {
+ base(db, "contact_key");
+ init({jid, key, sig, signed_data});
+ }
+
+ internal void migrate_from_v0(Database db) {
+ string column_list = "";
+ foreach (Column c in columns) {
+ if (c.min_version <= VERSION && c.max_version >= VERSION) {
+ if (column_list == "") {
+ column_list = c.name;
+ } else {
+ column_list += ", " + c.name;
+ }
+ }
+ }
+ db.exec(@"ALTER TABLE $name RENAME TO _$(name)_1");
+ create_table_at_version(VERSION);
+ db.exec(@"INSERT INTO $name ($column_list) SELECT $column_list FROM _$(name)_1");
+ db.exec(@"DROP TABLE _$(name)_1");
+ }
+ }
+
+ public AccountSetting account_setting_table { get; private set; }
+ public ContactKey contact_key_table { get; private set; }
+
+ public Database(string filename) {
+ base(filename, VERSION);
+ this.account_setting_table = new AccountSetting(this);
+ this.contact_key_table = new ContactKey(this);
+ init({account_setting_table, contact_key_table});
+
+ try {
+ exec("PRAGMA journal_mode = WAL");
+ exec("PRAGMA synchronous = NORMAL");
+ exec("PRAGMA secure_delete = ON");
+ } catch (Error e) {
+ error("Failed to set OpenPGP database properties: %s", e.message);
+ }
+ }
+
+ public void set_contact_signature(Jid jid, string sig, string signed_data) {
+ contact_key_table.upsert()
+ .value(contact_key_table.jid, jid.to_string(), true)
+ .value(contact_key_table.sig, sig)
+ .value(contact_key_table.signed_data, signed_data)
+ .value_null(contact_key_table.key)
+ .perform();
+ }
+
+ public void clear_contact_signature(Jid jid) {
+ contact_key_table.upsert()
+ .value(contact_key_table.jid, jid.to_string(), true)
+ .value_null(contact_key_table.sig)
+ .value_null(contact_key_table.signed_data)
+ .perform();
+ }
+
+ public void set_contact_key(Jid jid, string key) {
+ contact_key_table.upsert()
+ .value(contact_key_table.jid, jid.to_string(), true)
+ .value(contact_key_table.key, key)
+ .perform();
+ }
+
+ public string? get_contact_key(Jid jid) {
+ return get_contact_key_row(jid)[contact_key_table.key];
+ }
+
+ public RowOption get_contact_key_row(Jid jid) {
+ return contact_key_table.select({contact_key_table.key})
+ .with(contact_key_table.jid, "=", jid.to_string()).row();
+ }
+
+ public void set_account_key(Account account, string key) {
+ account_setting_table.upsert()
+ .value(account_setting_table.account_id, account.id, true)
+ .value(account_setting_table.key, key)
+ .value_null(account_setting_table.signed_data)
+ .value_null(account_setting_table.sig)
+ .perform();
+ }
+
+ public void set_account_signature(Account account, string signed_data, string sig) {
+ account_setting_table.update()
+ .with(account_setting_table.account_id, "=", account.id)
+ .set(account_setting_table.signed_data, signed_data)
+ .set(account_setting_table.sig, sig)
+ .perform();
+ }
+
+ public string? get_account_signature(Account account, string signed_data) {
+ return account_setting_table.select({account_setting_table.sig})
+ .with(account_setting_table.account_id, "=", account.id)
+ .with(account_setting_table.signed_data, "=", signed_data)[account_setting_table.sig];
+ }
+
+ public string? get_account_key(Account account) {
+ return account_setting_table.select({account_setting_table.key})
+ .with(account_setting_table.account_id, "=", account.id)[account_setting_table.key];
+ }
+
+ public override void migrate(long oldVersion) {
+ if (oldVersion < 1) {
+ contact_key_table.migrate_from_v0(this);
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/encryption_list_entry.vala b/main/src/plugins/openpgp/encryption_list_entry.vala
new file mode 100644
index 00000000..cf5da8c4
--- /dev/null
+++ b/main/src/plugins/openpgp/encryption_list_entry.vala
@@ -0,0 +1,72 @@
+using Gee;
+using Gtk;
+
+using Dino.Entities;
+using Xmpp;
+
+namespace Dino.Plugins.OpenPgp {
+
+private class EncryptionListEntry : Plugins.EncryptionListEntry, Object {
+
+ private StreamInteractor stream_interactor;
+ private Database db;
+
+ public EncryptionListEntry(StreamInteractor stream_interactor, Database db) {
+ this.stream_interactor = stream_interactor;
+ this.db = db;
+ }
+
+ public Entities.Encryption encryption { get {
+ return Encryption.PGP;
+ }}
+
+ public string name { get {
+ return "OpenPGP";
+ }}
+
+ public Object? get_encryption_icon(Entities.Conversation conversation, ContentItem content_item) {
+ return null;
+ }
+
+ public string? get_encryption_icon_name(Entities.Conversation conversation, ContentItem content_item) {
+ return null;
+ }
+
+ public void encryption_activated(Entities.Conversation conversation, Plugins.SetInputFieldStatus input_status_callback) {
+ try {
+ GPGHelper.get_public_key(db.get_account_key(conversation.account) ?? "");
+ } catch (Error e) {
+ input_status_callback(new Plugins.InputFieldStatus("You didn't configure OpenPGP for this account. You can do that in the Accounts Dialog.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ return;
+ }
+
+ if (conversation.type_ == Conversation.Type.CHAT) {
+ string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, conversation.counterpart);
+ if (key_id == null) {
+ input_status_callback(new Plugins.InputFieldStatus("This contact does not support %s encryption.".printf("OpenPGP"), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ return;
+ }
+ try {
+ GPGHelper.get_keylist(key_id);
+ } catch (Error e) {
+ input_status_callback(new Plugins.InputFieldStatus("This contact's OpenPGP key is not in your keyring.", Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ }
+ } else if (conversation.type_ == Conversation.Type.GROUPCHAT) {
+ Gee.List<Jid> muc_jids = new Gee.ArrayList<Jid>();
+ Gee.List<Jid>? occupants = stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account);
+ if (occupants != null) muc_jids.add_all(occupants);
+ Gee.List<Jid>? offline_members = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account);
+ if (offline_members != null) muc_jids.add_all(offline_members);
+
+ foreach (Jid jid in muc_jids) {
+ string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, jid);
+ if (key_id == null) {
+ input_status_callback(new Plugins.InputFieldStatus("A member's OpenPGP key is not in your keyring: %s / %s.".printf(jid.to_string(), key_id), Plugins.InputFieldStatus.MessageType.ERROR, Plugins.InputFieldStatus.InputState.NO_SEND));
+ return;
+ }
+ }
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/encryption_preferences_entry.vala b/main/src/plugins/openpgp/encryption_preferences_entry.vala
new file mode 100644
index 00000000..b03f6acd
--- /dev/null
+++ b/main/src/plugins/openpgp/encryption_preferences_entry.vala
@@ -0,0 +1,82 @@
+using Adw;
+using Dino.Entities;
+using Gtk;
+
+namespace Dino.Plugins.OpenPgp {
+
+ public class PgpPreferencesEntry : Plugins.EncryptionPreferencesEntry {
+
+ private Plugin plugin;
+
+ public PgpPreferencesEntry(Plugin plugin) {
+ this.plugin = plugin;
+ }
+
+ public override Object? get_widget(Account account, WidgetType type) {
+ if (type != WidgetType.GTK4) return null;
+ StringList string_list = new StringList(null);
+ string_list.append(_("Querying GnuPG"));
+
+ Adw.PreferencesGroup preferences_group = new Adw.PreferencesGroup() { title="OpenPGP" };
+ populate_string_list.begin(account, preferences_group);
+
+ return preferences_group;
+ }
+
+ public override string id { get { return "pgp_preferences_encryption"; }}
+
+ private async void populate_string_list(Account account, Adw.PreferencesGroup preferences_group) {
+ var keys = yield get_pgp_keys();
+
+ if (keys == null) {
+ preferences_group.add(new Adw.ActionRow() { title=_("Announce key"), subtitle=_("Error in GnuPG") });
+ return;
+ }
+ if (keys.size == 0) {
+ preferences_group.add(new Adw.ActionRow() { title=_("Announce key"), subtitle= _("No keys available. Generate one!") });
+ return;
+ }
+
+ StringList string_list = new StringList(null);
+ var drop_down = new Adw.ComboRow() { title = _("Announce key") };
+ drop_down.model = string_list;
+ preferences_group.add(drop_down);
+
+ string_list.append(_("Disabled"));
+ for (int i = 0; i < keys.size; i++) {
+ string_list.append(@"$(keys[i].uids[0].uid)\n$(keys[i].fpr.substring(24, 16))");
+ if (keys[i].fpr == plugin.db.get_account_key(account)) {
+ drop_down.selected = i + 1;
+ }
+ }
+
+ drop_down.notify["selected"].connect(() => {
+ var key_id = drop_down.selected == 0 ? "" : keys[(int)drop_down.selected - 1].fpr;
+ string? signed_data = null, sig = null;
+ if (plugin.modules.has_key(account)) {
+ plugin.modules[account].set_private_key_id(key_id, ref signed_data, ref sig);
+ }
+ plugin.db.set_account_key(account, key_id);
+ if (signed_data != null && sig != null) {
+ plugin.db.set_account_signature(account, signed_data, sig);
+ }
+ });
+ }
+
+ private static async Gee.List<GPG.Key> get_pgp_keys() {
+ Gee.List<GPG.Key> keys = null;
+ SourceFunc callback = get_pgp_keys.callback;
+ new Thread<void*> (null, () => { // Querying GnuPG might take some time
+ try {
+ keys = GPGHelper.get_keylist(null, true);
+ } catch (Error e) {
+ warning(e.message);
+ }
+ Idle.add((owned)callback);
+ return null;
+ });
+ yield;
+ return keys;
+ }
+ }
+} \ No newline at end of file
diff --git a/main/src/plugins/openpgp/file_transfer/file_decryptor.vala b/main/src/plugins/openpgp/file_transfer/file_decryptor.vala
new file mode 100644
index 00000000..2deb5823
--- /dev/null
+++ b/main/src/plugins/openpgp/file_transfer/file_decryptor.vala
@@ -0,0 +1,50 @@
+using Dino.Entities;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class PgpFileDecryptor : FileDecryptor, Object {
+
+ public Encryption get_encryption() {
+ return Encryption.PGP;
+ }
+
+ public FileReceiveData prepare_get_meta_info(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) {
+ return receive_data;
+ }
+
+ public FileMeta prepare_download_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) {
+ return file_meta;
+ }
+
+ public bool can_decrypt_file(Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) {
+ return file_transfer.file_name.has_suffix("pgp") ||
+ (file_transfer.content_type != null && file_transfer.content_type.get_mime_type() == "application/pgp-encrypted");
+ }
+
+ public async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError {
+ try {
+ uint8[] buf = new uint8[256];
+ ByteArray data = new ByteArray();
+ size_t len = -1;
+ do {
+ len = yield encrypted_stream.read_async(buf);
+ data.append(buf[0:len]);
+ } while(len > 0);
+
+ GPGHelper.DecryptedData clear_data = GPGHelper.decrypt_data(data.data);
+ file_transfer.encryption = Encryption.PGP;
+ if (clear_data.filename != null && clear_data.filename != "") {
+ debug("Decrypting file %s from %s", clear_data.filename, file_transfer.file_name);
+ file_transfer.file_name = clear_data.filename;
+ } else if (file_transfer.file_name.has_suffix(".pgp")) {
+ debug("Decrypting file %s from %s", file_transfer.file_name.substring(0, file_transfer.file_name.length - 4), file_transfer.file_name);
+ file_transfer.file_name = file_transfer.file_name.substring(0, file_transfer.file_name.length - 4);
+ }
+ return new MemoryInputStream.from_data(clear_data.data, GLib.free);
+ } catch (Error e) {
+ throw new FileReceiveError.DECRYPTION_FAILED("PGP file decryption error: %s".printf(e.message));
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/file_transfer/file_encryptor.vala b/main/src/plugins/openpgp/file_transfer/file_encryptor.vala
new file mode 100644
index 00000000..66e93bd9
--- /dev/null
+++ b/main/src/plugins/openpgp/file_transfer/file_encryptor.vala
@@ -0,0 +1,45 @@
+using Dino.Entities;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class PgpFileEncryptor : Dino.FileEncryptor, Object {
+
+ StreamInteractor stream_interactor;
+
+ public PgpFileEncryptor(StreamInteractor stream_interactor) {
+ this.stream_interactor = stream_interactor;
+ }
+
+ public bool can_encrypt_file(Conversation conversation, FileTransfer file_transfer) {
+ return conversation.encryption == Encryption.PGP;
+ }
+
+ public FileMeta encrypt_file(Conversation conversation, FileTransfer file_transfer) throws FileSendError {
+ FileMeta file_meta = new FileMeta();
+
+ try {
+ GPG.Key[] keys = stream_interactor.get_module(Manager.IDENTITY).get_key_fprs(conversation);
+ uint8[] enc_content = GPGHelper.encrypt_file(file_transfer.get_file().get_path(), keys, GPG.EncryptFlags.ALWAYS_TRUST, file_transfer.file_name);
+ file_transfer.input_stream = new MemoryInputStream.from_data(enc_content, GLib.free);
+ file_transfer.encryption = Encryption.PGP;
+ file_transfer.server_file_name = Xmpp.random_uuid() + ".pgp";
+ file_meta.size = enc_content.length;
+ } catch (Error e) {
+ throw new FileSendError.ENCRYPTION_FAILED("PGP file encryption error: %s".printf(e.message));
+ }
+ debug("Encrypting file %s as %s", file_transfer.file_name, file_transfer.server_file_name);
+
+ return file_meta;
+ }
+
+ public FileSendData? preprocess_send_file(Conversation conversation, FileTransfer file_transfer, FileSendData file_send_data, FileMeta file_meta) {
+ HttpFileSendData? send_data = file_send_data as HttpFileSendData;
+ if (send_data == null) return null;
+
+ send_data.encrypt_message = false;
+
+ return file_send_data;
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/gpgme_fix.c b/main/src/plugins/openpgp/gpgme_fix.c
new file mode 100644
index 00000000..a351bcbc
--- /dev/null
+++ b/main/src/plugins/openpgp/gpgme_fix.c
@@ -0,0 +1,12 @@
+#include <src/plugins/openpgp/gpgme_fix.h>
+
+GRecMutex gpgme_global_mutex = {0};
+
+gpgme_key_t gpgme_key_ref_vapi (gpgme_key_t key) {
+ gpgme_key_ref(key);
+ return key;
+}
+gpgme_key_t gpgme_key_unref_vapi (gpgme_key_t key) {
+ gpgme_key_unref(key);
+ return key;
+}
diff --git a/main/src/plugins/openpgp/gpgme_fix.h b/main/src/plugins/openpgp/gpgme_fix.h
new file mode 100644
index 00000000..714614fc
--- /dev/null
+++ b/main/src/plugins/openpgp/gpgme_fix.h
@@ -0,0 +1,12 @@
+#ifndef GPGME_FIX
+#define GPGME_FIX 1
+
+#include <glib.h>
+#include <gpgme.h>
+
+extern GRecMutex gpgme_global_mutex;
+
+gpgme_key_t gpgme_key_ref_vapi (gpgme_key_t key);
+gpgme_key_t gpgme_key_unref_vapi (gpgme_key_t key);
+
+#endif
diff --git a/main/src/plugins/openpgp/gpgme_helper.vala b/main/src/plugins/openpgp/gpgme_helper.vala
new file mode 100644
index 00000000..18d07c06
--- /dev/null
+++ b/main/src/plugins/openpgp/gpgme_helper.vala
@@ -0,0 +1,185 @@
+using Gee;
+using GPG;
+
+namespace GPGHelper {
+
+private static bool initialized = false;
+
+public static string encrypt_armor(string plain, Key[] keys, EncryptFlags flags) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data plain_data = Data.create_from_memory(plain.data, false);
+ Context context = Context.create();
+ context.set_armor(true);
+ Data enc_data = context.op_encrypt(keys, flags, plain_data);
+ return get_string_from_data(enc_data);
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static uint8[] encrypt_file(string uri, Key[] keys, EncryptFlags flags, string file_name) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data plain_data = Data.create_from_file(uri);
+ plain_data.set_file_name(file_name);
+ Context context = Context.create();
+ context.set_armor(true);
+ Data enc_data = context.op_encrypt(keys, flags, plain_data);
+ return get_uint8_from_data(enc_data);
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static string decrypt(string encr) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data enc_data = Data.create_from_memory(encr.data, false);
+ Context context = Context.create();
+ Data dec_data = context.op_decrypt(enc_data);
+ return get_string_from_data(dec_data);
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public class DecryptedData {
+ public uint8[] data { get; set; }
+ public string filename { get; set; }
+}
+
+public static DecryptedData decrypt_data(uint8[] data) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data enc_data = Data.create_from_memory(data, false);
+ Context context = Context.create();
+ Data dec_data = context.op_decrypt(enc_data);
+ DecryptResult* dec_res = context.op_decrypt_result();
+ return new DecryptedData() { data=get_uint8_from_data(dec_data), filename=dec_res->file_name};
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static string sign(string plain, SigMode mode, Key? key = null) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data plain_data = Data.create_from_memory(plain.data, false);
+ Context context = Context.create();
+ if (key != null) context.signers_add(key);
+ Data signed_data = context.op_sign(plain_data, mode);
+ return get_string_from_data(signed_data);
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static string? get_sign_key(string signature, string? text) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Data sig_data = Data.create_from_memory(signature.data, false);
+ Data text_data;
+ if (text != null) {
+ text_data = Data.create_from_memory(text.data, false);
+ } else {
+ text_data = Data.create();
+ }
+ Context context = Context.create();
+ context.op_verify(sig_data, text_data);
+ VerifyResult* verify_res = context.op_verify_result();
+ if (verify_res == null || verify_res.signatures == null) return null;
+ return verify_res.signatures.fpr;
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static Gee.List<Key> get_keylist(string? pattern = null, bool secret_only = false) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+
+ Gee.List<Key> keys = new ArrayList<Key>();
+ Context context = Context.create();
+ context.op_keylist_start(pattern, secret_only ? 1 : 0);
+ try {
+ while (true) {
+ Key key = context.op_keylist_next();
+ keys.add(key);
+ }
+ } catch (Error e) {
+ if (e.code != GPGError.ErrorCode.EOF) throw e;
+ }
+ context.op_keylist_end();
+ return keys;
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+public static Key? get_public_key(string sig) throws GLib.Error {
+ return get_key(sig, false);
+}
+
+public static Key? get_private_key(string sig) throws GLib.Error {
+ return get_key(sig, true);
+}
+
+private static Key? get_key(string sig, bool priv) throws GLib.Error {
+ global_mutex.lock();
+ try {
+ initialize();
+ Context context = Context.create();
+ Key key = context.get_key(sig, priv);
+ return key;
+ } finally {
+ global_mutex.unlock();
+ }
+}
+
+private static string get_string_from_data(Data data) {
+ const size_t BUF_SIZE = 256;
+ data.seek(0);
+ uint8[] buf = new uint8[BUF_SIZE + 1];
+ ssize_t len = 0;
+ string res = "";
+ do {
+ len = data.read(buf, BUF_SIZE);
+ if (len > 0) {
+ buf[len] = 0;
+ res += (string) buf;
+ }
+ } while (len > 0);
+ return res;
+}
+
+private static uint8[] get_uint8_from_data(Data data) {
+ const size_t BUF_SIZE = 256;
+ data.seek(0);
+ uint8[] buf = new uint8[BUF_SIZE + 1];
+ ssize_t len = 0;
+ ByteArray res = new ByteArray();
+ do {
+ len = data.read(buf, BUF_SIZE);
+ if (len > 0) {
+ res.append(buf[0:len]);
+ }
+ } while (len > 0);
+ return res.data;
+}
+
+private static void initialize() {
+ if (!initialized) {
+ check_version();
+ initialized = true;
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/manager.vala b/main/src/plugins/openpgp/manager.vala
new file mode 100644
index 00000000..23cc93c2
--- /dev/null
+++ b/main/src/plugins/openpgp/manager.vala
@@ -0,0 +1,123 @@
+using Gee;
+using Xmpp;
+
+using Xmpp;
+using Dino.Entities;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class Manager : StreamInteractionModule, Object {
+ public static ModuleIdentity<Manager> IDENTITY = new ModuleIdentity<Manager>("pgp_manager");
+ public string id { get { return IDENTITY.id; } }
+
+ public const string MESSAGE_ENCRYPTED = "pgp";
+
+ private StreamInteractor stream_interactor;
+ private Database db;
+ private ReceivedMessageListener received_message_listener = new ReceivedMessageListener();
+
+ public static void start(StreamInteractor stream_interactor, Database db) {
+ Manager m = new Manager(stream_interactor, db);
+ stream_interactor.add_module(m);
+ }
+
+ private Manager(StreamInteractor stream_interactor, Database db) {
+ this.stream_interactor = stream_interactor;
+ this.db = db;
+
+ stream_interactor.account_added.connect(on_account_added);
+ stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(received_message_listener);
+ stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(check_encypt);
+ }
+
+ public GPG.Key[] get_key_fprs(Conversation conversation) throws Error {
+ Gee.List<string> keys = new Gee.ArrayList<string>();
+ keys.add(db.get_account_key(conversation.account));
+ if (conversation.type_ == Conversation.Type.GROUPCHAT) {
+ Gee.List<Jid> muc_jids = new Gee.ArrayList<Jid>();
+ Gee.List<Jid>? occupants = stream_interactor.get_module(MucManager.IDENTITY).get_occupants(conversation.counterpart, conversation.account);
+ if (occupants != null) muc_jids.add_all(occupants);
+ Gee.List<Jid>? offline_members = stream_interactor.get_module(MucManager.IDENTITY).get_offline_members(conversation.counterpart, conversation.account);
+ if (occupants != null) muc_jids.add_all(offline_members);
+
+ foreach (Jid jid in muc_jids) {
+ string? key_id = stream_interactor.get_module(Manager.IDENTITY).get_key_id(conversation.account, jid);
+ if (key_id != null && GPGHelper.get_keylist(key_id).size > 0 && !keys.contains(key_id)) {
+ keys.add(key_id);
+ }
+ }
+ } else {
+ string? key_id = get_key_id(conversation.account, conversation.counterpart);
+ if (key_id != null) {
+ keys.add(key_id);
+ }
+ }
+ GPG.Key[] gpgkeys = new GPG.Key[keys.size];
+ for (int i = 0; i < keys.size; i++) {
+ try {
+ GPG.Key key = GPGHelper.get_public_key(keys[i]);
+ if (key != null) gpgkeys[i] = key;
+ } catch (Error e) {}
+ }
+
+ return gpgkeys;
+ }
+
+ private void check_encypt(Entities.Message message, Xmpp.MessageStanza message_stanza, Conversation conversation) {
+ try {
+ if (message.encryption == Encryption.PGP) {
+ GPG.Key[] keys = get_key_fprs(conversation);
+ XmppStream? stream = stream_interactor.get_stream(conversation.account);
+ if (stream != null) {
+ bool encrypted = stream.get_module(Module.IDENTITY).encrypt(message_stanza, keys);
+ if (!encrypted) message.marked = Entities.Message.Marked.WONTSEND;
+ }
+ }
+ } catch (Error e) {
+ message.marked = Entities.Message.Marked.WONTSEND;
+ }
+ }
+
+ public string? get_key_id(Account account, Jid jid) {
+ Jid search_jid = stream_interactor.get_module(MucManager.IDENTITY).is_groupchat_occupant(jid, account) ? jid : jid.bare_jid;
+ Qlite.RowOption key_row = db.get_contact_key_row(search_jid);
+ if (!key_row.is_present()) return null;
+ string? key_id = key_row.get(db.contact_key_table.key);
+ if (key_id != null) return key_id;
+ string? sig = key_row.get(db.contact_key_table.sig);
+ string? signed_data = key_row.get(db.contact_key_table.signed_data);
+ if (sig == null || signed_data == null) return null;
+ key_id = get_sign_key(sig, signed_data);
+ if (key_id != null) {
+ db.set_contact_key(search_jid, key_id);
+ }
+ return key_id;
+ }
+
+ private void on_account_added(Account account) {
+ stream_interactor.module_manager.get_module(account, Module.IDENTITY).received_jid_presence_signature.connect((stream, jid, sig, signed_data) => {
+ on_jid_signature_received(account, jid, sig, signed_data);
+ });
+ }
+
+ private void on_jid_signature_received(Account account, Jid jid, string sig, string signed_data) {
+ Jid set_jid = stream_interactor.get_module(MucManager.IDENTITY).is_groupchat_occupant(jid, account) ? jid : jid.bare_jid;
+ db.set_contact_signature(set_jid, sig, signed_data);
+ }
+
+ private class ReceivedMessageListener : MessageListener {
+
+ public string[] after_actions_const = new string[]{ };
+ public override string action_group { get { return "DECRYPT"; } }
+ public override string[] after_actions { get { return after_actions_const; } }
+
+ public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) {
+ if (MessageFlag.get_flag(stanza) != null && MessageFlag.get_flag(stanza).decrypted) {
+ message.encryption = Encryption.PGP;
+ }
+ return false;
+ }
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/plugin.vala b/main/src/plugins/openpgp/plugin.vala
new file mode 100644
index 00000000..67b94455
--- /dev/null
+++ b/main/src/plugins/openpgp/plugin.vala
@@ -0,0 +1,47 @@
+using Gee;
+
+using Dino.Entities;
+
+namespace Dino.Plugins.OpenPgp {
+
+public class Plugin : Plugins.RootInterface, Object {
+ public Dino.Application app;
+ public Database db;
+ public HashMap<Account, Module> modules = new HashMap<Account, Module>(Account.hash_func, Account.equals_func);
+
+ private EncryptionListEntry list_entry;
+ private ContactDetailsProvider contact_details_provider;
+
+ public void registered(Dino.Application app) {
+ this.app = app;
+ this.db = new Database(Path.build_filename(Application.get_storage_dir(), "pgp.db"));
+ this.list_entry = new EncryptionListEntry(app.stream_interactor, db);
+ this.contact_details_provider = new ContactDetailsProvider(app.stream_interactor);
+
+ app.plugin_registry.register_encryption_list_entry(list_entry);
+ app.plugin_registry.register_encryption_preferences_entry(new PgpPreferencesEntry(this));
+ app.plugin_registry.register_contact_details_entry(contact_details_provider);
+ app.stream_interactor.module_manager.initialize_account_modules.connect(on_initialize_account_modules);
+
+ Manager.start(app.stream_interactor, db);
+ app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new PgpFileEncryptor(app.stream_interactor));
+ app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new PgpFileDecryptor());
+ JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.PGP, new JingleFileEncryptionHelperTransferOnly());
+ }
+
+ public void shutdown() { }
+
+ private void on_initialize_account_modules(Account account, ArrayList<Xmpp.XmppStreamModule> modules) {
+ string signed_data = "";
+ string? sig = db.get_account_signature(account, signed_data);
+ bool sig_was_unknown = sig == null;
+ Module module = new Module(db.get_account_key(account), ref signed_data, ref sig);
+ if (sig_was_unknown && sig != null) {
+ db.set_account_signature(account, signed_data, sig);
+ }
+ this.modules[account] = module;
+ modules.add(module);
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/stream_module.vala b/main/src/plugins/openpgp/stream_module.vala
new file mode 100644
index 00000000..9985df26
--- /dev/null
+++ b/main/src/plugins/openpgp/stream_module.vala
@@ -0,0 +1,159 @@
+using GPG;
+
+using Xmpp;
+
+namespace Dino.Plugins.OpenPgp {
+ private const string NS_URI = "jabber:x";
+ private const string NS_URI_ENCRYPTED = NS_URI + ":encrypted";
+ private const string NS_URI_SIGNED = NS_URI + ":signed";
+
+ public class Module : XmppStreamModule {
+ public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0027_current_pgp_usage");
+
+ public signal void received_jid_presence_signature(XmppStream stream, Jid jid, string sig, string signed_data);
+
+ private string? signed_status = null;
+ private Key? own_key = null;
+ private ReceivedPipelineDecryptListener received_pipeline_decrypt_listener = new ReceivedPipelineDecryptListener();
+
+ public Module(string? own_key_id = null, ref string? signed_data, ref string? sig) {
+ set_private_key_id(own_key_id, ref signed_data, ref sig);
+ }
+
+ public void set_private_key_id(string? own_key_id, ref string? signed_data, ref string? sig) {
+ if (own_key_id != null && sig == null) {
+ try {
+ own_key = GPGHelper.get_private_key(own_key_id);
+ if (own_key == null) warning("Can't get PGP private key");
+ } catch (Error e) { }
+ if (own_key != null) {
+ if (signed_data == null) signed_data = "";
+ sig = gpg_sign(signed_data, own_key);
+ }
+ }
+ signed_status = own_key_id != null ? sig : null;
+ }
+
+ public bool encrypt(MessageStanza message, GPG.Key[] keys) {
+ string? enc_body = gpg_encrypt(message.body, keys);
+ if (enc_body != null) {
+ message.stanza.put_node(new StanzaNode.build("x", NS_URI_ENCRYPTED).add_self_xmlns().put_node(new StanzaNode.text(enc_body)));
+ message.body = "[This message is OpenPGP encrypted (see XEP-0027)]";
+ Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI_ENCRYPTED);
+ return true;
+ }
+ return false;
+ }
+
+ public override void attach(XmppStream stream) {
+ stream.get_module(Presence.Module.IDENTITY).received_presence.connect(on_received_presence);
+ stream.get_module(Presence.Module.IDENTITY).pre_send_presence_stanza.connect(on_pre_send_presence_stanza);
+ stream.get_module(MessageModule.IDENTITY).received_pipeline.connect(received_pipeline_decrypt_listener);
+ }
+
+ public override void detach(XmppStream stream) {
+ stream.get_module(Presence.Module.IDENTITY).received_presence.disconnect(on_received_presence);
+ stream.get_module(Presence.Module.IDENTITY).pre_send_presence_stanza.disconnect(on_pre_send_presence_stanza);
+ stream.get_module(MessageModule.IDENTITY).received_pipeline.disconnect(received_pipeline_decrypt_listener);
+ }
+
+ public override string get_ns() { return NS_URI; }
+ public override string get_id() { return IDENTITY.id; }
+
+ private void on_received_presence(XmppStream stream, Presence.Stanza presence) {
+ StanzaNode x_node = presence.stanza.get_subnode("x", NS_URI_SIGNED);
+ if (x_node == null) return;
+ string? sig = x_node.get_string_content();
+ if (sig == null) return;
+ string signed_data = presence.status == null ? "" : presence.status;
+ received_jid_presence_signature(stream, presence.from, sig, signed_data);
+ }
+
+ private void on_pre_send_presence_stanza(XmppStream stream, Presence.Stanza presence) {
+ if (presence.type_ == Presence.Stanza.TYPE_AVAILABLE && signed_status != null) {
+ presence.stanza.put_node(new StanzaNode.build("x", NS_URI_SIGNED).add_self_xmlns().put_node(new StanzaNode.text(signed_status)));
+ }
+ }
+
+ private static string? gpg_encrypt(string plain, GPG.Key[] keys) {
+ string encr;
+ try {
+ encr = GPGHelper.encrypt_armor(plain, keys, GPG.EncryptFlags.ALWAYS_TRUST);
+ } catch (Error e) {
+ return null;
+ }
+ int encryption_start = encr.index_of("\n\n") + 2;
+ return encr.substring(encryption_start, encr.length - "\n-----END PGP MESSAGE-----".length - encryption_start);
+ }
+
+ private static string? gpg_sign(string str, Key key) {
+ string signed;
+ try {
+ signed = GPGHelper.sign(str, GPG.SigMode.CLEAR, key);
+ } catch (Error e) {
+ return null;
+ }
+ int signature_start = signed.index_of("-----BEGIN PGP SIGNATURE-----");
+ signature_start = signed.index_of("\n\n", signature_start) + 2;
+ return signed.substring(signature_start, signed.length - "\n-----END PGP SIGNATURE-----".length - signature_start);
+ }
+ }
+
+ public class MessageFlag : Xmpp.MessageFlag {
+ public const string id = "pgp";
+
+ public bool decrypted = false;
+
+ public static MessageFlag? get_flag(MessageStanza message) {
+ return (MessageFlag) message.get_flag(NS_URI, id);
+ }
+
+ public override string get_ns() { return NS_URI; }
+ public override string get_id() { return id; }
+ }
+
+public class ReceivedPipelineDecryptListener : StanzaListener<MessageStanza> {
+
+ private string[] after_actions_const = {"MODIFY_BODY"};
+
+ public override string action_group { get { return "ENCRYPT_BODY"; } }
+ public override string[] after_actions { get { return after_actions_const; } }
+
+ public override async bool run(XmppStream stream, MessageStanza message) {
+ string? encrypted = get_cyphertext(message);
+ if (encrypted != null) {
+ MessageFlag flag = new MessageFlag();
+ message.add_flag(flag);
+ string? decrypted = yield gpg_decrypt(encrypted);
+ if (decrypted != null) {
+ flag.decrypted = true;
+ message.body = decrypted;
+ }
+ }
+ return false;
+ }
+
+ private static async string? gpg_decrypt(string enc) {
+ SourceFunc callback = gpg_decrypt.callback;
+ string? res = null;
+ new Thread<void*> (null, () => {
+ string armor = "-----BEGIN PGP MESSAGE-----\n\n" + enc + "\n-----END PGP MESSAGE-----";
+ try {
+ res = GPGHelper.decrypt(armor);
+ } catch (Error e) {
+ res = null;
+ }
+ Idle.add((owned) callback);
+ return null;
+ });
+ yield;
+ return res;
+ }
+
+ private string? get_cyphertext(MessageStanza message) {
+ StanzaNode? x_node = message.stanza.get_subnode("x", NS_URI_ENCRYPTED);
+ return x_node == null ? null : x_node.get_string_content();
+ }
+}
+
+}
diff --git a/main/src/plugins/openpgp/util.vala b/main/src/plugins/openpgp/util.vala
new file mode 100644
index 00000000..3c7a7594
--- /dev/null
+++ b/main/src/plugins/openpgp/util.vala
@@ -0,0 +1,29 @@
+using Gtk;
+
+using Dino.Entities;
+using Xmpp.Util;
+
+namespace Dino.Plugins.OpenPgp {
+
+public static string markup_id(string s, bool is_fingerprint) {
+ string markup = is_fingerprint ? "" : "0x";
+ for (int i = 0; i < s.length; i += 4) {
+ string four_chars = s.substring(i, 4).down();
+
+ if (i == 4 * 5) markup += "\n";
+ markup += four_chars;
+ if (is_fingerprint) markup += " ";
+ }
+ return "<span font_family='monospace' font='9'>" + markup + "</span>";
+}
+
+private static string? get_sign_key(string sig, string signed_text) {
+ string armor = "-----BEGIN PGP MESSAGE-----\n\n" + sig + "\n-----END PGP MESSAGE-----";
+ string? sign_key = null;
+ try {
+ sign_key = GPGHelper.get_sign_key(armor, signed_text);
+ } catch (Error e) { }
+ return sign_key;
+}
+
+}
diff --git a/main/src/plugins/rtp/codec_util.vala b/main/src/plugins/rtp/codec_util.vala
new file mode 100644
index 00000000..9f21f5a6
--- /dev/null
+++ b/main/src/plugins/rtp/codec_util.vala
@@ -0,0 +1,418 @@
+using Gee;
+using Xmpp;
+using Xmpp.Xep;
+
+public class Dino.Plugins.Rtp.CodecUtil {
+ private Set<string> supported_elements = new HashSet<string>();
+ private Set<string> unsupported_elements = new HashSet<string>();
+
+ public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) {
+ Gst.Caps caps = new Gst.Caps.simple("application/x-rtp",
+ "media", typeof(string), media,
+ "payload", typeof(int), payload_type.id);
+ //"channels", typeof(int), payloadType.channels,
+ //"max-ptime", typeof(int), payloadType.maxptime);
+ unowned Gst.Structure s = caps.get_structure(0);
+ if (payload_type.clockrate != 0) {
+ s.set("clock-rate", typeof(int), payload_type.clockrate);
+ }
+ if (payload_type.name != null) {
+ s.set("encoding-name", typeof(string), payload_type.name.up());
+ }
+ if (incoming) {
+ foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) {
+ if (rtcp_fb.subtype == null) {
+ s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true);
+ } else {
+ s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true);
+ }
+ }
+ }
+ return caps;
+ }
+
+ public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) {
+ if (payload_type.name != null) return payload_type.name.down();
+ if (media == "audio") {
+ switch (payload_type.id) {
+ case 0:
+ return "pcmu";
+ case 8:
+ return "pcma";
+ }
+ }
+ return null;
+ }
+
+ public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) {
+ return get_media_type(media, get_codec_from_payload(media, payload_type));
+ }
+
+ public static string? get_media_type(string media, string? codec) {
+ if (codec == null) return null;
+ if (media == "audio") {
+ switch (codec) {
+ case "pcma":
+ return "audio/x-alaw";
+ case "pcmu":
+ return "audio/x-mulaw";
+ }
+ }
+ return @"$media/x-$codec";
+ }
+
+ public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) {
+ return get_pay_candidate(media, get_codec_from_payload(media, payload_type));
+ }
+
+ public static string? get_pay_candidate(string media, string? codec) {
+ if (codec == null) return null;
+ return @"rtp$(codec)pay";
+ }
+
+ public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) {
+ return get_depay_candidate(media, get_codec_from_payload(media, payload_type));
+ }
+
+ public static string? get_depay_candidate(string media, string? codec) {
+ if (codec == null) return null;
+ return @"rtp$(codec)depay";
+ }
+
+ public static string[] get_encode_candidates(string media, string? codec) {
+ if (codec == null) return new string[0];
+ if (media == "audio") {
+ switch (codec) {
+ case "opus":
+ return new string[] {"opusenc"};
+ case "speex":
+ return new string[] {"speexenc"};
+ case "pcma":
+ return new string[] {"alawenc"};
+ case "pcmu":
+ return new string[] {"mulawenc"};
+ case "g722":
+ return new string[] {"avenc_g722"};
+ }
+ } else if (media == "video") {
+ switch (codec) {
+ case "h264":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkh264enc",
+#endif
+#if ENABLE_VAAPI
+ "vaapih264enc",
+#endif
+ "x264enc"
+ };
+ case "vp9":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkvp9enc",
+#endif
+#if ENABLE_VAAPI
+ "vaapivp9enc",
+#endif
+ "vp9enc"
+ };
+ case "vp8":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkvp8enc",
+#endif
+#if ENABLE_VAAPI
+ "vaapivp8enc",
+#endif
+ "vp8enc"
+ };
+ }
+ }
+ return new string[0];
+ }
+
+ public static string[] get_decode_candidates(string media, string? codec) {
+ if (codec == null) return new string[0];
+ if (media == "audio") {
+ switch (codec) {
+ case "opus":
+ return new string[] {"opusdec"};
+ case "speex":
+ return new string[] {"speexdec"};
+ case "pcma":
+ return new string[] {"alawdec"};
+ case "pcmu":
+ return new string[] {"mulawdec"};
+ case "g722":
+ return new string[] {"avdec_g722"};
+ }
+ } else if (media == "video") {
+ switch (codec) {
+ case "h264":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkh264dec",
+#endif
+#if ENABLE_VAAPI
+ "vaapih264dec",
+#endif
+ null
+ };
+ case "vp9":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkvp9dec",
+#endif
+#if ENABLE_VAAPI
+ "vaapivp9dec",
+#endif
+ "vp9dec"
+ };
+ case "vp8":
+ return new string[] {
+#if ENABLE_MSDK
+ "msdkvp8dec",
+#endif
+#if ENABLE_VAAPI
+ "vaapivp8dec",
+#endif
+ "vp8dec"
+ };
+ }
+ }
+ return new string[0];
+ }
+
+ public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
+ if (encode == "msdkh264enc") return "capsfilter caps=video/x-raw,format=NV12 ! ";
+ if (encode == "vaapih264enc") return "capsfilter caps=video/x-raw,format=NV12 ! ";
+ return null;
+ }
+
+ public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
+ // H264
+ if (encode == "msdkh264enc") return @" rate-control=vbr";
+ if (encode == "vaapih264enc") return @" rate-control=vbr";
+ if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency bframes=0 cabac=false dct8x8=false";
+
+ // VP8
+ if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90";
+ if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4";
+
+ // VP9
+ if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90";
+ if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4";
+
+ // OPUS
+ if (encode == "opusenc") {
+ if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true";
+ return " audio-type=voice";
+ }
+
+ return null;
+ }
+
+ public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
+ // H264
+ if (media == "video" && codec == "h264") return " ! capsfilter caps=video/x-h264,profile=constrained-baseline ! h264parse";
+ if (media == "video" && codec == "vp8" && encode == "vp8enc") return " ! capsfilter caps=video/x-vp8,profile=(string)1";
+ return null;
+ }
+
+ public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) {
+ Gst.Bin? encode_bin = encode_element as Gst.Bin;
+ if (encode_bin == null) return 0;
+ string? codec = get_codec_from_payload(media, payload_type);
+ string? encode_name = get_encode_element_name(media, codec);
+ if (encode_name == null) return 0;
+ Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode");
+
+ switch (encode_name) {
+ case "msdkh264enc":
+ case "vaapih264enc":
+ case "x264enc":
+ case "msdkvp9enc":
+ case "vaapivp9enc":
+ case "msdkvp8enc":
+ case "vaapivp8enc":
+ bitrate = uint.min(2048000, bitrate);
+ encode.set("bitrate", bitrate);
+ return bitrate;
+ case "vp9enc":
+ case "vp8enc":
+ bitrate = uint.min(2147483, bitrate);
+ encode.set("target-bitrate", bitrate * 1024);
+ return bitrate;
+ }
+
+ return 0;
+ }
+
+ public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) {
+ Gst.Bin? encode_bin = encode_element as Gst.Bin;
+ if (encode_bin == null) return;
+ Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+ rescale_caps.set("caps", caps);
+ }
+
+ public Gst.Caps? get_rescale_caps(Gst.Element encode_element) {
+ Gst.Bin? encode_bin = encode_element as Gst.Bin;
+ if (encode_bin == null) return null;
+ Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+ Gst.Caps caps;
+ rescale_caps.get("caps", out caps);
+ return caps;
+ }
+
+ public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
+ return null;
+ }
+
+ public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
+ if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true";
+ if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100";
+ if (decode == "vp8dec" || decode == "vp9dec") return " threads=8";
+ return null;
+ }
+
+ public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
+ return null;
+ }
+
+ public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
+ if (codec == "vp8") return " wait-for-keyframe=true";
+ return null;
+ }
+
+ public bool is_element_supported(string? element_name) {
+ if (element_name == null) return false;
+ if (unsupported_elements.contains(element_name)) return false;
+ if (supported_elements.contains(element_name)) return true;
+ var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name");
+ if (test_element != null) {
+ supported_elements.add(element_name);
+ return true;
+ } else {
+ warning("%s is not installed or supported on this system", element_name);
+ unsupported_elements.add(element_name);
+ return false;
+ }
+ }
+
+ public string? get_encode_element_name(string media, string? codec) {
+ if (get_pay_element_name(media, codec) == null) return null;
+ foreach (string candidate in get_encode_candidates(media, codec)) {
+ if (is_element_supported(candidate)) return candidate;
+ }
+ return null;
+ }
+
+ public string? get_pay_element_name(string media, string? codec) {
+ string? candidate = get_pay_candidate(media, codec);
+ if (candidate != null && is_element_supported(candidate)) return candidate;
+ return null;
+ }
+
+ public string? get_decode_element_name(string media, string? codec) {
+ if (get_depay_element_name(media, codec) == null) return null;
+ foreach (string candidate in get_decode_candidates(media, codec)) {
+ if (is_element_supported(candidate)) return candidate;
+ }
+ return null;
+ }
+
+ public string? get_depay_element_name(string media, string? codec) {
+ string? candidate = get_depay_candidate(media, codec);
+ if (candidate != null && is_element_supported(candidate)) return candidate;
+ return null;
+ }
+
+ public void mark_element_unsupported(string element_name) {
+ unsupported_elements.add(element_name);
+ }
+
+ public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) {
+ if (codec == null) return null;
+ string base_name = name ?? @"encode-$codec-$(Random.next_int())";
+ string? depay = get_depay_element_name(media, codec);
+ string? decode = element_name ?? get_decode_element_name(media, codec);
+ if (depay == null || decode == null) return null;
+ string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? "";
+ string decode_args = get_decode_args(media, codec, decode, payload_type) ?? "";
+ string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? "";
+ string depay_args = get_depay_args(media, codec, decode, payload_type) ?? "";
+ string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : "";
+ return @"queue ! $depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample";
+ }
+
+ public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
+ string? codec = get_codec_from_payload(media, payload_type);
+ string base_name = name ?? @"decode_$(codec)_$(Random.next_int())";
+ string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name);
+ if (desc == null) return null;
+ debug("Pipeline to decode %s %s: %s", media, codec, desc);
+ Gst.Element bin = Gst.parse_bin_from_description(desc, true);
+ bin.name = name;
+ return bin;
+ }
+
+ public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) {
+ string? desc1 = get_encode_bin_without_payloader_description(media, codec, payload_type, element_name, name);
+ string? desc2 = get_payloader_bin_description(media, codec, payload_type, name);
+ return @"$desc1 ! $desc2";
+ }
+
+ public string? get_payloader_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? name = null) {
+ if (codec == null) return null;
+ string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
+ string? pay = get_pay_element_name(media, codec);
+ if (pay == null) return null;
+ return @"$pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay";
+ }
+
+ public string? get_encode_bin_without_payloader_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) {
+ if (codec == null) return null;
+ string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
+ string? encode = element_name ?? get_encode_element_name(media, codec);
+ if (encode == null) return null;
+ string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? "";
+ string encode_args = get_encode_args(media, codec, encode, payload_type) ?? "";
+ string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? "";
+ string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps";
+ return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix";
+ }
+
+ public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
+ string? codec = get_codec_from_payload(media, payload_type);
+ string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
+ string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name);
+ if (desc == null) return null;
+ debug("Pipeline to encode %s %s: %s", media, codec, desc);
+ Gst.Element bin = Gst.parse_bin_from_description(desc, true);
+ bin.name = name;
+ return bin;
+ }
+
+ public Gst.Element? get_encode_bin_without_payloader(string media, JingleRtp.PayloadType payload_type, string? name = null) {
+ string? codec = get_codec_from_payload(media, payload_type);
+ string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
+ string? desc = get_encode_bin_without_payloader_description(media, codec, payload_type, null, base_name);
+ if (desc == null) return null;
+ debug("Pipeline to encode %s %s without payloader: %s", media, codec, desc);
+ Gst.Element bin = Gst.parse_bin_from_description(desc, true);
+ bin.name = name;
+ return bin;
+ }
+
+ public Gst.Element? get_payloader_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
+ string? codec = get_codec_from_payload(media, payload_type);
+ string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
+ string? desc = get_payloader_bin_description(media, codec, payload_type, base_name);
+ if (desc == null) return null;
+ debug("Pipeline to payload %s %s: %s", media, codec, desc);
+ Gst.Element bin = Gst.parse_bin_from_description(desc, true);
+ bin.name = name;
+ return bin;
+ }
+
+}
diff --git a/main/src/plugins/rtp/device.vala b/main/src/plugins/rtp/device.vala
new file mode 100644
index 00000000..bdb597b6
--- /dev/null
+++ b/main/src/plugins/rtp/device.vala
@@ -0,0 +1,592 @@
+using Xmpp.Xep.JingleRtp;
+using Gee;
+
+public enum Dino.Plugins.Rtp.DeviceProtocol {
+ OTHER,
+ PIPEWIRE,
+ V4L2,
+ PULSEAUDIO,
+ ALSA
+}
+
+public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
+ private const int[] common_widths = {320, 352, 416, 480, 640, 960, 1280, 1920, 2560, 3840};
+
+ public Plugin plugin { get; private set; }
+ public CodecUtil codec_util { get { return plugin.codec_util; } }
+ public Gst.Device device { get; private set; }
+
+ public string id { owned get { return device_name; }}
+ public string display_name { owned get { return device_display_name; }}
+ public string? detail_name { owned get {
+ if (device.properties == null) return null;
+ if (device.properties.has_field("alsa.card_name")) return device.properties.get_string("alsa.card_name");
+ if (device.properties.has_field("alsa.name")) return device.properties.get_string("alsa.name");
+ if (device.properties.has_field("alsa.id")) return device.properties.get_string("alsa.id");
+ if (device.properties.has_field("api.v4l2.cap.card")) return device.properties.get_string("api.v4l2.cap.card");
+ return null;
+ }}
+ public string? media { owned get {
+ if (device.has_classes("Audio")) {
+ return "audio";
+ } else if (device.has_classes("Video")) {
+ return "video";
+ } else {
+ return null;
+ }
+ }}
+ public bool incoming { get { return is_sink; } }
+
+ public Gst.Pipeline pipe { get { return plugin.pipe; }}
+ public bool is_source { get { return device.has_classes("Source"); }}
+ public bool is_sink { get { return device.has_classes("Sink"); }}
+ public bool is_monitor { get { return (device.properties != null && device.properties.get_string("device.class") == "monitor") || (protocol == DeviceProtocol.PIPEWIRE && device.has_classes("Stream")); } }
+ public bool is_default { get {
+ if (device.properties == null) return false;
+ bool ret;
+ device.properties.get_boolean("is-default", out ret);
+ return ret;
+ }}
+ public DeviceProtocol protocol { get {
+ if (device.properties == null) return DeviceProtocol.OTHER;
+ if (device.properties.has_name("pulse-proplist")) return DeviceProtocol.PULSEAUDIO;
+ if (device.properties.has_name("pipewire-proplist")) return DeviceProtocol.PIPEWIRE;
+ if (device.properties.has_name("v4l2deviceprovider")) return DeviceProtocol.V4L2;
+ return DeviceProtocol.OTHER;
+ }}
+
+ private string device_name;
+ private string device_display_name;
+
+ private Gst.Caps device_caps;
+ private Gst.Element element;
+ private Gst.Element tee;
+ private Gst.Element dsp;
+ private Gst.Base.Aggregator mixer;
+ private Gst.Element filter;
+ private int links;
+
+ // Codecs
+ private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
+ private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
+
+ // Payloaders
+ private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
+ private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
+ private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func);
+
+ // Bitrate
+ private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func);
+
+ private class CodecBitrate {
+ public uint bitrate;
+ public int64 timestamp;
+
+ public CodecBitrate(uint bitrate) {
+ this.bitrate = bitrate;
+ this.timestamp = get_monotonic_time();
+ }
+ }
+
+ public Device(Plugin plugin, Gst.Device device) {
+ this.plugin = plugin;
+ update(device);
+ }
+
+ public bool matches(Gst.Device device) {
+ if (this.device.name == device.name) return true;
+ return false;
+ }
+
+ public void update(Gst.Device device) {
+ this.device = device;
+ this.device_name = device.name;
+ this.device_display_name = device.display_name;
+ }
+
+ public Gst.Element? link_sink() {
+ if (!is_sink) return null;
+ if (element == null) create();
+ links++;
+ if (mixer != null) {
+ Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())");
+ pipe.add(rate);
+ rate.link(mixer);
+ return rate;
+ }
+ if (media == "audio") return filter;
+ return element;
+ }
+
+ public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) {
+ if (!is_source) return null;
+ if (element == null) create();
+ links++;
+ if (payload_type != null && ssrc != 0 && tee != null) {
+ bool new_codec = false;
+ string? codec = CodecUtil.get_codec_from_payload(media, payload_type);
+ if (!codecs.has_key(payload_type)) {
+ codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder");
+ pipe.add(codecs[payload_type]);
+ new_codec = true;
+ }
+ if (!codec_tees.has_key(payload_type)) {
+ codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee");
+ codec_tees[payload_type].@set("allow-not-linked", true);
+ pipe.add(codec_tees[payload_type]);
+ codecs[payload_type].link(codec_tees[payload_type]);
+ }
+ if (!payloaders.has_key(payload_type)) {
+ payloaders[payload_type] = new HashMap<uint, Gst.Element>();
+ }
+ if (!payloaders[payload_type].has_key(ssrc)) {
+ payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)");
+ var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay");
+ payload.ssrc = ssrc;
+ payload.seqnum_offset = seqnum_offset;
+ if (timestamp_offset != 0) {
+ payload.timestamp_offset = timestamp_offset;
+ }
+ pipe.add(payloaders[payload_type][ssrc]);
+ codec_tees[payload_type].link(payloaders[payload_type][ssrc]);
+ debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset);
+ }
+ if (!payloader_tees.has_key(payload_type)) {
+ payloader_tees[payload_type] = new HashMap<uint, Gst.Element>();
+ }
+ if (!payloader_tees[payload_type].has_key(ssrc)) {
+ payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee");
+ payloader_tees[payload_type][ssrc].@set("allow-not-linked", true);
+ pipe.add(payloader_tees[payload_type][ssrc]);
+ payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]);
+ }
+ if (!payloader_links.has_key(payload_type)) {
+ payloader_links[payload_type] = new HashMap<uint, uint>();
+ }
+ if (!payloader_links[payload_type].has_key(ssrc)) {
+ payloader_links[payload_type][ssrc] = 1;
+ } else {
+ payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1;
+ }
+ if (new_codec) {
+ tee.link(codecs[payload_type]);
+ }
+ return payloader_tees[payload_type][ssrc];
+ }
+ if (tee != null) return tee;
+ return element;
+ }
+
+ private static double get_target_bitrate(Gst.Caps caps) {
+ if (caps == null || caps.get_size() == 0) return uint.MAX;
+ unowned Gst.Structure? that = caps.get_structure(0);
+ int num = 0, den = 0, width = 0, height = 0;
+ if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX;
+ if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX;
+ if (!that.has_field("framerate")) return uint.MAX;
+ Value framerate = that.get_value("framerate");
+ if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX;
+ num = Gst.Value.get_fraction_numerator(framerate);
+ den = Gst.Value.get_fraction_denominator(framerate);
+ double pxs = ((double)num/(double)den) * (double)width * (double)height;
+ double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0;
+ if (br < 128.0) return 128.0;
+ return br;
+ }
+
+ private Gst.Caps get_active_caps(PayloadType payload_type) {
+ return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps;
+ }
+
+ private void apply_caps(PayloadType payload_type, Gst.Caps caps) {
+ plugin.pause();
+ debug("Set scaled caps to %s", caps.to_string());
+ codec_util.update_rescale_caps(codecs[payload_type], caps);
+ plugin.unpause();
+ }
+
+ private void apply_width(PayloadType payload_type, int new_width, uint bitrate) {
+ int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den;
+ device_caps.get_structure(0).get_int("width", out device_caps_width);
+ device_caps.get_structure(0).get_int("height", out device_caps_height);
+ device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den);
+ Gst.Caps active_caps = get_active_caps(payload_type);
+ if (active_caps != null && active_caps.get_size() > 0) {
+ active_caps.get_structure(0).get_int("width", out active_caps_width);
+ } else {
+ active_caps_width = device_caps_width;
+ }
+ if (new_width == active_caps_width) return;
+ int new_height = device_caps_height * new_width / device_caps_width;
+ if ((new_height % 2) != 0) {
+ new_height += 1;
+ }
+ Gst.Caps new_caps;
+ if (device_caps_framerate_den != 0) {
+ new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
+ } else {
+ new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null);
+ }
+ double required_bitrate = get_target_bitrate(new_caps);
+ debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate);
+ if (bitrate < required_bitrate && new_width > active_caps_width) return;
+ apply_caps(payload_type, new_caps);
+ }
+
+ public void update_bitrate(PayloadType payload_type, uint bitrate) {
+ if (codecs.has_key(payload_type)) {
+ lock(codec_bitrates);
+ if (!codec_bitrates.has_key(payload_type)) {
+ codec_bitrates[payload_type] = new ArrayList<CodecBitrate>();
+ }
+ codec_bitrates[payload_type].add(new CodecBitrate(bitrate));
+ var remove = new ArrayList<CodecBitrate>();
+ foreach (CodecBitrate rate in codec_bitrates[payload_type]) {
+ if (rate.timestamp < get_monotonic_time() - 5000000L) {
+ remove.add(rate);
+ continue;
+ }
+ if (rate.bitrate < bitrate) {
+ bitrate = rate.bitrate;
+ }
+ }
+ codec_bitrates[payload_type].remove_all(remove);
+ if (media == "video") {
+ if (bitrate < 128) bitrate = 128;
+ Gst.Caps active_caps = get_active_caps(payload_type);
+ double max_bitrate = get_target_bitrate(device_caps) * 2;
+ double current_target_bitrate = get_target_bitrate(active_caps);
+ int device_caps_width, active_caps_width;
+ device_caps.get_structure(0).get_int("width", out device_caps_width);
+ if (active_caps != null && active_caps.get_size() > 0) {
+ active_caps.get_structure(0).get_int("width", out active_caps_width);
+ } else {
+ active_caps_width = device_caps_width;
+ }
+ if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) {
+ // Lower video resolution
+ int i = 1;
+ for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) {
+ debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
+ }
+ apply_width(payload_type, common_widths[i-1], bitrate);
+ } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) {
+ // Higher video resolution
+ int i = 0;
+ for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++);
+ if (common_widths[i] != active_caps_width) {
+ debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
+ }
+ if (common_widths[i] > device_caps_width) {
+ // We never scale up, so just stick with what the device gives
+ apply_width(payload_type, device_caps_width, bitrate);
+ } else if (common_widths[i] != active_caps_width) {
+ apply_width(payload_type, common_widths[i], bitrate);
+ }
+ }
+ if (bitrate > max_bitrate) bitrate = (uint) max_bitrate;
+ }
+ codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate);
+ unlock(codec_bitrates);
+ }
+ }
+
+ public void unlink(Gst.Element? link = null) {
+ if (links <= 0) {
+ critical("Link count below zero.");
+ return;
+ }
+ if (link != null && is_source && tee != null) {
+ PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key;
+ uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key;
+ payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1;
+ if (payloader_links[payload_type][ssrc] == 0) {
+ plugin.pause();
+
+ codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]);
+ payloaders[payload_type][ssrc].set_locked_state(true);
+ payloaders[payload_type][ssrc].set_state(Gst.State.NULL);
+ payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]);
+ pipe.remove(payloaders[payload_type][ssrc]);
+ payloaders[payload_type].unset(ssrc);
+ payloader_tees[payload_type][ssrc].set_locked_state(true);
+ payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL);
+ pipe.remove(payloader_tees[payload_type][ssrc]);
+ payloader_tees[payload_type].unset(ssrc);
+
+ payloader_links[payload_type].unset(ssrc);
+ plugin.unpause();
+ }
+ if (payloader_links[payload_type].size == 0) {
+ plugin.pause();
+
+ tee.unlink(codecs[payload_type]);
+ codecs[payload_type].set_locked_state(true);
+ codecs[payload_type].set_state(Gst.State.NULL);
+ codecs[payload_type].unlink(codec_tees[payload_type]);
+ pipe.remove(codecs[payload_type]);
+ codecs.unset(payload_type);
+ codec_tees[payload_type].set_locked_state(true);
+ codec_tees[payload_type].set_state(Gst.State.NULL);
+ pipe.remove(codec_tees[payload_type]);
+ codec_tees.unset(payload_type);
+
+ payloaders.unset(payload_type);
+ payloader_tees.unset(payload_type);
+ payloader_links.unset(payload_type);
+ plugin.unpause();
+ }
+ }
+ if (link != null && is_sink && mixer != null) {
+ plugin.pause();
+ link.set_locked_state(true);
+ Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer();
+ link.get_static_pad("src").unlink(mixer_sink_pad);
+ mixer_sink_pad.set_active(false);
+ link.set_state(Gst.State.NULL);
+ pipe.remove(link);
+ mixer.release_request_pad(mixer_sink_pad);
+ plugin.unpause();
+ }
+ links--;
+ if (links == 0) {
+ destroy();
+ }
+ }
+
+ private Gst.Caps get_best_caps() {
+ if (media == "audio") {
+ return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1");
+ } else if (media == "video" && device.caps.get_size() > 0) {
+ int best_index = -1;
+ Value? best_fraction = null;
+ int best_fps = 0;
+ int best_width = 0;
+ int best_height = 0;
+ for (int i = 0; i < device.caps.get_size(); i++) {
+ unowned Gst.Structure? that = device.caps.get_structure(i);
+ unowned Gst.CapsFeatures? features = device.caps.get_features(i);
+ Value? best_fraction_now = null;
+ if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue;
+ int num = 0, den = 0, width = 0, height = 0;
+ if (!that.has_field("framerate")) continue;
+ Value framerate = that.get_value("framerate");
+ if (framerate.type() == typeof(Gst.Fraction)) {
+ num = Gst.Value.get_fraction_numerator(framerate);
+ den = Gst.Value.get_fraction_denominator(framerate);
+ } else if (framerate.type() == typeof(Gst.ValueList)) {
+ for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) {
+ Value fraction = Gst.ValueList.get_value(framerate, j);
+ int in_num = Gst.Value.get_fraction_numerator(fraction);
+ int in_den = Gst.Value.get_fraction_denominator(fraction);
+ int fps = den > 0 ? (num/den) : 0;
+ int in_fps = in_den > 0 ? (in_num/in_den) : 0;
+ if (in_fps > fps) {
+ best_fraction_now = fraction;
+ num = in_num;
+ den = in_den;
+ }
+ }
+ } else {
+ debug("Unknown type for framerate: %s", framerate.type_name());
+ }
+ if (den == 0) continue;
+ if (!that.has_field("width") || !that.get_int("width", out width)) continue;
+ if (!that.has_field("height") || !that.get_int("height", out height)) continue;
+ int fps = num/den;
+ if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) {
+ best_fps = fps;
+ best_width = width;
+ best_height = height;
+ best_index = i;
+ best_fraction = best_fraction_now;
+ }
+ }
+ if (best_index == -1) {
+ // No caps in first round, try without framerate
+ int target_pixel_count = 1280 * 720;
+ double target_pixel_ratio = 16.0 / 9.0;
+ double best_diversion_factor = 0.0;
+
+ for (int i = 0; i < device.caps.get_size(); i++) {
+ unowned Gst.Structure? that = device.caps.get_structure(i);
+ unowned Gst.CapsFeatures? features = device.caps.get_features(i);
+ if (!that.has_name("video/x-raw") || !features.contains("memory:SystemMemory")) continue;
+ int width = 0, height = 0;
+ if (!that.has_field("width") || !that.get_int("width", out width) || width == 0) continue;
+ if (!that.has_field("height") || !that.get_int("height", out height) || height == 0) continue;
+
+ int pixel_count = width * height;
+ double pixel_count_diversion_factor;
+ if (pixel_count > target_pixel_count)
+ pixel_count_diversion_factor = (double)pixel_count / target_pixel_count;
+ else
+ pixel_count_diversion_factor = (double)target_pixel_count / pixel_count;
+
+ double pixel_ratio;
+ if (width > height)
+ pixel_ratio = (double)width / height;
+ else
+ pixel_ratio = (double)height / width;
+
+ double ratio_diversion_factor;
+ if (pixel_ratio > target_pixel_ratio)
+ ratio_diversion_factor = pixel_ratio / target_pixel_ratio;
+ else
+ ratio_diversion_factor = target_pixel_ratio / pixel_ratio;
+
+ // 1.0 is the best possible value. Any diversion from
+ // target_pixel_count or target_pixel_ratio increases it.
+ double diversion_factor = pixel_count_diversion_factor * ratio_diversion_factor;
+
+ if (best_index == -1 || diversion_factor < best_diversion_factor) {
+ best_index = i;
+ best_width = width;
+ best_height = height;
+ best_diversion_factor = diversion_factor;
+ }
+ }
+ }
+ Gst.Caps res = caps_copy_nth(device.caps, best_index);
+ unowned Gst.Structure? that = res.get_structure(0);
+ Value? framerate = that.get_value("framerate");
+ if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) {
+ that.set_value("framerate", best_fraction);
+ } else if (framerate == null) {
+ // The source does not support framerate negotiation. Just set a
+ // reasonable value. The source might be able to honor it and
+ // should ignore it otherwise.
+ that.@set("framerate", typeof(Gst.Fraction), 30, 1, null);
+ }
+ debug("Selected caps %s", res.to_string());
+ return res;
+ } else if (device.caps.get_size() > 0) {
+ return caps_copy_nth(device.caps, 0);
+ } else {
+ return new Gst.Caps.any();
+ }
+ }
+
+ // Backport from gst_caps_copy_nth added in GStreamer 1.16
+ private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) {
+ Gst.Caps target = new Gst.Caps.empty();
+ target.flags = source.flags;
+ target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy());
+ return target;
+ }
+
+ private void create() {
+ debug("Creating device %s", id);
+ plugin.pause();
+ element = device.create_element(id);
+ if (is_sink) {
+ element.@set("async", false);
+ element.@set("sync", false);
+ }
+ pipe.add(element);
+ device_caps = get_best_caps();
+ if (is_source) {
+ element.@set("do-timestamp", true);
+ filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
+ filter.@set("caps", device_caps);
+ pipe.add(filter);
+ element.link(filter);
+#if WITH_VOICE_PROCESSOR
+ if (media == "audio" && plugin.echoprobe != null) {
+ dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume);
+ dsp.name = @"dsp_$id";
+ pipe.add(dsp);
+ filter.link(dsp);
+ }
+#endif
+ tee = Gst.ElementFactory.make("tee", @"tee_$id");
+ tee.@set("allow-not-linked", true);
+ pipe.add(tee);
+ (dsp ?? filter).link(tee);
+ }
+ if (is_sink && media == "audio") {
+ mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id");
+ pipe.add(mixer);
+ if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) {
+ mixer.link(plugin.echoprobe);
+ plugin.echoprobe.link(element);
+ } else {
+ filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
+ filter.@set("caps", device_caps);
+ pipe.add(filter);
+ mixer.link(filter);
+ filter.link(element);
+ }
+ }
+ plugin.unpause();
+ }
+
+ private void destroy() {
+ if (is_sink) {
+ if (mixer != null) {
+ int linked_sink_pads = 0;
+ mixer.foreach_sink_pad((_, pad) => {
+ if (pad.is_linked()) linked_sink_pads++;
+ return true;
+ });
+ if (linked_sink_pads > 0) {
+ warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads);
+ }
+ mixer.unlink(plugin.echoprobe ?? element);
+ }
+ if (filter != null) {
+ filter.set_locked_state(true);
+ filter.set_state(Gst.State.NULL);
+ filter.unlink(element);
+ pipe.remove(filter);
+ filter = null;
+ }
+ if (plugin.echoprobe != null) {
+ plugin.echoprobe.unlink(element);
+ }
+ }
+ element.set_locked_state(true);
+ element.set_state(Gst.State.NULL);
+ if (filter != null) element.unlink(filter);
+ else if (is_source) element.unlink(tee);
+ pipe.remove(element);
+ element = null;
+ if (mixer != null) {
+ mixer.set_locked_state(true);
+ mixer.set_state(Gst.State.NULL);
+ pipe.remove(mixer);
+ mixer = null;
+ }
+ if (is_source) {
+ if (filter != null) {
+ filter.set_locked_state(true);
+ filter.set_state(Gst.State.NULL);
+ filter.unlink(dsp ?? tee);
+ pipe.remove(filter);
+ filter = null;
+ }
+ if (dsp != null) {
+ dsp.set_locked_state(true);
+ dsp.set_state(Gst.State.NULL);
+ dsp.unlink(tee);
+ pipe.remove(dsp);
+ dsp = null;
+ }
+ if (tee != null) {
+ int linked_src_pads = 0;
+ tee.foreach_src_pad((_, pad) => {
+ if (pad.is_linked()) linked_src_pads++;
+ return true;
+ });
+ if (linked_src_pads != 0) {
+ warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads);
+ }
+ tee.set_locked_state(true);
+ tee.set_state(Gst.State.NULL);
+ pipe.remove(tee);
+ tee = null;
+ }
+ }
+ debug("Destroyed device %s", id);
+ }
+}
diff --git a/main/src/plugins/rtp/gst_fixes.c b/main/src/plugins/rtp/gst_fixes.c
new file mode 100644
index 00000000..86b2b598
--- /dev/null
+++ b/main/src/plugins/rtp/gst_fixes.c
@@ -0,0 +1,10 @@
+#include <gst/video/video.h>
+
+GstVideoInfo *gst_video_frame_get_video_info(GstVideoFrame *frame) {
+ return &frame->info;
+}
+
+void *gst_video_frame_get_data(GstVideoFrame *frame, size_t* length) {
+ *length = frame->info.height * frame->info.stride[0];
+ return frame->data[0];
+} \ No newline at end of file
diff --git a/main/src/plugins/rtp/module.vala b/main/src/plugins/rtp/module.vala
new file mode 100644
index 00000000..be153b10
--- /dev/null
+++ b/main/src/plugins/rtp/module.vala
@@ -0,0 +1,235 @@
+using Gee;
+using Xmpp;
+using Xmpp.Xep;
+
+public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
+ private Set<string> supported_codecs = new HashSet<string>();
+ private Set<string> unsupported_codecs = new HashSet<string>();
+ public Plugin plugin { get; private set; }
+ public CodecUtil codec_util { get {
+ return plugin.codec_util;
+ }}
+
+ public Module(Plugin plugin) {
+ base();
+ this.plugin = plugin;
+ }
+
+ private async bool pipeline_works(string media, string element_desc) {
+ var supported = false;
+ string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output";
+ try {
+ var pipeline = Gst.parse_launch(pipeline_desc);
+ var output = ((Gst.Bin) pipeline).get_by_name("output") as Gst.App.Sink;
+ SourceFunc callback = pipeline_works.callback;
+ var finished = false;
+ output.emit_signals = true;
+ output.new_sample.connect(() => {
+ if (!finished) {
+ finished = true;
+ supported = true;
+ Idle.add(() => {
+ callback();
+ return Source.REMOVE;
+ });
+ }
+ return Gst.FlowReturn.EOS;
+ });
+ pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => {
+ if (message.type == Gst.MessageType.ERROR && !finished) {
+ Error e;
+ string d;
+ message.parse_error(out e, out d);
+ debug("pipeline [%s] failed: %s", pipeline_desc, e.message);
+ debug(d);
+ finished = true;
+ callback();
+ }
+ return true;
+ });
+ Timeout.add(2000, () => {
+ if (!finished) {
+ finished = true;
+ callback();
+ }
+ return Source.REMOVE;
+ });
+ pipeline.set_state(Gst.State.PLAYING);
+ yield;
+ pipeline.set_state(Gst.State.NULL);
+ } catch (Error e) {
+ debug("pipeline [%s] failed: %s", pipeline_desc, e.message);
+ }
+ return supported;
+ }
+
+ public override async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) {
+ string? codec = CodecUtil.get_codec_from_payload(media, payload_type);
+ if (codec == null) return false;
+ if (unsupported_codecs.contains(codec)) return false;
+ if (supported_codecs.contains(codec)) return true;
+
+ string? encode_element = codec_util.get_encode_element_name(media, codec);
+ string? decode_element = codec_util.get_decode_element_name(media, codec);
+ if (encode_element == null || decode_element == null) {
+ warning("No suitable encoder or decoder found for %s", codec);
+ unsupported_codecs.add(codec);
+ return false;
+ }
+
+ string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element);
+ while (!(yield pipeline_works(media, encode_bin))) {
+ debug("%s not suited for encoding %s", encode_element, codec);
+ codec_util.mark_element_unsupported(encode_element);
+ encode_element = codec_util.get_encode_element_name(media, codec);
+ if (encode_element == null) {
+ warning("No suitable encoder found for %s", codec);
+ unsupported_codecs.add(codec);
+ return false;
+ }
+ encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element);
+ }
+ debug("using %s to encode %s", encode_element, codec);
+
+ string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element);
+ while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) {
+ debug("%s not suited for decoding %s", decode_element, codec);
+ codec_util.mark_element_unsupported(decode_element);
+ decode_element = codec_util.get_decode_element_name(media, codec);
+ if (decode_element == null) {
+ warning("No suitable decoder found for %s", codec);
+ unsupported_codecs.add(codec);
+ return false;
+ }
+ decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element);
+ }
+ debug("using %s to decode %s", decode_element, codec);
+
+ supported_codecs.add(codec);
+ return true;
+ }
+
+ public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) {
+ if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true;
+ return false;
+ }
+
+ public override Gee.List<JingleRtp.HeaderExtension> get_suggested_header_extensions(string media) {
+ Gee.List<JingleRtp.HeaderExtension> exts = new ArrayList<JingleRtp.HeaderExtension>();
+ if (media == "video") {
+ exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation"));
+ }
+ return exts;
+ }
+
+ public async void add_if_supported(Gee.List<JingleRtp.PayloadType> list, string media, JingleRtp.PayloadType payload_type) {
+ if (yield is_payload_supported(media, payload_type)) {
+ list.add(payload_type);
+ }
+ }
+
+ public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) {
+ Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func);
+ if (media == "audio") {
+ var opus = new JingleRtp.PayloadType() { channels = 1, clockrate = 48000, name = "opus", id = 111, channels = 2 };
+ opus.parameters["useinbandfec"] = "1";
+ var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 112 };
+ var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 113 };
+ var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 114 };
+ var g722 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "G722", id = 9 };
+ var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 };
+ var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 };
+ yield add_if_supported(list, media, opus);
+ yield add_if_supported(list, media, speex32);
+ yield add_if_supported(list, media, speex16);
+ yield add_if_supported(list, media, speex8);
+ yield add_if_supported(list, media, g722);
+ yield add_if_supported(list, media, pcmu);
+ yield add_if_supported(list, media, pcma);
+ } else if (media == "video") {
+ var rtcp_fbs = new ArrayList<JingleRtp.RtcpFeedback>();
+ rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb"));
+ rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir"));
+ rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack"));
+ rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli"));
+#if ENABLE_H264
+ var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 };
+ yield add_if_supported(list, media, h264);
+ h264.rtcp_fbs.add_all(rtcp_fbs);
+#endif
+#if ENABLE_VP9
+ var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 };
+ vp9.rtcp_fbs.add_all(rtcp_fbs);
+ yield add_if_supported(list, media, vp9);
+#endif
+ var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 };
+ vp8.rtcp_fbs.add_all(rtcp_fbs);
+ yield add_if_supported(list, media, vp8);
+ } else {
+ warning("Unsupported media type: %s", media);
+ }
+ return list;
+ }
+
+ public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List<JingleRtp.PayloadType> payloads) {
+ if (media == "audio" || media == "video") {
+ foreach (JingleRtp.PayloadType type in payloads) {
+ if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone());
+ }
+ } else {
+ warning("Unsupported media type: %s", media);
+ }
+ return null;
+ }
+
+ public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) {
+ var iter = type.rtcp_fbs.iterator();
+ while (iter.next()) {
+ var fb = iter.@get();
+ switch (fb.type_) {
+ case "goog-remb":
+ if (fb.subtype != null) iter.remove();
+ break;
+ case "ccm":
+ if (fb.subtype != "fir") iter.remove();
+ break;
+ case "nack":
+ if (fb.subtype != null && fb.subtype != "pli") iter.remove();
+ break;
+ default:
+ iter.remove();
+ break;
+ }
+ }
+ return type;
+ }
+
+ public override JingleRtp.Stream create_stream(Jingle.Content content) {
+ return plugin.open_stream(content);
+ }
+
+ public override void close_stream(JingleRtp.Stream stream) {
+ var rtp_stream = stream as Rtp.Stream;
+ plugin.close_stream(rtp_stream);
+ }
+
+ public override JingleRtp.Crypto? generate_local_crypto() {
+ uint8[] key_and_salt = new uint8[30];
+ Crypto.randomize(key_and_salt);
+ return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt);
+ }
+
+ public override JingleRtp.Crypto? pick_remote_crypto(Gee.List<JingleRtp.Crypto> cryptos) {
+ foreach (JingleRtp.Crypto crypto in cryptos) {
+ if (crypto.is_valid) return crypto;
+ }
+ return null;
+ }
+
+ public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) {
+ if (remote == null || !remote.is_valid) return null;
+ uint8[] key_and_salt = new uint8[30];
+ Crypto.randomize(key_and_salt);
+ return remote.rekey(key_and_salt);
+ }
+}
diff --git a/main/src/plugins/rtp/participant.vala b/main/src/plugins/rtp/participant.vala
new file mode 100644
index 00000000..1ca13191
--- /dev/null
+++ b/main/src/plugins/rtp/participant.vala
@@ -0,0 +1,39 @@
+using Gee;
+using Xmpp;
+
+public class Dino.Plugins.Rtp.Participant {
+ public Jid full_jid { get; private set; }
+
+ protected Gst.Pipeline pipe;
+ private Map<Stream, uint32> ssrcs = new HashMap<Stream, uint32>();
+
+ public Participant(Gst.Pipeline pipe, Jid full_jid) {
+ this.pipe = pipe;
+ this.full_jid = full_jid;
+ }
+
+ public uint32 get_ssrc(Stream stream) {
+ if (ssrcs.has_key(stream)) {
+ return ssrcs[stream];
+ }
+ return 0;
+ }
+
+ public void set_ssrc(Stream stream, uint32 ssrc) {
+ if (ssrcs.has_key(stream)) {
+ warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]);
+ } else {
+ stream.on_destroy.connect(unset_ssrc);
+ }
+ ssrcs[stream] = ssrc;
+ }
+
+ public void unset_ssrc(Stream stream) {
+ ssrcs.unset(stream);
+ stream.on_destroy.disconnect(unset_ssrc);
+ }
+
+ public string to_string() {
+ return @"participant $full_jid";
+ }
+} \ No newline at end of file
diff --git a/main/src/plugins/rtp/plugin.vala b/main/src/plugins/rtp/plugin.vala
new file mode 100644
index 00000000..d09b7edf
--- /dev/null
+++ b/main/src/plugins/rtp/plugin.vala
@@ -0,0 +1,504 @@
+using Gee;
+using Xmpp;
+using Xmpp.Xep;
+
+public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
+ public Dino.Application app { get; private set; }
+ public CodecUtil codec_util { get; private set; }
+ public Gst.DeviceMonitor? device_monitor { get; private set; }
+ public Gst.Pipeline? pipe { get; private set; }
+ public Gst.Bin? rtpbin { get; private set; }
+ public Gst.Element? echoprobe { get; private set; }
+
+ private Gee.List<Stream> streams = new ArrayList<Stream>();
+ private Gee.List<Device> devices = new ArrayList<Device>();
+ private int64 last_devices_refresh = 0;
+ // private Gee.List<Participant> participants = new ArrayList<Participant>();
+
+ public void registered(Dino.Application app) {
+ this.app = app;
+ this.codec_util = new CodecUtil();
+ app.add_option_group(Gst.init_get_option_group());
+ app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => {
+ list.add(new Module(this));
+ });
+ app.plugin_registry.video_call_plugin = this;
+ }
+
+ private int pause_count = 0;
+ public void pause() {
+// if (pause_count == 0) {
+// debug("Pausing pipe for modifications");
+// pipe.set_state(Gst.State.PAUSED);
+// }
+ pause_count++;
+ }
+ public void unpause() {
+ pause_count--;
+ if (pause_count == 0) {
+ debug("Continue pipe after modifications");
+ pipe.set_state(Gst.State.PLAYING);
+ }
+ if (pause_count < 0) warning("Pause count below zero!");
+ }
+
+ private void handle_existing_devices(Gst.DeviceMonitor device_monitor) {
+ var new_devices = new ArrayList<Device>();
+ foreach (Gst.Device device in device_monitor.get_devices()) {
+ if (device.properties == null) continue;
+ if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue;
+ if (device.properties.get_string("device.class") == "monitor") continue;
+ var pre_device = devices.first_match((it) => it.matches(device));
+ if (pre_device != null) {
+ if (!new_devices.contains(pre_device)) new_devices.add(pre_device);
+ continue;
+ }
+ var new_device = new Device(this, device);
+ devices.add(new_device);
+ new_devices.add(new_device);
+ }
+ devices.retain_all(new_devices);
+ }
+
+ private void refresh_devices() {
+ if (device_monitor != null) return; // No manual refresh needed, we're monitoring actively.
+ if (last_devices_refresh > 0 && last_devices_refresh + 5000000 > get_monotonic_time()) return; // Already current
+ var device_monitor = new Gst.DeviceMonitor();
+ device_monitor.show_all = true;
+ device_monitor.start();
+ handle_existing_devices(device_monitor);
+ device_monitor.stop();
+ last_devices_refresh = get_monotonic_time();
+ }
+
+ private void start_device_monitor() {
+ if (device_monitor != null) return;
+ device_monitor = new Gst.DeviceMonitor();
+ device_monitor.show_all = true;
+ device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message);
+ device_monitor.start();
+ handle_existing_devices(device_monitor);
+ }
+
+ private void stop_device_monitor() {
+ if (device_monitor == null) return;
+ device_monitor.stop();
+ device_monitor = null;
+ }
+
+ private void init_call_pipe() {
+ if (pipe != null) return;
+ debug("Creating call pipe.");
+ start_device_monitor();
+ pipe = new Gst.Pipeline(null);
+
+ // RTP
+ rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin;
+ if (rtpbin == null) {
+ warning("RTP not supported");
+ pipe = null;
+ return;
+ }
+ rtpbin.pad_added.connect(on_rtp_pad_added);
+ rtpbin.@set("latency", 100);
+ rtpbin.@set("do-lost", true);
+// rtpbin.@set("do-sync-event", true);
+ rtpbin.@set("drop-on-latency", true);
+ rtpbin.connect("signal::request-pt-map", request_pt_map, this);
+ pipe.add(rtpbin);
+
+#if WITH_VOICE_PROCESSOR
+ // Audio echo probe
+ echoprobe = new EchoProbe();
+ if (echoprobe != null) pipe.add(echoprobe);
+#endif
+
+ // Pipeline
+ pipe.auto_flush_bus = true;
+ pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => {
+ on_pipe_bus_message(message);
+ return true;
+ });
+ pipe.set_state(Gst.State.PLAYING);
+ }
+
+ public void destroy_call_pipe_if_unused() {
+ if (streams.is_empty && !VideoWidget.has_instances()) {
+ destroy_call_pipe();
+ }
+ }
+
+ private void destroy_call_pipe() {
+ if (pipe == null) return;
+ pipe.set_state(Gst.State.NULL);
+ rtpbin = null;
+#if WITH_VOICE_PROCESSOR
+ echoprobe = null;
+#endif
+ pipe = null;
+ stop_device_monitor();
+ debug("Call pipe destroyed");
+ }
+
+ private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) {
+ debug("request-pt-map");
+ return null;
+ }
+
+ private void on_rtp_pad_added(Gst.Pad pad) {
+ debug("pad added: %s", pad.name);
+ if (pad.name.has_prefix("recv_rtp_src_")) {
+ string[] split = pad.name.split("_");
+ uint8 rtpid = (uint8)int.parse(split[3]);
+ foreach (Stream stream in streams) {
+ if (stream.rtpid == rtpid) {
+ stream.on_ssrc_pad_added((uint32) split[4].to_uint64(), pad);
+ }
+ }
+ }
+ if (pad.name.has_prefix("send_rtp_src_")) {
+ string[] split = pad.name.split("_");
+ uint8 rtpid = (uint8)int.parse(split[3]);
+ debug("pad %s for stream %hhu", pad.name, rtpid);
+ foreach (Stream stream in streams) {
+ if (stream.rtpid == rtpid) {
+ stream.on_send_rtp_src_added(pad);
+ }
+ }
+ }
+ }
+
+ private void on_pipe_bus_message(Gst.Message message) {
+ switch (message.type) {
+ case Gst.MessageType.ERROR:
+ Error error;
+ string str;
+ message.parse_error(out error, out str);
+ warning("Error in pipeline: %s", error.message);
+ debug(str);
+ break;
+ case Gst.MessageType.WARNING:
+ Error error;
+ string str;
+ message.parse_warning(out error, out str);
+ warning("Warning in pipeline: %s", error.message);
+ debug(str);
+ break;
+ case Gst.MessageType.CLOCK_LOST:
+ debug("Clock lost. Restarting");
+ pipe.set_state(Gst.State.READY);
+ pipe.set_state(Gst.State.PLAYING);
+ break;
+ case Gst.MessageType.STATE_CHANGED:
+ // Ignore
+ break;
+ case Gst.MessageType.STREAM_STATUS:
+ Gst.StreamStatusType status;
+ Gst.Element owner;
+ message.parse_stream_status(out status, out owner);
+ if (owner != null) {
+ debug("%s stream changed status to %s", owner.name, status.to_string());
+ }
+ break;
+ case Gst.MessageType.ELEMENT:
+ unowned Gst.Structure struc = message.get_structure();
+ if (struc != null && message.src is Gst.Element) {
+ debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string());
+ }
+ break;
+ case Gst.MessageType.NEW_CLOCK:
+ debug("New clock.");
+ break;
+ case Gst.MessageType.TAG:
+ // Ignore
+ break;
+ case Gst.MessageType.QOS:
+ // Ignore
+ break;
+ case Gst.MessageType.LATENCY:
+ if (message.src != null && message.src.name != null && message.src is Gst.Element) {
+ Gst.Query latency_query = new Gst.Query.latency();
+ if (((Gst.Element)message.src).query(latency_query)) {
+ bool live;
+ Gst.ClockTime min_latency, max_latency;
+ latency_query.parse_latency(out live, out min_latency, out max_latency);
+ debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string());
+ }
+ }
+ break;
+ default:
+ debug("Pipe bus message: %s", message.type.to_string());
+ break;
+ }
+ }
+
+ private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) {
+ Gst.Device? old_gst_device = null;
+ Gst.Device? gst_device = null;
+ Device? device = null;
+ switch (message.type) {
+ case Gst.MessageType.DEVICE_ADDED:
+ message.parse_device_added(out gst_device);
+ if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE;
+ device = new Device(this, gst_device);
+ devices.add(device);
+ break;
+ case Gst.MessageType.DEVICE_CHANGED:
+ message.parse_device_changed(out gst_device, out old_gst_device);
+ device = devices.first_match((it) => it.matches(old_gst_device));
+ if (device != null) device.update(gst_device);
+ break;
+ case Gst.MessageType.DEVICE_REMOVED:
+ message.parse_device_removed(out gst_device);
+ device = devices.first_match((it) => it.matches(gst_device));
+ if (device != null) devices.remove(device);
+ break;
+ default:
+ break;
+ }
+ if (device != null) {
+ devices_changed(device.media, device.is_sink);
+ }
+ return Source.CONTINUE;
+ }
+
+ public uint8 next_free_id() {
+ uint8 rtpid = 0;
+ while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) {
+ rtpid++;
+ }
+ return rtpid;
+ }
+
+ // public Participant get_participant(Jid full_jid, bool self) {
+// foreach (Participant participant in participants) {
+// if (participant.full_jid.equals(full_jid)) {
+// return participant;
+// }
+// }
+// Participant participant;
+// if (self) {
+// participant = new SelfParticipant(pipe, full_jid);
+// } else {
+// participant = new Participant(pipe, full_jid);
+// }
+// participants.add(participant);
+// return participant;
+// }
+
+ public Stream open_stream(Xmpp.Xep.Jingle.Content content) {
+ init_call_pipe();
+ var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters;
+ if (content_params == null) return null;
+ Stream stream;
+ if (content_params.media == "video") {
+ stream = new VideoStream(this, content);
+ } else {
+ stream = new Stream(this, content);
+ }
+ streams.add(stream);
+ return stream;
+ }
+
+ public void close_stream(Stream stream) {
+ streams.remove(stream);
+ stream.destroy();
+ destroy_call_pipe_if_unused();
+ }
+
+ public void shutdown() {
+ stop_device_monitor();
+ destroy_call_pipe();
+ Gst.deinit();
+ }
+
+ public bool supported() {
+ return codec_util.is_element_supported("rtpbin");
+ }
+
+ public VideoCallWidget? create_widget(WidgetType type) {
+ init_call_pipe();
+ if (type == WidgetType.GTK4) {
+ return new VideoWidget(this);
+ }
+ return null;
+ }
+
+ public Gee.List<MediaDevice> get_devices(string media, bool incoming) {
+ refresh_devices();
+ Gee.List<MediaDevice> devices;
+ if (media == "video" && !incoming) {
+ devices = get_video_sources();
+ } else if (media == "audio") {
+ devices = get_audio_devices(incoming);
+ } else {
+ devices = new ArrayList<MediaDevice>();
+ devices.add_all_iterator(this.devices.filter(it => it.media == media && (incoming && it.is_sink || !incoming && it.is_source) && !it.is_monitor));
+ }
+ devices.sort((media_left, media_right) => {
+ return strcmp(media_left.id, media_right.id);
+ });
+
+ return devices;
+ }
+
+ public Gee.List<MediaDevice> get_audio_devices(bool incoming) {
+ ArrayList<MediaDevice> pulse_devices = new ArrayList<MediaDevice>();
+ ArrayList<MediaDevice> other_devices = new ArrayList<MediaDevice>();
+
+ foreach (Device device in devices) {
+ if (device.media != "audio") continue;
+ if (incoming && !device.is_sink || !incoming && !device.is_source) continue;
+
+ // Skip monitors
+ if (device.is_monitor) continue;
+
+ if (device.protocol == DeviceProtocol.PULSEAUDIO) {
+ pulse_devices.add(device);
+ } else {
+ other_devices.add(device);
+ }
+ }
+
+ // If we have any pulseaudio devices, present only those. Don't want duplicated devices from pipewire and pulseaudio.
+ return pulse_devices.size > 0 ? pulse_devices : other_devices;
+ }
+
+ public Gee.List<MediaDevice> get_video_sources() {
+ ArrayList<MediaDevice> pipewire_devices = new ArrayList<MediaDevice>();
+ ArrayList<MediaDevice> other_devices = new ArrayList<MediaDevice>();
+
+ foreach (Device device in devices) {
+ if (device.media != "video") continue;
+ if (device.is_sink) continue;
+
+ // Skip monitors
+ if (device.is_monitor) continue;
+
+ bool is_color = false;
+ for (int i = 0; i < device.device.caps.get_size(); i++) {
+ unowned Gst.Structure structure = device.device.caps.get_structure(i);
+ if (!structure.has_field("format")) continue;
+ // "format" might be an array and get_string() will then return null. We just assume arrays to be fine.
+ string? format = structure.get_string("format");
+ if (format == null || !format.has_prefix("GRAY")) {
+ is_color = true;
+ }
+ }
+
+ // Don't allow grey-scale devices
+ if (!is_color) continue;
+
+ if (device.protocol == DeviceProtocol.PIPEWIRE) {
+ pipewire_devices.add(device);
+ } else {
+ other_devices.add(device);
+ }
+ }
+
+ // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux.
+ return pipewire_devices.size > 0 ? pipewire_devices : other_devices;
+ }
+
+ private int get_max_fps(Device device) {
+ int fps = 0;
+ for (int i = 0; i < device.device.caps.get_size(); i++) {
+ unowned Gst.Structure structure = device.device.caps.get_structure(i);
+
+ if (structure.has_field("framerate")) {
+ Value framerate = structure.get_value("framerate");
+ if (framerate.type() == typeof(Gst.Fraction)) {
+ int num = Gst.Value.get_fraction_numerator(framerate);
+ int den = Gst.Value.get_fraction_denominator(framerate);
+ fps = int.max(fps, num / den);
+ } else if (framerate.type() == typeof(Gst.ValueList)) {
+ for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) {
+ Value fraction = Gst.ValueList.get_value(framerate, j);
+ int num = Gst.Value.get_fraction_numerator(fraction);
+ int den = Gst.Value.get_fraction_denominator(fraction);
+ fps = int.max(fps, num / den);
+ }
+ } else {
+ debug("Unknown type for framerate %s on device %s", framerate.type_name(), device.display_name);
+ }
+ }
+ }
+
+ debug("Max framerate for device %s: %d", device.display_name, fps);
+ return fps;
+ }
+
+ public MediaDevice? get_preferred_device(string media, bool incoming) {
+ Gee.List<Device> devices = new ArrayList<Device>();
+ foreach (MediaDevice media_device in get_devices(media, incoming)) {
+ if (media_device is Device) devices.add((Device)media_device);
+ }
+ if (devices.is_empty) {
+ warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media);
+ return null;
+ }
+
+ // Take default if present
+ foreach (Device device in devices) {
+ if (device.is_default) {
+ debug("Using %s for %s %s as it's default", device.display_name, incoming ? "incoming" : "outgoing", media);
+ return device;
+ }
+ }
+
+ if (media == "video") {
+ // Pick best FPS
+ int max_fps = -1;
+ Device? max_fps_device = null;
+ foreach (Device device in devices) {
+ int fps = get_max_fps(device);
+ if (fps > max_fps || max_fps_device == null) {
+ max_fps = fps;
+ max_fps_device = device;
+ }
+ }
+ debug("Using %s for %s %s as it has max FPS (%d)", max_fps_device.display_name, incoming ? "incoming" : "outgoing", media, max_fps);
+ return max_fps_device;
+ } else {
+ // Pick any
+ Device? device = devices.first();
+ debug("Using %s for %s %s as it's first pick", device.display_name, incoming ? "incoming" : "outgoing", media);
+ return device;
+ }
+ }
+
+ public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream? stream, bool incoming) {
+ Stream? plugin_stream = stream as Stream?;
+ if (plugin_stream == null) return null;
+ MediaDevice? device = incoming ? plugin_stream.output_device : plugin_stream.input_device;
+ return device ?? get_preferred_device(stream.media, incoming);
+ }
+
+ public void dump_dot() {
+ if (pipe == null) return;
+ string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)";
+ Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name);
+ print(@"Stored pipe details as $name\n");
+ }
+
+ public void set_pause(Xmpp.Xep.JingleRtp.Stream? stream, bool pause) {
+ Stream? plugin_stream = stream as Stream?;
+ if (plugin_stream == null) return;
+ if (pause) {
+ plugin_stream.pause();
+ } else {
+ plugin_stream.unpause();
+ }
+ }
+
+ public void set_device(Xmpp.Xep.JingleRtp.Stream? stream, MediaDevice? device) {
+ Device? real_device = device as Device?;
+ Stream? plugin_stream = stream as Stream?;
+ if (real_device == null || plugin_stream == null) return;
+ if (real_device.is_source) {
+ plugin_stream.input_device = real_device;
+ } else if (real_device.is_sink) {
+ plugin_stream.output_device = real_device;
+ }
+ }
+}
diff --git a/main/src/plugins/rtp/stream.vala b/main/src/plugins/rtp/stream.vala
new file mode 100644
index 00000000..9a46d9ff
--- /dev/null
+++ b/main/src/plugins/rtp/stream.vala
@@ -0,0 +1,851 @@
+using Gee;
+using Xmpp;
+
+public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
+ public uint8 rtpid { get; private set; }
+
+ public Plugin plugin { get; private set; }
+ public Gst.Pipeline pipe { get {
+ return plugin.pipe;
+ }}
+ public Gst.Element rtpbin { get {
+ return plugin.rtpbin;
+ }}
+ public CodecUtil codec_util { get {
+ return plugin.codec_util;
+ }}
+ private Gst.App.Sink send_rtp;
+ private Gst.App.Sink send_rtcp;
+ private Gst.App.Src recv_rtp;
+ private Gst.App.Src recv_rtcp;
+ private Gst.Element decode;
+ private Gst.RTP.BaseDepayload decode_depay;
+ private Gst.Element input;
+ private Gst.Pad input_pad;
+ private Gst.Element output;
+ private Gst.Element session;
+
+ private Device _input_device;
+ public Device input_device { get { return _input_device; } set {
+ if (sending && !paused) {
+ var input = this.input;
+ set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null);
+ if (this._input_device != null) this._input_device.unlink(input);
+ }
+ this._input_device = value;
+ }}
+ private Device _output_device;
+ public Device output_device { get { return _output_device; } set {
+ if (output != null) remove_output(output);
+ if (value != null && receiving) add_output(value.link_sink());
+ this._output_device = value;
+ }}
+
+ public bool created { get; private set; default = false; }
+ public bool paused { get; private set; default = false; }
+ private bool push_recv_data = false;
+ private uint our_ssrc = Random.next_int();
+ private int next_seqnum_offset = -1;
+ private uint32 next_timestamp_offset_base = 0;
+ private int64 next_timestamp_offset_stamp = 0;
+ private uint32 next_timestamp_offset { get {
+ if (next_timestamp_offset_base == 0) return 0;
+ int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp;
+ return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate);
+ } }
+ private uint32 participant_ssrc = 0;
+
+ private Gst.Pad recv_rtcp_sink_pad;
+ private Gst.Pad recv_rtp_sink_pad;
+ private Gst.Pad recv_rtp_src_pad;
+ private Gst.Pad send_rtcp_src_pad;
+ private Gst.Pad send_rtp_sink_pad;
+ private Gst.Pad send_rtp_src_pad;
+
+ private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session();
+
+ public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) {
+ base(content);
+ this.plugin = plugin;
+ this.rtpid = plugin.next_free_id();
+
+ content.notify["senders"].connect_after(on_senders_changed);
+ }
+
+ public void on_senders_changed() {
+ if (sending && input == null) {
+ input_device = input_device;
+ }
+ if (receiving && output == null) {
+ output_device = output_device;
+ }
+ }
+
+ public override void create() {
+ plugin.pause();
+
+ // Create i/o if needed
+
+ if (input == null && sending) {
+ input_device = input_device;
+ }
+ if (output == null && receiving && media == "audio") {
+ output_device = output_device;
+ }
+
+ // Create app elements
+ send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink;
+ send_rtp.async = false;
+ send_rtp.caps = CodecUtil.get_caps(media, payload_type, false);
+ send_rtp.emit_signals = true;
+ send_rtp.sync = true;
+ send_rtp.drop = true;
+ send_rtp.wait_on_eos = false;
+ send_rtp.new_sample.connect(on_new_sample);
+ send_rtp.new_serialized_event.connect(on_new_event);
+ send_rtp.connect("signal::eos", on_eos_static, this);
+ pipe.add(send_rtp);
+
+ send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink;
+ send_rtcp.async = false;
+ send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp");
+ send_rtcp.emit_signals = true;
+ send_rtcp.sync = true;
+ send_rtcp.drop = true;
+ send_rtcp.wait_on_eos = false;
+ send_rtcp.new_sample.connect(on_new_sample);
+ send_rtcp.connect("signal::eos", on_eos_static, this);
+ pipe.add(send_rtcp);
+
+ recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src;
+ recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true);
+ recv_rtp.do_timestamp = true;
+ recv_rtp.format = Gst.Format.TIME;
+ recv_rtp.is_live = true;
+ pipe.add(recv_rtp);
+
+ recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src;
+ recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp");
+ recv_rtcp.do_timestamp = true;
+ recv_rtcp.format = Gst.Format.TIME;
+ recv_rtcp.is_live = true;
+ pipe.add(recv_rtcp);
+
+ // Connect RTCP
+ send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid");
+ send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink"));
+ recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid");
+ recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad);
+
+ // Connect input
+ send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid");
+ if (input != null) {
+ input_pad = input.get_request_pad(@"src_$rtpid");
+ input_pad.link(send_rtp_sink_pad);
+ }
+
+ // Connect output
+ decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid");
+ decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay");
+ pipe.add(decode);
+ if (output != null) {
+ decode.link(output);
+ }
+
+ // Connect RTP
+ recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid");
+ recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad);
+
+ created = true;
+ push_recv_data = true;
+ plugin.unpause();
+
+ GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session);
+ if (session != null && remb_enabled) {
+ Object internal_session;
+ session.@get("internal-session", out internal_session);
+ if (internal_session != null) {
+ internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this);
+ }
+ Timeout.add(1000, () => remb_adjust());
+ }
+ if (input_device != null && media == "video") {
+ input_device.update_bitrate(payload_type, target_send_bitrate);
+ }
+ }
+
+ private int last_packets_lost = -1;
+ private uint64 last_packets_received = 0;
+ private uint64 last_octets_received = 0;
+ private uint max_target_receive_bitrate = 0;
+ private int64 last_remb_time = 0;
+ private bool remb_adjust() {
+ unowned Gst.Structure? stats;
+ if (session == null) {
+ debug("Session for %u finished, turning off remb adjustment", rtpid);
+ return Source.REMOVE;
+ }
+ session.get("stats", out stats);
+ if (stats == null) {
+ warning("No stats for session %u", rtpid);
+ return Source.REMOVE;
+ }
+ unowned ValueArray? source_stats;
+ stats.get("source-stats", typeof(ValueArray), out source_stats);
+ if (source_stats == null) {
+ warning("No source-stats for session %u", rtpid);
+ return Source.REMOVE;
+ }
+
+ if (input_device == null) return Source.CONTINUE;
+
+ foreach (Value value in source_stats.values) {
+ unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed();
+ uint32 ssrc;
+ if (!source_stat.get_uint("ssrc", out ssrc)) continue;
+ if (ssrc == participant_ssrc) {
+ int packets_lost;
+ uint64 packets_received, octets_received;
+ source_stat.get_int("packets-lost", out packets_lost);
+ source_stat.get_uint64("packets-received", out packets_received);
+ source_stat.get_uint64("octets-received", out octets_received);
+ int new_lost = packets_lost - last_packets_lost;
+ if (new_lost < 0) new_lost = 0;
+ uint64 new_received = packets_received - last_packets_received;
+ if (packets_received < last_packets_received) new_received = 0;
+ uint64 new_octets = octets_received - last_octets_received;
+ if (octets_received < last_octets_received) octets_received = 0;
+ if (new_received == 0) continue;
+ last_packets_lost = packets_lost;
+ last_packets_received = packets_received;
+ last_octets_received = octets_received;
+ double loss_rate = (double)new_lost / (double)(new_lost + new_received);
+ uint new_target_receive_bitrate;
+ if (new_lost <= 0 || loss_rate < 0.02) {
+ new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate);
+ } else if (loss_rate > 0.1) {
+ new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate);
+ } else {
+ new_target_receive_bitrate = target_receive_bitrate;
+ }
+ if (last_remb_time == 0) {
+ last_remb_time = get_monotonic_time();
+ } else {
+ int64 time_now = get_monotonic_time();
+ int64 time_diff = time_now - last_remb_time;
+ last_remb_time = time_now;
+ uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0);
+ new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate));
+ max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate);
+ new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate);
+ }
+ new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16
+ if (new_target_receive_bitrate != target_receive_bitrate) {
+ target_receive_bitrate = new_target_receive_bitrate;
+ uint8[] data = new uint8[] {
+ 143, 206, 0, 5,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 'R', 'E', 'M', 'B',
+ 1, 0, 0, 0,
+ 0, 0, 0, 0
+ };
+ data[4] = (uint8)((our_ssrc >> 24) & 0xff);
+ data[5] = (uint8)((our_ssrc >> 16) & 0xff);
+ data[6] = (uint8)((our_ssrc >> 8) & 0xff);
+ data[7] = (uint8)(our_ssrc & 0xff);
+ uint8 br_exp = 0;
+ uint32 br_mant = target_receive_bitrate * 1000;
+ uint8 bits = (uint8)Math.log2(br_mant);
+ if (bits > 16) {
+ br_exp = (uint8)bits - 16;
+ br_mant = br_mant >> br_exp;
+ }
+ data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3));
+ data[18] = (uint8)((br_mant >> 8) & 0xff);
+ data[19] = (uint8)(br_mant & 0xff);
+ data[20] = (uint8)((ssrc >> 24) & 0xff);
+ data[21] = (uint8)((ssrc >> 16) & 0xff);
+ data[22] = (uint8)((ssrc >> 8) & 0xff);
+ data[23] = (uint8)(ssrc & 0xff);
+ encrypt_and_send_rtcp(data);
+ }
+ }
+ }
+ return Source.CONTINUE;
+ }
+
+ private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) {
+ if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) {
+ // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03
+ uint8[] data;
+ fci.extract_dup(0, fci.get_size(), out data);
+ if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return;
+ uint8 br_exp = data[5] >> 2;
+ uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7];
+ self.target_send_bitrate = (br_mant << br_exp) / 1000;
+ self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate);
+ }
+ }
+
+ private void prepare_local_crypto() {
+ if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) {
+ crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt);
+ debug("Setting up encryption with key params %s", local_crypto.key_params);
+ }
+ }
+
+ bool flip = false;
+ uint8 rotation = 0;
+
+ private bool on_new_event(Gst.App.Sink sink) {
+ if (sink == null || sink != send_rtp) {
+ return false;
+ }
+ Gst.MiniObject obj = sink.try_pull_object(0);
+ if (obj.type == typeof(Gst.Event)) {
+ unowned Gst.TagList tags;
+ if (((Gst.Event)obj).type == Gst.EventType.TAG) {
+ ((Gst.Event)obj).parse_tag(out tags);
+ Gst.Video.OrientationMethod orientation_method;
+ Gst.Video.Orientation.from_tag(tags, out orientation_method);
+ switch (orientation_method) {
+ case Gst.Video.OrientationMethod.IDENTITY:
+ case Gst.Video.OrientationMethod.VERT:
+ default:
+ rotation = 0;
+ break;
+ case Gst.Video.OrientationMethod.@90R:
+ case Gst.Video.OrientationMethod.UL_LR:
+ rotation = 1;
+ break;
+ case Gst.Video.OrientationMethod.@180:
+ case Gst.Video.OrientationMethod.HORIZ:
+ rotation = 2;
+ break;
+ case Gst.Video.OrientationMethod.@90L:
+ case Gst.Video.OrientationMethod.UR_LL:
+ rotation = 3;
+ break;
+ }
+ switch (orientation_method) {
+ case Gst.Video.OrientationMethod.IDENTITY:
+ case Gst.Video.OrientationMethod.@90R:
+ case Gst.Video.OrientationMethod.@180:
+ case Gst.Video.OrientationMethod.@90L:
+ default:
+ flip = false;
+ break;
+ case Gst.Video.OrientationMethod.VERT:
+ case Gst.Video.OrientationMethod.UL_LR:
+ case Gst.Video.OrientationMethod.HORIZ:
+ case Gst.Video.OrientationMethod.UR_LL:
+ flip = true;
+ break;
+ }
+ }
+ }
+ return false;
+ }
+
+
+ private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) {
+ if (sink == null) {
+ debug("Sink is null");
+ return Gst.FlowReturn.EOS;
+ }
+ if (sink != send_rtp && sink != send_rtcp) {
+ warning("unknown sample");
+ return Gst.FlowReturn.NOT_SUPPORTED;
+ }
+ Gst.Sample sample = sink.pull_sample();
+ Gst.Buffer buffer = sample.get_buffer();
+ if (sink == send_rtp) {
+ uint buffer_ssrc = 0, buffer_seq = 0;
+ Gst.RTP.Buffer rtp_buffer;
+ if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) {
+ buffer_ssrc = rtp_buffer.get_ssrc();
+ buffer_seq = rtp_buffer.get_seq();
+ next_seqnum_offset = rtp_buffer.get_seq() + 1;
+ next_timestamp_offset_base = rtp_buffer.get_timestamp();
+ next_timestamp_offset_stamp = get_monotonic_time();
+ rtp_buffer.unmap();
+ }
+ if (our_ssrc != buffer_ssrc) {
+ warning_once("Sending RTP %s buffer seq %u with SSRC %u when our ssrc is %u", media, buffer_seq, buffer_ssrc, our_ssrc);
+ }
+ }
+
+ if (sink == send_rtp) {
+ Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation");
+ if (ext != null) {
+ buffer = (Gst.Buffer) buffer.make_writable();
+ Gst.RTP.Buffer rtp_buffer;
+ if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.WRITE, out rtp_buffer)) {
+ uint8[] extension_data = new uint8[1];
+ bool camera = false;
+ extension_data[0] = extension_data[0] | (rotation & 0x3);
+ if (flip) extension_data[0] = extension_data[0] | 0x4;
+ if (camera) extension_data[0] = extension_data[0] | 0x8;
+ rtp_buffer.add_extension_onebyte_header(ext.id, extension_data);
+ }
+ }
+ }
+
+ prepare_local_crypto();
+
+ uint8[] data;
+ buffer.extract_dup(0, buffer.get_size(), out data);
+ if (sink == send_rtp) {
+ encrypt_and_send_rtp((owned) data);
+ } else if (sink == send_rtcp) {
+ encrypt_and_send_rtcp((owned) data);
+ }
+ return Gst.FlowReturn.OK;
+ }
+
+ private void encrypt_and_send_rtp(owned uint8[] data) {
+ Bytes bytes;
+ if (crypto_session.has_encrypt) {
+ bytes = new Bytes.take(crypto_session.encrypt_rtp(data));
+ } else {
+ bytes = new Bytes.take(data);
+ }
+ on_send_rtp_data(bytes);
+ }
+
+ private void encrypt_and_send_rtcp(owned uint8[] data) {
+ Bytes bytes;
+ if (crypto_session.has_encrypt) {
+ bytes = new Bytes.take(crypto_session.encrypt_rtcp(data));
+ } else {
+ bytes = new Bytes.take(data);
+ }
+ if (rtcp_mux) {
+ on_send_rtp_data(bytes);
+ } else {
+ on_send_rtcp_data(bytes);
+ }
+ }
+
+ private static Gst.PadProbeReturn drop_probe() {
+ return Gst.PadProbeReturn.DROP;
+ }
+
+ private static void on_eos_static(Gst.App.Sink sink, Stream self) {
+ debug("EOS on %s", sink.name);
+ if (sink == self.send_rtp) {
+ Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; });
+ } else if (sink == self.send_rtcp) {
+ Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; });
+ }
+ }
+
+ private void on_send_rtp_eos() {
+ if (send_rtp_src_pad != null) {
+ send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink"));
+ send_rtp_src_pad = null;
+ }
+ send_rtp.set_locked_state(true);
+ send_rtp.set_state(Gst.State.NULL);
+ // This happens async, so pipe might be gone by now.
+ if (pipe != null) pipe.remove(send_rtp);
+ send_rtp = null;
+ debug("Stopped sending RTP for %u", rtpid);
+ }
+
+ private void on_send_rtcp_eos() {
+ send_rtcp.set_locked_state(true);
+ send_rtcp.set_state(Gst.State.NULL);
+ // This happens async, so pipe might be gone by now.
+ if (pipe != null) pipe.remove(send_rtcp);
+ send_rtcp = null;
+ debug("Stopped sending RTCP for %u", rtpid);
+ }
+
+ public override void destroy() {
+ // Stop network communication
+ push_recv_data = false;
+ if (recv_rtp != null) recv_rtp.end_of_stream();
+ if (recv_rtcp != null) recv_rtcp.end_of_stream();
+ if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample);
+ if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample);
+
+ // Disconnect input device
+ if (input != null) {
+ input_pad.unlink(send_rtp_sink_pad);
+ input.release_request_pad(input_pad);
+ input_pad = null;
+ }
+ if (this._input_device != null) {
+ if (!paused) this._input_device.unlink(input);
+ this._input_device = null;
+ this.input = null;
+ }
+
+ // Inject EOS
+ if (send_rtp_sink_pad != null) {
+ send_rtp_sink_pad.send_event(new Gst.Event.eos());
+ }
+
+ // Disconnect decode
+ if (recv_rtp_src_pad != null) {
+ recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe);
+ recv_rtp_src_pad.unlink(decode.get_static_pad("sink"));
+ }
+
+ // Disconnect output
+ if (output != null) {
+ decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe);
+ decode.unlink(output);
+ }
+
+ // Disconnect output device
+ if (this._output_device != null) {
+ this._output_device.unlink(output);
+ this._output_device = null;
+ }
+ output = null;
+
+ // Destroy decode
+ if (decode != null) {
+ decode.set_locked_state(true);
+ decode.set_state(Gst.State.NULL);
+ pipe.remove(decode);
+ decode = null;
+ decode_depay = null;
+ }
+
+ // Disconnect and remove RTP input
+ if (recv_rtp != null) {
+ recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad);
+ recv_rtp.set_locked_state(true);
+ recv_rtp.set_state(Gst.State.NULL);
+ pipe.remove(recv_rtp);
+ recv_rtp = null;
+ }
+
+ // Disconnect and remove RTCP input
+ if (recv_rtcp != null) {
+ recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad);
+ recv_rtcp.set_locked_state(true);
+ recv_rtcp.set_state(Gst.State.NULL);
+ pipe.remove(recv_rtcp);
+ recv_rtcp = null;
+ }
+
+ // Release rtp pads
+ if (send_rtp_sink_pad != null) {
+ rtpbin.release_request_pad(send_rtp_sink_pad);
+ send_rtp_sink_pad = null;
+ }
+ if (recv_rtp_sink_pad != null) {
+ rtpbin.release_request_pad(recv_rtp_sink_pad);
+ recv_rtp_sink_pad = null;
+ }
+ if (send_rtcp_src_pad != null) {
+ rtpbin.release_request_pad(send_rtcp_src_pad);
+ send_rtcp_src_pad = null;
+ }
+ if (recv_rtcp_sink_pad != null) {
+ rtpbin.release_request_pad(recv_rtcp_sink_pad);
+ recv_rtcp_sink_pad = null;
+ }
+ }
+
+ private void prepare_remote_crypto() {
+ if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) {
+ crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt);
+ debug("Setting up decryption with key params %s", remote_crypto.key_params);
+ }
+ }
+
+ private uint16 previous_incoming_video_orientation_degree = uint16.MAX;
+ public signal void incoming_video_orientation_changed(uint16 degree);
+
+ public override void on_recv_rtp_data(Bytes bytes) {
+ if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) {
+ on_recv_rtcp_data(bytes);
+ return;
+ }
+
+ if (push_recv_data) {
+ prepare_remote_crypto();
+
+ Gst.Buffer buffer;
+ if (crypto_session.has_decrypt) {
+ try {
+ buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtp(bytes.get_data()));
+ } catch (Error e) {
+ warning("%s (%d)", e.message, e.code);
+ return;
+ }
+ } else {
+ buffer = new Gst.Buffer.wrapped_bytes(bytes);
+ }
+
+ Gst.RTP.Buffer rtp_buffer;
+ if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) {
+ uint buffer_ssrc = rtp_buffer.get_ssrc();
+ uint buffer_seq = rtp_buffer.get_seq();
+ if (rtp_buffer.get_extension()) {
+ Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation");
+ if (ext != null) {
+ unowned uint8[] extension_data;
+ if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) {
+ bool camera = (extension_data[0] & 0x8) > 0;
+ bool flip = (extension_data[0] & 0x4) > 0;
+ uint8 rotation = extension_data[0] & 0x3;
+ uint16 rotation_degree = uint16.MAX;
+ switch(rotation) {
+ case 0: rotation_degree = 0; break;
+ case 1: rotation_degree = 90; break;
+ case 2: rotation_degree = 180; break;
+ case 3: rotation_degree = 270; break;
+ }
+ if (rotation_degree != previous_incoming_video_orientation_degree) {
+ incoming_video_orientation_changed(rotation_degree);
+ previous_incoming_video_orientation_degree = rotation_degree;
+ }
+ }
+ }
+ }
+ rtp_buffer.unmap();
+ }
+
+ recv_rtp.push_buffer((owned) buffer);
+ }
+ }
+
+ public override void on_recv_rtcp_data(Bytes bytes) {
+ if (push_recv_data) {
+ prepare_remote_crypto();
+
+ Gst.Buffer buffer;
+ if (crypto_session.has_decrypt) {
+ try {
+ buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtcp(bytes.get_data()));
+ } catch (Error e) {
+ warning("%s (%d)", e.message, e.code);
+ return;
+ }
+ } else {
+ buffer = new Gst.Buffer.wrapped_bytes(bytes);
+ }
+
+ recv_rtcp.push_buffer((owned) buffer);
+ }
+ }
+
+ public override void on_rtp_ready() {
+ // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame.
+ // Send a full frame to let the counterpart display our video asap
+ rtpbin.send_event(new Gst.Event.custom(
+ Gst.EventType.CUSTOM_UPSTREAM,
+ new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null))
+ );
+ }
+
+ public override void on_rtcp_ready() {
+ int rtp_session_id = (int) rtpid;
+ uint64 max_delay = int.MAX;
+ Object rtp_session;
+ bool rtp_sent;
+ GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session);
+ GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent);
+ debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string());
+ }
+
+ public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) {
+ debug("New ssrc %u with pad %s", ssrc, pad.name);
+ if (participant_ssrc != 0 && participant_ssrc != ssrc) {
+ warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc);
+ return;
+ }
+ participant_ssrc = ssrc;
+ recv_rtp_src_pad = pad;
+ if (decode != null) {
+ plugin.pause();
+ debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name);
+ recv_rtp_src_pad.link(decode.get_static_pad("sink"));
+ plugin.unpause();
+ }
+ }
+
+ public void on_send_rtp_src_added(Gst.Pad pad) {
+ send_rtp_src_pad = pad;
+ if (send_rtp != null) {
+ plugin.pause();
+ debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name);
+ send_rtp_src_pad.link(send_rtp.get_static_pad("sink"));
+ plugin.unpause();
+ }
+ }
+
+ public void set_input(Gst.Element? input) {
+ set_input_and_pause(input, paused);
+ }
+
+ private void set_input_and_pause(Gst.Element? input, bool paused) {
+ if (created && this.input != null) {
+ this.input_pad.unlink(send_rtp_sink_pad);
+ this.input.release_request_pad(this.input_pad);
+ this.input_pad = null;
+ this.input = null;
+ }
+
+ this.input = input;
+ this.paused = paused;
+
+ if (created && sending && !paused && input != null) {
+ plugin.pause();
+ input_pad = input.get_request_pad(@"src_$rtpid");
+ input_pad.link(send_rtp_sink_pad);
+ plugin.unpause();
+ }
+ }
+
+ public void pause() {
+ if (paused) return;
+ var input = this.input;
+ set_input_and_pause(null, true);
+ if (input != null && input_device != null) input_device.unlink(input);
+ }
+
+ public void unpause() {
+ if (!paused) return;
+ set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false);
+ input_device.update_bitrate(payload_type, target_send_bitrate);
+ }
+
+ public uint get_participant_ssrc(Xmpp.Jid participant) {
+ if (participant.equals(content.session.peer_full_jid)) {
+ return participant_ssrc;
+ }
+ return 0;
+ }
+
+ ulong block_probe_handler_id = 0;
+ public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) {
+ if (output != null) {
+ critical("add_output() invoked more than once");
+ return;
+ }
+ if (participant != null) {
+ critical("add_output() invoked with participant when not supported");
+ return;
+ }
+ this.output = element;
+ if (created) {
+ plugin.pause();
+ decode.link(element);
+ if (block_probe_handler_id != 0) {
+ decode.get_static_pad("src").remove_probe(block_probe_handler_id);
+ }
+ plugin.unpause();
+ }
+ }
+
+ public virtual void remove_output(Gst.Element element) {
+ if (output != element) {
+ critical("remove_output() invoked without prior add_output()");
+ return;
+ }
+ if (created) {
+ block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe);
+ decode.unlink(element);
+ }
+ if (this._output_device != null) {
+ this._output_device.unlink(element);
+ this._output_device = null;
+ }
+ this.output = null;
+ }
+}
+
+public class Dino.Plugins.Rtp.VideoStream : Stream {
+ private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>();
+ private Gst.Element output_tee;
+ private Gst.Element rotate;
+ private ulong incoming_video_orientation_changed_handler;
+
+ public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) {
+ base(plugin, content);
+ if (media != "video") critical("VideoStream created for non-video media");
+ }
+
+ public override void create() {
+ incoming_video_orientation_changed_handler = incoming_video_orientation_changed.connect(on_video_orientation_changed);
+ plugin.pause();
+ rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid");
+ pipe.add(rotate);
+ output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid");
+ output_tee.@set("allow-not-linked", true);
+ pipe.add(output_tee);
+ rotate.link(output_tee);
+ add_output(rotate);
+ base.create();
+ foreach (Gst.Element output in outputs) {
+ output_tee.link(output);
+ }
+ plugin.unpause();
+ }
+
+ private void on_video_orientation_changed(uint16 degree) {
+ if (rotate != null) {
+ switch (degree) {
+ case 0:
+ rotate.@set("method", 0);
+ break;
+ case 90:
+ rotate.@set("method", 1);
+ break;
+ case 180:
+ rotate.@set("method", 2);
+ break;
+ case 270:
+ rotate.@set("method", 3);
+ break;
+ }
+ }
+ }
+
+ public override void destroy() {
+ foreach (Gst.Element output in outputs) {
+ output_tee.unlink(output);
+ }
+ base.destroy();
+ rotate.set_locked_state(true);
+ rotate.set_state(Gst.State.NULL);
+ rotate.unlink(output_tee);
+ pipe.remove(rotate);
+ rotate = null;
+ output_tee.set_locked_state(true);
+ output_tee.set_state(Gst.State.NULL);
+ pipe.remove(output_tee);
+ output_tee = null;
+ disconnect(incoming_video_orientation_changed_handler);
+ }
+
+ public override void add_output(Gst.Element element, Xmpp.Jid? participant) {
+ if (element == output_tee || element == rotate) {
+ base.add_output(element);
+ return;
+ }
+ outputs.add(element);
+ if (output_tee != null) {
+ output_tee.link(element);
+ }
+ }
+
+ public override void remove_output(Gst.Element element) {
+ if (element == output_tee || element == rotate) {
+ base.remove_output(element);
+ return;
+ }
+ outputs.remove(element);
+ if (output_tee != null) {
+ output_tee.unlink(element);
+ }
+ }
+}
diff --git a/main/src/plugins/rtp/video_widget.vala b/main/src/plugins/rtp/video_widget.vala
new file mode 100644
index 00000000..af6eb14e
--- /dev/null
+++ b/main/src/plugins/rtp/video_widget.vala
@@ -0,0 +1,293 @@
+private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame);
+[CCode (array_length_type = "size_t", type = "void*")]
+private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame);
+
+public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object {
+ private Gdk.Paintable image;
+ private double pixel_aspect_ratio;
+
+ public override Gdk.PaintableFlags get_flags() {
+ return 0;
+ }
+
+ public void snapshot(Gdk.Snapshot snapshot, double width, double height) {
+ if (image != null) image.snapshot(snapshot, width, height);
+ }
+
+ public override Gdk.Paintable get_current_image() {
+ if (image != null) return image;
+ return Gdk.Paintable.empty(0, 0);
+ }
+
+ public override int get_intrinsic_width() {
+ if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width());
+ return 0;
+ }
+
+ public override int get_intrinsic_height() {
+ if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height());
+ return 0;
+ }
+
+ public override double get_intrinsic_aspect_ratio() {
+ if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio();
+ return 0.0;
+ }
+
+ public override void dispose() {
+ image = null;
+ base.dispose();
+ }
+
+ private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) {
+ if (paintable == image) return;
+ bool size_changed = image == null ||
+ this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() ||
+ image.get_intrinsic_height() != paintable.get_intrinsic_height() ||
+ image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio();
+
+ if (image != null) this.image.dispose();
+ this.image = paintable;
+ this.pixel_aspect_ratio = pixel_aspect_ratio;
+
+ if (size_changed) invalidate_size();
+ invalidate_contents();
+ }
+
+ public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) {
+ Idle.add(() => {
+ set_paintable(texture, pixel_aspect_ratio);
+ return Source.REMOVE;
+ }, Priority.DEFAULT);
+ }
+}
+
+public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink {
+ internal Paintable paintable = new Paintable();
+ private Gst.Video.Info info = new Gst.Video.Info();
+
+ class construct {
+ set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team <team@dino.im>");
+ add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }")));
+ }
+
+ construct {
+ set_drop_out_of_segment(false);
+ }
+
+ public override bool set_info(Gst.Caps caps, Gst.Video.Info info) {
+ this.info = info;
+ return true;
+ }
+
+ public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) {
+ if (buffer.pts != -1) {
+ start = buffer.pts;
+ if (buffer.duration != -1) {
+ end = start + buffer.duration;
+ } else if (info.fps_n > 0) {
+ end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n);
+ }
+ }
+ }
+
+ public override Gst.Caps get_caps(Gst.Caps? filter) {
+ Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }");
+
+ if (filter != null) {
+ return filter.intersect(caps, Gst.CapsIntersectMode.FIRST);
+ } else {
+ return caps;
+ }
+ }
+
+ private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) {
+ switch (format) {
+ case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8;
+ case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8;
+ case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8;
+ case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8;
+ case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8;
+ case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8;
+ default:
+ warning("Unsupported video format: %s", format.to_string());
+ return Gdk.MemoryFormat.A8R8G8B8;
+ }
+ }
+
+ private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) {
+ Gst.Video.Frame frame = Gst.Video.Frame();
+ Gdk.Texture texture;
+
+ if (frame.map(info, buffer, Gst.MapFlags.READ)) {
+ unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame);
+ Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame));
+ texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]);
+ pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d);
+ frame.unmap();
+ } else {
+ texture = null;
+ }
+ return texture;
+ }
+
+ private void queue_buffer(Gst.Buffer buf) {
+ double pixel_aspect_ratio;
+ Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio);
+ if (texture != null) {
+ paintable.queue_set_texture(texture, pixel_aspect_ratio);
+ }
+ }
+
+ public override Gst.FlowReturn show_frame(Gst.Buffer buf) {
+ @lock.lock();
+ queue_buffer(buf);
+ @lock.unlock();
+
+ return Gst.FlowReturn.OK;
+ }
+}
+
+public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget {
+ private const int RECAPS_AFTER_CHANGE = 5;
+ private static uint last_id = 0;
+
+ public uint id { get; private set; }
+ public Plugin plugin { get; private set; }
+ public Gst.Pipeline pipe { get {
+ return plugin.pipe;
+ }}
+
+ private bool attached;
+ private Device? connected_device;
+ private Gst.Element? connected_device_element;
+ private Stream? connected_stream;
+ private Gst.Element prepare;
+ private Gst.Caps last_input_caps;
+ private Gst.Caps last_caps;
+ private int recaps_since_change;
+ private Sink sink;
+ private Gtk.Picture widget;
+
+ private static uint active_widgets = 0;
+
+ public static bool has_instances() {
+ return active_widgets > 0;
+ }
+
+ private static void notify_weak(Object widget_object) {
+ if (active_widgets > 0) {
+ debug("Video widget %p destroyed. left=%u", widget_object, active_widgets);
+ active_widgets--;
+ }
+ if (active_widgets == 0 && widget_object is VideoWidget) {
+ ((VideoWidget)widget_object).plugin.destroy_call_pipe_if_unused();
+ }
+ }
+
+ public VideoWidget(Plugin plugin) {
+ this.plugin = plugin;
+ this.layout_manager = new Gtk.BinLayout();
+
+ id = last_id++;
+ sink = new Sink() { async = false, sync = true };
+ widget = new Gtk.Picture.for_paintable(sink.paintable);
+ widget.insert_after(this, null);
+ active_widgets++;
+ debug("Video widget %p created. total=%u", this, active_widgets);
+ this.weak_ref(notify_weak);
+ }
+
+ public void input_caps_changed(GLib.Object pad, ParamSpec spec) {
+ Gst.Caps? caps = ((Gst.Pad)pad).caps;
+ if (caps == null) {
+ debug("Input: No caps");
+ return;
+ }
+
+ int width, height;
+ caps.get_structure(0).get_int("width", out width);
+ caps.get_structure(0).get_int("height", out height);
+ debug("Input resolution changed: %ix%i", width, height);
+ // Invoke signal on GTK main loop as recipients are likely to use it for doing GTK operations
+ Idle.add(() => {
+ resolution_changed(width, height);
+ return Source.REMOVE;
+ });
+ last_input_caps = caps;
+ }
+
+ public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) {
+ if (sink == null) return;
+ detach();
+ if (stream.media != "video") return;
+ connected_stream = stream as Stream?;
+ if (connected_stream == null) return;
+ plugin.pause();
+ pipe.add(sink);
+ prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true);
+ prepare.name = @"video_widget_$(id)_prepare";
+ prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+ pipe.add(prepare);
+ connected_stream.add_output(prepare);
+ prepare.link(sink);
+ sink.set_locked_state(false);
+ plugin.unpause();
+ attached = true;
+ }
+
+ public void display_device(MediaDevice media_device) {
+ if (sink == null) return;
+ detach();
+ connected_device = media_device as Device;
+ if (connected_device == null) return;
+ plugin.pause();
+ pipe.add(sink);
+ prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
+ prepare.name = @"video_widget_$(id)_prepare";
+ if (prepare is Gst.Bin) {
+ ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+ } else {
+ prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+ }
+ pipe.add(prepare);
+ connected_device_element = connected_device.link_source();
+ connected_device_element.link(prepare);
+ prepare.link(sink);
+ sink.set_locked_state(false);
+ plugin.unpause();
+ attached = true;
+ }
+
+ public void detach() {
+ if (sink == null) return;
+ if (attached) {
+ debug("Detaching");
+ if (connected_stream != null) {
+ connected_stream.remove_output(prepare);
+ connected_stream = null;
+ }
+ if (connected_device != null) {
+ connected_device_element.unlink(sink);
+ connected_device_element = null;
+ connected_device.unlink();
+ connected_device = null;
+ }
+ prepare.set_locked_state(true);
+ prepare.set_state(Gst.State.NULL);
+ pipe.remove(prepare);
+ prepare = null;
+ sink.set_locked_state(true);
+ sink.set_state(Gst.State.NULL);
+ pipe.remove(sink);
+ attached = false;
+ }
+ }
+
+ public override void dispose() {
+ detach();
+ if (widget != null) widget.unparent();
+ widget = null;
+ sink = null;
+ }
+}
diff --git a/main/src/plugins/rtp/voice_processor.vala b/main/src/plugins/rtp/voice_processor.vala
new file mode 100644
index 00000000..064684c5
--- /dev/null
+++ b/main/src/plugins/rtp/voice_processor.vala
@@ -0,0 +1,209 @@
+using Gst;
+
+namespace Dino.Plugins.Rtp {
+public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf);
+}
+
+public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter {
+ private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
+ private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
+ public Audio.Info audio_info { get; private set; }
+ public signal void on_new_buffer(owned Buffer buffer);
+ public signal void on_new_delay(int delay);
+ private uint period_samples;
+ private uint period_size;
+ public int delay { get; private set; }
+ private Base.Adapter adapter = new Base.Adapter();
+
+ static construct {
+ add_static_pad_template(sink_template);
+ add_static_pad_template(src_template);
+ set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team <contact@dino.im>");
+ }
+
+ construct {
+ set_passthrough(true);
+ }
+
+ public override bool setup(Audio.Info info) {
+ audio_info = info;
+ period_samples = info.rate / 100; // 10ms buffers
+ period_size = period_samples * info.bpf;
+ return true;
+ }
+
+ public override bool src_event(owned Event event) {
+ Query query = new Query.latency();
+ if (event.type == EventType.LATENCY && srcpad != null && srcpad.query(query)) {
+ ClockTime upstream_latency;
+ query.parse_latency(null, out upstream_latency, null);
+ int delay = this.delay;
+ if (upstream_latency != CLOCK_TIME_NONE) {
+ delay = (int) (upstream_latency / MSECOND);
+ } else {
+ delay = 0;
+ }
+ if (delay != this.delay) {
+ debug("Delay adjusted from %ms to %dms", this.delay, delay);
+ this.delay = delay;
+ on_new_delay(delay);
+ }
+ }
+ return base.src_event((owned) event);
+ }
+
+ public override FlowReturn transform_ip(Buffer buf) {
+ lock (adapter) {
+ adapter.push(adjust_to_running_time(this, buf));
+ while (adapter.available() > period_size) {
+ on_new_buffer(adapter.take_buffer(period_size));
+ }
+ }
+ return FlowReturn.OK;
+ }
+
+ public override bool stop() {
+ adapter.clear();
+ return true;
+ }
+}
+
+public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
+ private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
+ private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
+ public Audio.Info audio_info { get; private set; }
+ private ulong process_outgoing_buffer_handler_id;
+ private ulong process_stream_delay_handler_id;
+ private uint adjust_delay_timeout_id;
+ private uint period_samples;
+ private uint period_size;
+ private Base.Adapter adapter = new Base.Adapter();
+ private EchoProbe? echo_probe;
+ private Audio.StreamVolume? stream_volume;
+ private ClockTime last_reverse;
+ private void* native;
+
+ static construct {
+ add_static_pad_template(sink_template);
+ add_static_pad_template(src_template);
+ set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team <contact@dino.im>");
+ }
+
+ construct {
+ set_passthrough(false);
+ }
+
+ public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) {
+ this.echo_probe = echo_probe;
+ this.stream_volume = stream_volume;
+ }
+
+ private static extern void* init_native(int stream_delay);
+ private static extern void destroy_native(void* native);
+ private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer);
+ private static extern void process_stream(void* native, Audio.Info info, Buffer buffer);
+ private static extern void adjust_stream_delay(void* native);
+ private static extern void set_stream_delay(void* native, int delay);
+ private static extern void notify_gain_level(void* native, int gain_level);
+ private static extern int get_suggested_gain_level(void* native);
+ private static extern bool get_stream_has_voice(void* native);
+
+ public override bool setup(Audio.Info info) {
+ debug("VoiceProcessor.setup(%s)", info.to_caps().to_string());
+ audio_info = info;
+ period_samples = info.rate / 100; // 10ms buffers
+ period_size = period_samples * info.bpf;
+ adapter.clear();
+ return true;
+ }
+
+ public override bool start() {
+ native = init_native(echo_probe.delay);
+ if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) {
+ process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer);
+ }
+ if (process_stream_delay_handler_id == 0 && echo_probe != null) {
+ process_stream_delay_handler_id = echo_probe.on_new_delay.connect(process_stream_delay);
+ }
+ if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) {
+ stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume;
+ }
+ return true;
+ }
+
+ private bool adjust_delay() {
+ if (native != null) {
+ adjust_stream_delay(native);
+ return Source.CONTINUE;
+ } else {
+ adjust_delay_timeout_id = 0;
+ return Source.REMOVE;
+ }
+ }
+
+ private void process_outgoing_buffer(owned Buffer buffer) {
+ if (buffer.pts != uint64.MAX) {
+ last_reverse = buffer.pts;
+ }
+ if (native != null) {
+ buffer = (Buffer) buffer.make_writable();
+ analyze_reverse_stream(native, echo_probe.audio_info, buffer);
+ }
+ if (adjust_delay_timeout_id == 0 && echo_probe != null) {
+ adjust_delay_timeout_id = Timeout.add(1000, adjust_delay);
+ }
+ }
+
+ private void process_stream_delay(int stream_delay) {
+ if (native != null) {
+ set_stream_delay(native, stream_delay);
+ }
+ }
+
+ public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) {
+ lock (adapter) {
+ if (is_discont) {
+ adapter.clear();
+ }
+ adapter.push(adjust_to_running_time(this, input));
+ }
+ return FlowReturn.OK;
+ }
+
+ public override FlowReturn generate_output(out Buffer output_buffer) {
+ lock (adapter) {
+ if (adapter.available() >= period_size) {
+ output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable();
+ int old_gain_level = 0;
+ if (stream_volume != null) {
+ old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0);
+ notify_gain_level(native, old_gain_level);
+ }
+ process_stream(native, audio_info, output_buffer);
+ if (stream_volume != null) {
+ int new_gain_level = get_suggested_gain_level(native);
+ if (old_gain_level != new_gain_level) {
+ debug("Gain: %i -> %i", old_gain_level, new_gain_level);
+ stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0);
+ }
+ }
+ }
+ }
+ return FlowReturn.OK;
+ }
+
+ public override bool stop() {
+ if (process_outgoing_buffer_handler_id != 0) {
+ echo_probe.disconnect(process_outgoing_buffer_handler_id);
+ process_outgoing_buffer_handler_id = 0;
+ }
+ if (adjust_delay_timeout_id != 0) {
+ Source.remove(adjust_delay_timeout_id);
+ adjust_delay_timeout_id = 0;
+ }
+ adapter.clear();
+ destroy_native(native);
+ native = null;
+ return true;
+ }
+} \ No newline at end of file
diff --git a/main/src/plugins/rtp/voice_processor_native.cpp b/main/src/plugins/rtp/voice_processor_native.cpp
new file mode 100644
index 00000000..71cc8a54
--- /dev/null
+++ b/main/src/plugins/rtp/voice_processor_native.cpp
@@ -0,0 +1,256 @@
+#include <algorithm>
+#include <gst/gst.h>
+#include <gst/audio/audio.h>
+#if defined(WEBRTC0)
+#include <webrtc/modules/audio_processing/include/audio_processing.h>
+#include <webrtc/modules/interface/module_common_types.h>
+#include <webrtc/system_wrappers/include/trace.h>
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+#include <modules/audio_processing/include/audio_processing.h>
+#else
+#error "Need to define WEBRTC0, WEBRTC1 or WEBRTC2"
+#endif
+
+#define SAMPLE_RATE 48000
+#define SAMPLE_CHANNELS 1
+
+struct _DinoPluginsRtpVoiceProcessorNative {
+#if defined(WEBRTC0)
+ webrtc::AudioProcessing *apm;
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm;
+#endif
+ gint stream_delay = 0;
+ gint last_median = 0;
+ gint last_poor_delays = 0;
+};
+
+extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) {
+ GstBuffer *copy = gst_buffer_copy(buffer);
+ GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer));
+ return copy;
+}
+
+extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) {
+ auto *native = new _DinoPluginsRtpVoiceProcessorNative();
+ native->stream_delay = stream_delay;
+#if defined(WEBRTC0)
+ webrtc::Config config;
+ config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true));
+ config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85));
+ native->apm = webrtc::AudioProcessing::Create(config);
+
+ webrtc::AudioProcessing *apm = native->apm;
+ webrtc::ProcessingConfig pconfig;
+ pconfig.streams[webrtc::ProcessingConfig::kInputStream] =
+ webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ pconfig.streams[webrtc::ProcessingConfig::kOutputStream] =
+ webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] =
+ webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] =
+ webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ apm->Initialize(pconfig);
+ apm->high_pass_filter()->Enable(true);
+ apm->echo_cancellation()->enable_drift_compensation(false);
+ apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression);
+ apm->echo_cancellation()->enable_delay_logging(true);
+ apm->echo_cancellation()->Enable(true);
+ apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate);
+ apm->noise_suppression()->Enable(true);
+ apm->gain_control()->set_analog_level_limits(0, 255);
+ apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog);
+ apm->gain_control()->set_target_level_dbfs(3);
+ apm->gain_control()->set_compression_gain_db(9);
+ apm->gain_control()->enable_limiter(true);
+ apm->gain_control()->Enable(true);
+ apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood);
+ apm->voice_detection()->Enable(true);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ webrtc::AudioProcessing::Config config;
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = webrtc::AudioProcessingBuilder().Create();
+ native->apm = apm;
+ config.high_pass_filter.enabled = true;
+ config.echo_canceller.enabled = true;
+ config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::Level::kModerate;
+ config.noise_suppression.enabled = true;
+ config.gain_controller1.target_level_dbfs = 3;
+ config.gain_controller1.compression_gain_db = 9;
+ config.gain_controller1.enable_limiter = true;
+ config.gain_controller1.enabled = true;
+#ifdef WEBRTC1
+ config.level_estimation.enabled = true;
+ config.voice_detection.enabled = true;
+#endif
+ apm->ApplyConfig(config);
+#endif
+ return native;
+}
+
+extern "C" void
+dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ int err;
+
+#if defined(WEBRTC0)
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ webrtc::AudioProcessing *apm = native->apm;
+ GstMapInfo map;
+ gst_buffer_map(buffer, &map, GST_MAP_READ);
+
+ webrtc::AudioFrame frame;
+ frame.num_channels_ = info->channels;
+ frame.sample_rate_hz_ = info->rate;
+ frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf;
+ memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
+
+ apm->set_stream_delay_ms (native->stream_delay);
+ err = apm->AnalyzeReverseStream(&frame);
+
+ gst_buffer_unmap(buffer, &map);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+#ifdef WEBRTC1
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+#else
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS);
+#endif
+ GstAudioBuffer abuf;
+ if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) {
+ g_warning("voice_processor_native.cpp: analyze_reverse_stream: gst_audio_buffer_map failed");
+ return;
+ }
+
+ apm->set_stream_delay_ms (native->stream_delay);
+ auto * const data = (int16_t * const) abuf.planes[0];
+ err = apm->ProcessReverseStream (data, config, config, data);
+
+ gst_audio_buffer_unmap (&abuf);
+#endif
+ if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err);
+}
+
+extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+#if defined(WEBRTC0)
+ webrtc::AudioProcessing *apm = native->apm;
+ apm->gain_control()->set_stream_analog_level(gain_level);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+ apm->set_stream_analog_level(gain_level);
+#endif
+}
+
+extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ int level = 0;
+#if defined(WEBRTC0)
+ webrtc::AudioProcessing *apm = native->apm;
+ level = apm->gain_control()->stream_analog_level();
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+ level = apm->recommended_stream_analog_level();
+#endif
+ return level;
+}
+
+extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ bool has_voice = false;
+#if defined(WEBRTC0)
+ webrtc::AudioProcessing *apm = native->apm;
+ has_voice = apm->voice_detection()->stream_has_voice();
+#elif defined(WEBRTC1)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+ webrtc::AudioProcessingStats stats = apm->GetStatistics ();
+ has_voice = stats.voice_detected.value_or(false);
+#endif
+ return has_voice;
+}
+
+extern "C" void dino_plugins_rtp_voice_processor_set_stream_delay(void *native_ptr, gint stream_delay) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ native->stream_delay = stream_delay;
+}
+
+extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ int median, std, poor_delays;
+ float fraction_poor_delays;
+#if defined(WEBRTC0)
+ webrtc::AudioProcessing *apm = native->apm;
+ apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays);
+ poor_delays = (int)(fraction_poor_delays * 100.0);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+ webrtc::AudioProcessingStats stats = apm->GetStatistics();
+ median = stats.delay_median_ms.value_or(-1);
+ std = stats.delay_standard_deviation_ms.value_or(-1);
+ fraction_poor_delays = (float) stats.divergent_filter_fraction.value_or(-1.0);
+ poor_delays = (int) (fraction_poor_delays * 100.0);
+#endif
+ if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return;
+ g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays);
+ native->last_median = median;
+ native->last_poor_delays = poor_delays;
+#if defined(WEBRTC0)
+ if (poor_delays > 90) {
+ native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384);
+ g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay);
+ }
+#endif
+}
+
+extern "C" void
+dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+ int err;
+
+#if defined(WEBRTC0)
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+ webrtc::AudioProcessing *apm = native->apm;
+ GstMapInfo map;
+ gst_buffer_map(buffer, &map, GST_MAP_READWRITE);
+
+ webrtc::AudioFrame frame;
+ frame.num_channels_ = info->channels;
+ frame.sample_rate_hz_ = info->rate;
+ frame.samples_per_channel_ = info->rate / 100;
+ memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
+
+ apm->set_stream_delay_ms(native->stream_delay);
+ err = apm->ProcessStream(&frame);
+ if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf);
+
+ gst_buffer_unmap(buffer, &map);
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm = native->apm;
+#ifdef WEBRTC1
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
+#else
+ webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS);
+#endif
+ GstAudioBuffer abuf;
+ if (!gst_audio_buffer_map (&abuf, info, buffer, GST_MAP_READWRITE)) {
+ g_warning("voice_processor_native.cpp: process_stream: gst_audio_buffer_map failed");
+ return;
+ }
+
+ auto * const data = (int16_t * const) abuf.planes[0];
+ err = apm->ProcessStream (data, config, config, data);
+
+ gst_audio_buffer_unmap (&abuf);
+#endif
+ if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err);
+}
+
+extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) {
+ auto *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
+#if defined(WEBRTC0)
+ delete native->apm;
+ native->apm = NULL;
+#elif defined(WEBRTC1) || defined(WEBRTC2)
+ native->apm = nullptr;
+#endif
+ delete native;
+} \ No newline at end of file